broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declared, shadow, operation):
    class C:
        def __len__(self): return declared
    obj=C()
    obj.__len__=lambda: shadow
    if operation=='builtin': return len(obj)
    if operation=='direct': return obj.__len__()
    if operation=='class': return obj.__len__()
    return bool(obj)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('implicit ignores shadow', solve(2, 9, 'builtin'), 2)
check('direct sees shadow', solve(2, 9, 'direct'), 9)
check('class explicit sees declared', solve(2, 9, 'class'), 2)
check('zero native length false', solve(0, 9, 'truth'), False)
check('positive native length true', solve(2, 0, 'truth'), True)
check('zero shadow direct', solve(2, 0, 'direct'), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-special-method-type-lookup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declared, shadow, operation):
    class C:
        def __len__(self): return declared
    obj=C()
    obj.__len__=lambda: shadow
    if operation=='builtin': return len(obj)
    if operation=='direct': return obj.__len__()
    if operation=='class': return C.__len__(obj)
    return bool(obj.__len__())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('implicit ignores shadow', solve(2, 9, 'builtin'), 2)
check('direct sees shadow', solve(2, 9, 'direct'), 9)
check('class explicit sees declared', solve(2, 9, 'class'), 2)
check('zero native length false', solve(0, 9, 'truth'), False)
check('positive native length true', solve(2, 0, 'truth'), True)
check('zero shadow direct', solve(2, 0, 'direct'), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Uses the instance override for an explicit class method invocation and changes the required Python-language result.

A per-instance __len__ shadows direct attribute calls but does not change implicit len or truth testing. Return the selected native result; declared is an integer from 0 through sys.maxsize, and shadow is any nonnegative integer. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declared, shadow, operation):
    class C:
        def __len__(self): return declared
    obj=C()
    obj.__len__=lambda: shadow
    if operation=='builtin': return len(obj)
    if operation=='direct': return obj.__len__()
    if operation=='class': return C.__len__(obj)
    return bool(obj.__len__())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('implicit ignores shadow', solve(2, 9, 'builtin'), 2)
check('direct sees shadow', solve(2, 9, 'direct'), 9)
check('class explicit sees declared', solve(2, 9, 'class'), 2)
check('zero native length false', solve(0, 9, 'truth'), False)
check('positive native length true', solve(2, 0, 'truth'), True)
check('zero shadow direct', solve(2, 0, 'direct'), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xl-special-method-type-lookup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declared, shadow, operation):
    class C:
        def __len__(self): return declared
    obj=C()
    obj.__len__=lambda: shadow
    if operation=='builtin': return len(obj)
    if operation=='direct': return obj.__len__()
    if operation=='class': return C.__len__(obj)
    return obj is not None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('implicit ignores shadow', solve(2, 9, 'builtin'), 2)
check('direct sees shadow', solve(2, 9, 'direct'), 9)
check('class explicit sees declared', solve(2, 9, 'class'), 2)
check('zero native length false', solve(0, 9, 'truth'), False)
check('positive native length true', solve(2, 0, 'truth'), True)
check('zero shadow direct', solve(2, 0, 'direct'), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Bases truth on a shadowed instance __len__ and changes the required Python-language result.

A per-instance __len__ shadows direct attribute calls but does not change implicit len or truth testing. Return the selected native result; declared is an integer from 0 through sys.maxsize, and shadow is any nonnegative integer. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declared, shadow, operation):
    class C:
        def __len__(self): return declared
    obj=C()
    obj.__len__=lambda: shadow
    if operation=='builtin': return len(obj)
    if operation=='direct': return obj.__len__()
    if operation=='class': return C.__len__(obj)
    return obj is not None
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('implicit ignores shadow', solve(2, 9, 'builtin'), 2)
check('direct sees shadow', solve(2, 9, 'direct'), 9)
check('class explicit sees declared', solve(2, 9, 'class'), 2)
check('zero native length false', solve(0, 9, 'truth'), False)
check('positive native length true', solve(2, 0, 'truth'), True)
check('zero shadow direct', solve(2, 0, 'direct'), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-special-method-type-lookup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declared, shadow, operation):
    class C:
        def __len__(self): return declared
    obj=C()
    obj.__len__=lambda: shadow
    if operation=='builtin': return obj.__len__()
    if operation=='direct': return obj.__len__()
    if operation=='class': return C.__len__(obj)
    return bool(obj)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('implicit ignores shadow', solve(2, 9, 'builtin'), 2)
check('direct sees shadow', solve(2, 9, 'direct'), 9)
check('class explicit sees declared', solve(2, 9, 'class'), 2)
check('zero native length false', solve(0, 9, 'truth'), False)
check('positive native length true', solve(2, 0, 'truth'), True)
check('zero shadow direct', solve(2, 0, 'direct'), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Treats every allocated object as true regardless of its zero native length and changes the required Python-language result.

A per-instance __len__ shadows direct attribute calls but does not change implicit len or truth testing. Return the selected native result; declared is an integer from 0 through sys.maxsize, and shadow is any nonnegative integer. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, start):
    class Root:
        def trace(self): return ['Root']
    class A(Root):
        def trace(self): return ['A']+Root.trace(self)
    class B(Root):
        def trace(self): return ['B']+super().trace()
    def leaf_trace(self): return ['Leaf']+super(Leaf,self).trace()
    Leaf=type('Leaf',(A,B) if order=='AB' else (B,A),{'trace':leaf_trace})
    obj=Leaf()
    if start=='leaf': return obj.trace()
    if start=='after-leaf': return super(Leaf,obj).trace()
    if start=='after-A': return super(A,obj).trace()
    return super(B,obj).trace()
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('left-first diamond', solve('AB', 'leaf'), ['Leaf', 'A', 'B', 'Root'])
check('right-first diamond', solve('BA', 'leaf'), ['Leaf', 'B', 'A', 'Root'])
check('super after leaf', solve('AB', 'after-leaf'), ['A', 'B', 'Root'])
check('super A follows dynamic MRO', solve('AB', 'after-A'), ['B', 'Root'])
check('super B follows dynamic MRO', solve('BA', 'after-B'), ['A', 'Root'])
check('terminal next', solve('AB', 'after-B'), ['Root'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-cooperative-diamond-mro

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, start):
    class Root:
        def trace(self): return ['Root']
    class A(Root):
        def trace(self): return ['A']+super().trace()
    class B(Root):
        def trace(self): return ['B']+Root.trace(self)
    def leaf_trace(self): return ['Leaf']+super(Leaf,self).trace()
    Leaf=type('Leaf',(A,B) if order=='AB' else (B,A),{'trace':leaf_trace})
    obj=Leaf()
    if start=='leaf': return obj.trace()
    if start=='after-leaf': return super(Leaf,obj).trace()
    if start=='after-A': return super(A,obj).trace()
    return super(B,obj).trace()
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('left-first diamond', solve('AB', 'leaf'), ['Leaf', 'A', 'B', 'Root'])
check('right-first diamond', solve('BA', 'leaf'), ['Leaf', 'B', 'A', 'Root'])
check('super after leaf', solve('AB', 'after-leaf'), ['A', 'B', 'Root'])
check('super A follows dynamic MRO', solve('AB', 'after-A'), ['B', 'Root'])
check('super B follows dynamic MRO', solve('BA', 'after-B'), ['A', 'Root'])
check('terminal next', solve('AB', 'after-B'), ['Root'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Hardcodes A delegation to Root and skips a sibling in the MRO and changes the required Python-language result.

Traverse the actual C3 MRO cooperatively, appending each visited class once. Leaf bases are A,B or B,A and both derive from Root; start chooses normal dispatch or super after a specified class. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, start):
    class Root:
        def trace(self): return ['Root']
    class A(Root):
        def trace(self): return ['A']+super().trace()
    class B(Root):
        def trace(self): return ['B']+Root.trace(self)
    def leaf_trace(self): return ['Leaf']+super(Leaf,self).trace()
    Leaf=type('Leaf',(A,B) if order=='AB' else (B,A),{'trace':leaf_trace})
    obj=Leaf()
    if start=='leaf': return obj.trace()
    if start=='after-leaf': return super(Leaf,obj).trace()
    if start=='after-A': return super(A,obj).trace()
    return super(B,obj).trace()
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('left-first diamond', solve('AB', 'leaf'), ['Leaf', 'A', 'B', 'Root'])
check('right-first diamond', solve('BA', 'leaf'), ['Leaf', 'B', 'A', 'Root'])
check('super after leaf', solve('AB', 'after-leaf'), ['A', 'B', 'Root'])
check('super A follows dynamic MRO', solve('AB', 'after-A'), ['B', 'Root'])
check('super B follows dynamic MRO', solve('BA', 'after-B'), ['A', 'Root'])
check('terminal next', solve('AB', 'after-B'), ['Root'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-cooperative-diamond-mro

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, start):
    class Root:
        def trace(self): return ['Root']
    class A(Root):
        def trace(self): return ['A']+super().trace()
    class B(Root):
        def trace(self): return ['B']+super().trace()
    def leaf_trace(self): return ['Leaf']+A.trace(self)
    Leaf=type('Leaf',(A,B) if order=='AB' else (B,A),{'trace':leaf_trace})
    obj=Leaf()
    if start=='leaf': return obj.trace()
    if start=='after-leaf': return super(Leaf,obj).trace()
    if start=='after-A': return super(A,obj).trace()
    return super(B,obj).trace()
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('left-first diamond', solve('AB', 'leaf'), ['Leaf', 'A', 'B', 'Root'])
check('right-first diamond', solve('BA', 'leaf'), ['Leaf', 'B', 'A', 'Root'])
check('super after leaf', solve('AB', 'after-leaf'), ['A', 'B', 'Root'])
check('super A follows dynamic MRO', solve('AB', 'after-A'), ['B', 'Root'])
check('super B follows dynamic MRO', solve('BA', 'after-B'), ['A', 'Root'])
check('terminal next', solve('AB', 'after-B'), ['Root'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Hardcodes B delegation to Root and skips A in reversed base order and changes the required Python-language result.

Traverse the actual C3 MRO cooperatively, appending each visited class once. Leaf bases are A,B or B,A and both derive from Root; start chooses normal dispatch or super after a specified class. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, start):
    class Root:
        def trace(self): return ['Root']
    class A(Root):
        def trace(self): return ['A']+super().trace()
    class B(Root):
        def trace(self): return ['B']+super().trace()
    def leaf_trace(self): return ['Leaf']+A.trace(self)
    Leaf=type('Leaf',(A,B) if order=='AB' else (B,A),{'trace':leaf_trace})
    obj=Leaf()
    if start=='leaf': return obj.trace()
    if start=='after-leaf': return super(Leaf,obj).trace()
    if start=='after-A': return super(A,obj).trace()
    return super(B,obj).trace()
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('left-first diamond', solve('AB', 'leaf'), ['Leaf', 'A', 'B', 'Root'])
check('right-first diamond', solve('BA', 'leaf'), ['Leaf', 'B', 'A', 'Root'])
check('super after leaf', solve('AB', 'after-leaf'), ['A', 'B', 'Root'])
check('super A follows dynamic MRO', solve('AB', 'after-A'), ['B', 'Root'])
check('super B follows dynamic MRO', solve('BA', 'after-B'), ['A', 'Root'])
check('terminal next', solve('AB', 'after-B'), ['Root'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-cooperative-diamond-mro

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, start):
    class Root:
        def trace(self): return []
    class A(Root):
        def trace(self): return ['A']+super().trace()
    class B(Root):
        def trace(self): return ['B']+super().trace()
    def leaf_trace(self): return ['Leaf']+super(Leaf,self).trace()
    Leaf=type('Leaf',(A,B) if order=='AB' else (B,A),{'trace':leaf_trace})
    obj=Leaf()
    if start=='leaf': return obj.trace()
    if start=='after-leaf': return super(Leaf,obj).trace()
    if start=='after-A': return super(A,obj).trace()
    return super(B,obj).trace()
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('left-first diamond', solve('AB', 'leaf'), ['Leaf', 'A', 'B', 'Root'])
check('right-first diamond', solve('BA', 'leaf'), ['Leaf', 'B', 'A', 'Root'])
check('super after leaf', solve('AB', 'after-leaf'), ['A', 'B', 'Root'])
check('super A follows dynamic MRO', solve('AB', 'after-A'), ['B', 'Root'])
check('super B follows dynamic MRO', solve('BA', 'after-B'), ['A', 'Root'])
check('terminal next', solve('AB', 'after-B'), ['Root'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Starts at A regardless of the declared base order and changes the required Python-language result.

Traverse the actual C3 MRO cooperatively, appending each visited class once. Leaf bases are A,B or B,A and both derive from Root; start chooses normal dispatch or super after a specified class. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, start):
    class Root:
        def trace(self): return []
    class A(Root):
        def trace(self): return ['A']+super().trace()
    class B(Root):
        def trace(self): return ['B']+super().trace()
    def leaf_trace(self): return ['Leaf']+super(Leaf,self).trace()
    Leaf=type('Leaf',(A,B) if order=='AB' else (B,A),{'trace':leaf_trace})
    obj=Leaf()
    if start=='leaf': return obj.trace()
    if start=='after-leaf': return super(Leaf,obj).trace()
    if start=='after-A': return super(A,obj).trace()
    return super(B,obj).trace()
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('left-first diamond', solve('AB', 'leaf'), ['Leaf', 'A', 'B', 'Root'])
check('right-first diamond', solve('BA', 'leaf'), ['Leaf', 'B', 'A', 'Root'])
check('super after leaf', solve('AB', 'after-leaf'), ['A', 'B', 'Root'])
check('super A follows dynamic MRO', solve('AB', 'after-A'), ['B', 'Root'])
check('super B follows dynamic MRO', solve('BA', 'after-B'), ['A', 'Root'])
check('terminal next', solve('AB', 'after-B'), ['Root'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-cooperative-diamond-mro

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, start):
    class Root:
        def trace(self): return ['Root']
    class A(Root):
        def trace(self): return ['A']+super().trace()
    class B(Root):
        def trace(self): return ['B']+super().trace()
    def leaf_trace(self): return ['Leaf']+A.trace(self)+B.trace(self)
    Leaf=type('Leaf',(A,B) if order=='AB' else (B,A),{'trace':leaf_trace})
    obj=Leaf()
    if start=='leaf': return obj.trace()
    if start=='after-leaf': return super(Leaf,obj).trace()
    if start=='after-A': return super(A,obj).trace()
    return super(B,obj).trace()
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('left-first diamond', solve('AB', 'leaf'), ['Leaf', 'A', 'B', 'Root'])
check('right-first diamond', solve('BA', 'leaf'), ['Leaf', 'B', 'A', 'Root'])
check('super after leaf', solve('AB', 'after-leaf'), ['A', 'B', 'Root'])
check('super A follows dynamic MRO', solve('AB', 'after-A'), ['B', 'Root'])
check('super B follows dynamic MRO', solve('BA', 'after-B'), ['A', 'Root'])
check('terminal next', solve('AB', 'after-B'), ['Root'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Omits the terminal Root contribution from the cooperative chain and changes the required Python-language result.

Traverse the actual C3 MRO cooperatively, appending each visited class once. Leaf bases are A,B or B,A and both derive from Root; start chooses normal dispatch or super after a specified class. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, start):
    class Root:
        def trace(self): return ['Root']
    class A(Root):
        def trace(self): return ['A']+super().trace()
    class B(Root):
        def trace(self): return ['B']+super().trace()
    def leaf_trace(self): return ['Leaf']+A.trace(self)+B.trace(self)
    Leaf=type('Leaf',(A,B) if order=='AB' else (B,A),{'trace':leaf_trace})
    obj=Leaf()
    if start=='leaf': return obj.trace()
    if start=='after-leaf': return super(Leaf,obj).trace()
    if start=='after-A': return super(A,obj).trace()
    return super(B,obj).trace()
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('left-first diamond', solve('AB', 'leaf'), ['Leaf', 'A', 'B', 'Root'])
check('right-first diamond', solve('BA', 'leaf'), ['Leaf', 'B', 'A', 'Root'])
check('super after leaf', solve('AB', 'after-leaf'), ['A', 'B', 'Root'])
check('super A follows dynamic MRO', solve('AB', 'after-A'), ['B', 'Root'])
check('super B follows dynamic MRO', solve('BA', 'after-B'), ['A', 'Root'])
check('terminal next', solve('AB', 'after-B'), ['Root'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-cooperative-diamond-mro

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(order, start):
    class Root:
        def trace(self): return ['Root']
    class A(Root):
        def trace(self): return ['A']+Root.trace(self)
    class B(Root):
        def trace(self): return ['B']+super().trace()
    def leaf_trace(self): return ['Leaf']+super(Leaf,self).trace()
    Leaf=type('Leaf',(A,B) if order=='AB' else (B,A),{'trace':leaf_trace})
    obj=Leaf()
    if start=='leaf': return obj.trace()
    if start=='after-leaf': return super(Leaf,obj).trace()
    if start=='after-A': return super(A,obj).trace()
    return super(B,obj).trace()
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('left-first diamond', solve('AB', 'leaf'), ['Leaf', 'A', 'B', 'Root'])
check('right-first diamond', solve('BA', 'leaf'), ['Leaf', 'B', 'A', 'Root'])
check('super after leaf', solve('AB', 'after-leaf'), ['A', 'B', 'Root'])
check('super A follows dynamic MRO', solve('AB', 'after-A'), ['B', 'Root'])
check('super B follows dynamic MRO', solve('BA', 'after-B'), ['A', 'Root'])
check('terminal next', solve('AB', 'after-B'), ['Root'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Calls both bases explicitly and duplicates downstream cooperative contributions and changes the required Python-language result.

Traverse the actual C3 MRO cooperatively, appending each visited class once. Leaf bases are A,B or B,A and both derive from Root; start chooses normal dispatch or super after a specified class. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode, suffix):
    class Base:
        label='base'
        @staticmethod
        def read(cls,suffix): return [cls.__name__,cls.label,suffix]
    class Child(Base): label='child'
    try:
        if mode=='base': return Base.read(suffix)
        if mode=='child': return Child.read(suffix)
        if mode=='instance': return Child().read(suffix)
        bound=Child.read
        return bound(suffix)
    except (TypeError,AttributeError) as exc:
        return type(exc).__name__
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('base access', solve('base', 'x'), ['Base', 'base', 'x'])
check('subclass access', solve('child', 'y'), ['Child', 'child', 'y'])
check('instance access', solve('instance', None), ['Child', 'child', None])
check('saved bound method', solve('saved', 0), ['Child', 'child', 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-classmethod-dynamic-binding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode, suffix):
    class Base:
        label='base'

        def read(cls,suffix): return [cls.__name__,cls.label,suffix]
    class Child(Base): label='child'
    try:
        if mode=='base': return Base.read(suffix)
        if mode=='child': return Child.read(suffix)
        if mode=='instance': return Child().read(suffix)
        bound=Child.read
        return bound(suffix)
    except (TypeError,AttributeError) as exc:
        return type(exc).__name__
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('base access', solve('base', 'x'), ['Base', 'base', 'x'])
check('subclass access', solve('child', 'y'), ['Child', 'child', 'y'])
check('instance access', solve('instance', None), ['Child', 'child', None])
check('saved bound method', solve('saved', 0), ['Child', 'child', 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Makes the classmethod static and loses automatic class binding and changes the required Python-language result.

An inherited classmethod binds the accessing class, including access through an instance or a saved bound method. Return [bound class name,its label,suffix]. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode, suffix):
    class Base:
        label='base'

        def read(cls,suffix): return [cls.__name__,cls.label,suffix]
    class Child(Base): label='child'
    try:
        if mode=='base': return Base.read(suffix)
        if mode=='child': return Child.read(suffix)
        if mode=='instance': return Child().read(suffix)
        bound=Child.read
        return bound(suffix)
    except (TypeError,AttributeError) as exc:
        return type(exc).__name__
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('base access', solve('base', 'x'), ['Base', 'base', 'x'])
check('subclass access', solve('child', 'y'), ['Child', 'child', 'y'])
check('instance access', solve('instance', None), ['Child', 'child', None])
check('saved bound method', solve('saved', 0), ['Child', 'child', 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-classmethod-dynamic-binding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode, suffix):
    class Base:
        label='base'
        @classmethod
        def read(cls,suffix): return ['Base',Base.label,suffix]
    class Child(Base): label='child'
    try:
        if mode=='base': return Base.read(suffix)
        if mode=='child': return Child.read(suffix)
        if mode=='instance': return Child().read(suffix)
        bound=Child.read
        return bound(suffix)
    except (TypeError,AttributeError) as exc:
        return type(exc).__name__
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('base access', solve('base', 'x'), ['Base', 'base', 'x'])
check('subclass access', solve('child', 'y'), ['Child', 'child', 'y'])
check('instance access', solve('instance', None), ['Child', 'child', None])
check('saved bound method', solve('saved', 0), ['Child', 'child', 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Uses an ordinary instance method in place of a classmethod and changes the required Python-language result.

An inherited classmethod binds the accessing class, including access through an instance or a saved bound method. Return [bound class name,its label,suffix]. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode, suffix):
    class Base:
        label='base'
        @classmethod
        def read(cls,suffix): return ['Base',Base.label,suffix]
    class Child(Base): label='child'
    try:
        if mode=='base': return Base.read(suffix)
        if mode=='child': return Child.read(suffix)
        if mode=='instance': return Child().read(suffix)
        bound=Child.read
        return bound(suffix)
    except (TypeError,AttributeError) as exc:
        return type(exc).__name__
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('base access', solve('base', 'x'), ['Base', 'base', 'x'])
check('subclass access', solve('child', 'y'), ['Child', 'child', 'y'])
check('instance access', solve('instance', None), ['Child', 'child', None])
check('saved bound method', solve('saved', 0), ['Child', 'child', 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-classmethod-dynamic-binding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode, suffix):
    class Base:
        label='base'
        @classmethod
        def read(cls,suffix): return [type(cls).__name__,cls.label,suffix]
    class Child(Base): label='child'
    try:
        if mode=='base': return Base.read(suffix)
        if mode=='child': return Child.read(suffix)
        if mode=='instance': return Child().read(suffix)
        bound=Child.read
        return bound(suffix)
    except (TypeError,AttributeError) as exc:
        return type(exc).__name__
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('base access', solve('base', 'x'), ['Base', 'base', 'x'])
check('subclass access', solve('child', 'y'), ['Child', 'child', 'y'])
check('instance access', solve('instance', None), ['Child', 'child', None])
check('saved bound method', solve('saved', 0), ['Child', 'child', 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Uses the defining Base class instead of the bound subclass and changes the required Python-language result.

An inherited classmethod binds the accessing class, including access through an instance or a saved bound method. Return [bound class name,its label,suffix]. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode, suffix):
    class Base:
        label='base'
        @classmethod
        def read(cls,suffix): return [type(cls).__name__,cls.label,suffix]
    class Child(Base): label='child'
    try:
        if mode=='base': return Base.read(suffix)
        if mode=='child': return Child.read(suffix)
        if mode=='instance': return Child().read(suffix)
        bound=Child.read
        return bound(suffix)
    except (TypeError,AttributeError) as exc:
        return type(exc).__name__
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('base access', solve('base', 'x'), ['Base', 'base', 'x'])
check('subclass access', solve('child', 'y'), ['Child', 'child', 'y'])
check('instance access', solve('instance', None), ['Child', 'child', None])
check('saved bound method', solve('saved', 0), ['Child', 'child', 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-classmethod-dynamic-binding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode, suffix):
    class Base:
        label='base'
        @classmethod
        def read(cls,suffix): return [cls.__name__,Base.label,suffix]
    class Child(Base): label='child'
    try:
        if mode=='base': return Base.read(suffix)
        if mode=='child': return Child.read(suffix)
        if mode=='instance': return Child().read(suffix)
        bound=Child.read
        return bound(suffix)
    except (TypeError,AttributeError) as exc:
        return type(exc).__name__
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('base access', solve('base', 'x'), ['Base', 'base', 'x'])
check('subclass access', solve('child', 'y'), ['Child', 'child', 'y'])
check('instance access', solve('instance', None), ['Child', 'child', None])
check('saved bound method', solve('saved', 0), ['Child', 'child', 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reports the bound class metaclass instead of the class name and changes the required Python-language result.

An inherited classmethod binds the accessing class, including access through an instance or a saved bound method. Return [bound class name,its label,suffix]. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode, suffix):
    class Base:
        label='base'
        @classmethod
        def read(cls,suffix): return [cls.__name__,Base.label,suffix]
    class Child(Base): label='child'
    try:
        if mode=='base': return Base.read(suffix)
        if mode=='child': return Child.read(suffix)
        if mode=='instance': return Child().read(suffix)
        bound=Child.read
        return bound(suffix)
    except (TypeError,AttributeError) as exc:
        return type(exc).__name__
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('base access', solve('base', 'x'), ['Base', 'base', 'x'])
check('subclass access', solve('child', 'y'), ['Child', 'child', 'y'])
check('instance access', solve('instance', None), ['Child', 'child', None])
check('saved bound method', solve('saved', 0), ['Child', 'child', 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-classmethod-dynamic-binding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode, suffix):
    class Base:
        label='base'
        @staticmethod
        def read(cls,suffix): return [cls.__name__,cls.label,suffix]
    class Child(Base): label='child'
    try:
        if mode=='base': return Base.read(suffix)
        if mode=='child': return Child.read(suffix)
        if mode=='instance': return Child().read(suffix)
        bound=Child.read
        return bound(suffix)
    except (TypeError,AttributeError) as exc:
        return type(exc).__name__
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('base access', solve('base', 'x'), ['Base', 'base', 'x'])
check('subclass access', solve('child', 'y'), ['Child', 'child', 'y'])
check('instance access', solve('instance', None), ['Child', 'child', None])
check('saved bound method', solve('saved', 0), ['Child', 'child', 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Ignores subclass shadowing when reading the bound class label and changes the required Python-language result.

An inherited classmethod binds the accessing class, including access through an instance or a saved bound method. Return [bound class name,its label,suffix]. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, enabled):
    log=[]
    class Manager:
        def __enter__(self):
            log.append('enter')
            return self
        def __exit__(self, typ, exc, tb):
            log.append('exit:'+('none' if typ is None else typ.__name__))
            return enabled
    try:
        with Manager():
            log.append('body')
            if error=='value': raise ValueError('value')
            if error=='key': raise KeyError('key')
            if error=='type': raise TypeError('type')
        log.append('after')
    except Exception as exc:
        log.append('caught:'+type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal exit', solve('none', True), ['enter', 'body', 'exit:none', 'after'])
check('selected error suppressed', solve('value', True), ['enter', 'body', 'exit:ValueError', 'after'])
check('disabled propagation', solve('value', False), ['enter', 'body', 'exit:ValueError', 'caught:ValueError'])
check('unrelated error propagates', solve('key', True), ['enter', 'body', 'exit:KeyError', 'caught:KeyError'])
check('other unrelated type', solve('type', True), ['enter', 'body', 'exit:TypeError', 'caught:TypeError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-context-exception-suppression

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, enabled):
    log=[]
    class Manager:
        def __enter__(self):
            log.append('enter')
            return self
        def __exit__(self, typ, exc, tb):
            log.append('exit:'+('none' if typ is None else typ.__name__))
            return False
    try:
        with Manager():
            log.append('body')
            if error=='value': raise ValueError('value')
            if error=='key': raise KeyError('key')
            if error=='type': raise TypeError('type')
        log.append('after')
    except Exception as exc:
        log.append('caught:'+type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal exit', solve('none', True), ['enter', 'body', 'exit:none', 'after'])
check('selected error suppressed', solve('value', True), ['enter', 'body', 'exit:ValueError', 'after'])
check('disabled propagation', solve('value', False), ['enter', 'body', 'exit:ValueError', 'caught:ValueError'])
check('unrelated error propagates', solve('key', True), ['enter', 'body', 'exit:KeyError', 'caught:KeyError'])
check('other unrelated type', solve('type', True), ['enter', 'body', 'exit:TypeError', 'caught:TypeError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Suppresses unrelated exception types whenever suppression is enabled and changes the required Python-language result.

A context manager suppresses only ValueError when enabled. __exit__ observes the actual exception class or None on normal exit; unsuppressed exceptions reach the outer handler and skip after. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, enabled):
    log=[]
    class Manager:
        def __enter__(self):
            log.append('enter')
            return self
        def __exit__(self, typ, exc, tb):
            log.append('exit:'+('none' if typ is None else typ.__name__))
            return False
    try:
        with Manager():
            log.append('body')
            if error=='value': raise ValueError('value')
            if error=='key': raise KeyError('key')
            if error=='type': raise TypeError('type')
        log.append('after')
    except Exception as exc:
        log.append('caught:'+type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal exit', solve('none', True), ['enter', 'body', 'exit:none', 'after'])
check('selected error suppressed', solve('value', True), ['enter', 'body', 'exit:ValueError', 'after'])
check('disabled propagation', solve('value', False), ['enter', 'body', 'exit:ValueError', 'caught:ValueError'])
check('unrelated error propagates', solve('key', True), ['enter', 'body', 'exit:KeyError', 'caught:KeyError'])
check('other unrelated type', solve('type', True), ['enter', 'body', 'exit:TypeError', 'caught:TypeError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-context-exception-suppression

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, enabled):
    log=[]
    class Manager:
        def __enter__(self):
            log.append('enter')
            return self
        def __exit__(self, typ, exc, tb):
            log.append('exit:'+('none' if typ is None else typ.__name__))
            return typ is ValueError
    try:
        with Manager():
            log.append('body')
            if error=='value': raise ValueError('value')
            if error=='key': raise KeyError('key')
            if error=='type': raise TypeError('type')
        log.append('after')
    except Exception as exc:
        log.append('caught:'+type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal exit', solve('none', True), ['enter', 'body', 'exit:none', 'after'])
check('selected error suppressed', solve('value', True), ['enter', 'body', 'exit:ValueError', 'after'])
check('disabled propagation', solve('value', False), ['enter', 'body', 'exit:ValueError', 'caught:ValueError'])
check('unrelated error propagates', solve('key', True), ['enter', 'body', 'exit:KeyError', 'caught:KeyError'])
check('other unrelated type', solve('type', True), ['enter', 'body', 'exit:TypeError', 'caught:TypeError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Discards the truthy __exit__ decision and propagates ValueError and changes the required Python-language result.

A context manager suppresses only ValueError when enabled. __exit__ observes the actual exception class or None on normal exit; unsuppressed exceptions reach the outer handler and skip after. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, enabled):
    log=[]
    class Manager:
        def __enter__(self):
            log.append('enter')
            return self
        def __exit__(self, typ, exc, tb):
            log.append('exit:'+('none' if typ is None else typ.__name__))
            return typ is ValueError
    try:
        with Manager():
            log.append('body')
            if error=='value': raise ValueError('value')
            if error=='key': raise KeyError('key')
            if error=='type': raise TypeError('type')
        log.append('after')
    except Exception as exc:
        log.append('caught:'+type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal exit', solve('none', True), ['enter', 'body', 'exit:none', 'after'])
check('selected error suppressed', solve('value', True), ['enter', 'body', 'exit:ValueError', 'after'])
check('disabled propagation', solve('value', False), ['enter', 'body', 'exit:ValueError', 'caught:ValueError'])
check('unrelated error propagates', solve('key', True), ['enter', 'body', 'exit:KeyError', 'caught:KeyError'])
check('other unrelated type', solve('type', True), ['enter', 'body', 'exit:TypeError', 'caught:TypeError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-context-exception-suppression

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, enabled):
    log=[]
    class Manager:
        def __enter__(self):
            log.append('enter')
            return self
        def __exit__(self, typ, exc, tb):
            log.append('exit:none')
            return typ is ValueError and enabled
    try:
        with Manager():
            log.append('body')
            if error=='value': raise ValueError('value')
            if error=='key': raise KeyError('key')
            if error=='type': raise TypeError('type')
        log.append('after')
    except Exception as exc:
        log.append('caught:'+type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal exit', solve('none', True), ['enter', 'body', 'exit:none', 'after'])
check('selected error suppressed', solve('value', True), ['enter', 'body', 'exit:ValueError', 'after'])
check('disabled propagation', solve('value', False), ['enter', 'body', 'exit:ValueError', 'caught:ValueError'])
check('unrelated error propagates', solve('key', True), ['enter', 'body', 'exit:KeyError', 'caught:KeyError'])
check('other unrelated type', solve('type', True), ['enter', 'body', 'exit:TypeError', 'caught:TypeError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Suppresses ValueError even when the policy is disabled and changes the required Python-language result.

A context manager suppresses only ValueError when enabled. __exit__ observes the actual exception class or None on normal exit; unsuppressed exceptions reach the outer handler and skip after. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, enabled):
    log=[]
    class Manager:
        def __enter__(self):
            log.append('enter')
            return self
        def __exit__(self, typ, exc, tb):
            log.append('exit:none')
            return typ is ValueError and enabled
    try:
        with Manager():
            log.append('body')
            if error=='value': raise ValueError('value')
            if error=='key': raise KeyError('key')
            if error=='type': raise TypeError('type')
        log.append('after')
    except Exception as exc:
        log.append('caught:'+type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal exit', solve('none', True), ['enter', 'body', 'exit:none', 'after'])
check('selected error suppressed', solve('value', True), ['enter', 'body', 'exit:ValueError', 'after'])
check('disabled propagation', solve('value', False), ['enter', 'body', 'exit:ValueError', 'caught:ValueError'])
check('unrelated error propagates', solve('key', True), ['enter', 'body', 'exit:KeyError', 'caught:KeyError'])
check('other unrelated type', solve('type', True), ['enter', 'body', 'exit:TypeError', 'caught:TypeError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-context-exception-suppression

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, enabled):
    log=[]
    class Manager:
        def __enter__(self):
            log.append('enter')
            return self
        def __exit__(self, typ, exc, tb):
            log.append('exit:'+('none' if typ is None else typ.__name__))
            return typ is ValueError and not enabled
    try:
        with Manager():
            log.append('body')
            if error=='value': raise ValueError('value')
            if error=='key': raise KeyError('key')
            if error=='type': raise TypeError('type')
        log.append('after')
    except Exception as exc:
        log.append('caught:'+type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal exit', solve('none', True), ['enter', 'body', 'exit:none', 'after'])
check('selected error suppressed', solve('value', True), ['enter', 'body', 'exit:ValueError', 'after'])
check('disabled propagation', solve('value', False), ['enter', 'body', 'exit:ValueError', 'caught:ValueError'])
check('unrelated error propagates', solve('key', True), ['enter', 'body', 'exit:KeyError', 'caught:KeyError'])
check('other unrelated type', solve('type', True), ['enter', 'body', 'exit:TypeError', 'caught:TypeError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Passes a normal-exit placeholder to the exception-reporting path and changes the required Python-language result.

A context manager suppresses only ValueError when enabled. __exit__ observes the actual exception class or None on normal exit; unsuppressed exceptions reach the outer handler and skip after. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, enabled):
    log=[]
    class Manager:
        def __enter__(self):
            log.append('enter')
            return self
        def __exit__(self, typ, exc, tb):
            log.append('exit:'+('none' if typ is None else typ.__name__))
            return typ is ValueError and not enabled
    try:
        with Manager():
            log.append('body')
            if error=='value': raise ValueError('value')
            if error=='key': raise KeyError('key')
            if error=='type': raise TypeError('type')
        log.append('after')
    except Exception as exc:
        log.append('caught:'+type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal exit', solve('none', True), ['enter', 'body', 'exit:none', 'after'])
check('selected error suppressed', solve('value', True), ['enter', 'body', 'exit:ValueError', 'after'])
check('disabled propagation', solve('value', False), ['enter', 'body', 'exit:ValueError', 'caught:ValueError'])
check('unrelated error propagates', solve('key', True), ['enter', 'body', 'exit:KeyError', 'caught:KeyError'])
check('other unrelated type', solve('type', True), ['enter', 'body', 'exit:TypeError', 'caught:TypeError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-context-exception-suppression

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(error, enabled):
    log=[]
    class Manager:
        def __enter__(self):
            log.append('enter')
            return self
        def __exit__(self, typ, exc, tb):
            log.append('exit:'+('none' if typ is None else typ.__name__))
            return enabled
    try:
        with Manager():
            log.append('body')
            if error=='value': raise ValueError('value')
            if error=='key': raise KeyError('key')
            if error=='type': raise TypeError('type')
        log.append('after')
    except Exception as exc:
        log.append('caught:'+type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal exit', solve('none', True), ['enter', 'body', 'exit:none', 'after'])
check('selected error suppressed', solve('value', True), ['enter', 'body', 'exit:ValueError', 'after'])
check('disabled propagation', solve('value', False), ['enter', 'body', 'exit:ValueError', 'caught:ValueError'])
check('unrelated error propagates', solve('key', True), ['enter', 'body', 'exit:KeyError', 'caught:KeyError'])
check('other unrelated type', solve('type', True), ['enter', 'body', 'exit:TypeError', 'caught:TypeError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Suppresses exceptions when the policy is disabled instead of enabled and changes the required Python-language result.

A context manager suppresses only ValueError when enabled. __exit__ observes the actual exception class or None on normal exit; unsuppressed exceptions reach the outer handler and skip after. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(payload, fail, body_error):
    log=[]
    class Manager:
        def __init__(self): self.value='manager'
        def __enter__(self):
            log.append('enter')
            if fail: raise ValueError('enter failed')
            return payload
        def __exit__(self,typ,exc,tb): log.append('exit'); return False
    manager=Manager()
    try:
        with manager as bound:
            log.append(['body',manager.value])
            if body_error: raise KeyError('body failed')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary binding', solve('value', False, False), ['enter', ['body', 'value'], 'exit'])
check('false return remains bound', solve(0, False, False), ['enter', ['body', 0], 'exit'])
check('None return remains bound', solve(None, False, False), ['enter', ['body', None], 'exit'])
check('enter failure skips exit', solve('value', True, False), ['enter', 'ValueError'])
check('body failure still exits', solve('value', False, True), ['enter', ['body', 'value'], 'exit', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-context-enter-binding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(payload, fail, body_error):
    log=[]
    class Manager:
        def __init__(self): self.value='manager'
        def __enter__(self):
            log.append('enter')
            if fail: raise ValueError('enter failed')
            return payload
        def __exit__(self,typ,exc,tb): log.append('exit'); return False
    manager=Manager()
    try:
        with manager as bound:
            log.append(['body',bound or manager.value])
            if body_error: raise KeyError('body failed')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary binding', solve('value', False, False), ['enter', ['body', 'value'], 'exit'])
check('false return remains bound', solve(0, False, False), ['enter', ['body', 0], 'exit'])
check('None return remains bound', solve(None, False, False), ['enter', ['body', None], 'exit'])
check('enter failure skips exit', solve('value', True, False), ['enter', 'ValueError'])
check('body failure still exits', solve('value', False, True), ['enter', ['body', 'value'], 'exit', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Binds the manager object label instead of its __enter__ result and changes the required Python-language result.

Bind the exact __enter__ return value to the with target, including false values. A failing __enter__ prevents body and __exit__; body exceptions still call __exit__. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(payload, fail, body_error):
    log=[]
    class Manager:
        def __init__(self): self.value='manager'
        def __enter__(self):
            log.append('enter')
            if fail: raise ValueError('enter failed')
            return payload
        def __exit__(self,typ,exc,tb): log.append('exit'); return False
    manager=Manager()
    try:
        with manager as bound:
            log.append(['body',bound or manager.value])
            if body_error: raise KeyError('body failed')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary binding', solve('value', False, False), ['enter', ['body', 'value'], 'exit'])
check('false return remains bound', solve(0, False, False), ['enter', ['body', 0], 'exit'])
check('None return remains bound', solve(None, False, False), ['enter', ['body', None], 'exit'])
check('enter failure skips exit', solve('value', True, False), ['enter', 'ValueError'])
check('body failure still exits', solve('value', False, True), ['enter', ['body', 'value'], 'exit', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-context-enter-binding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(payload, fail, body_error):
    log=[]
    class Manager:
        def __init__(self): self.value='manager'
        def __enter__(self):
            log.append('enter')
            if fail: raise ValueError('enter failed')
            return None
        def __exit__(self,typ,exc,tb): log.append('exit'); return False
    manager=Manager()
    try:
        with manager as bound:
            log.append(['body',bound])
            if body_error: raise KeyError('body failed')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary binding', solve('value', False, False), ['enter', ['body', 'value'], 'exit'])
check('false return remains bound', solve(0, False, False), ['enter', ['body', 0], 'exit'])
check('None return remains bound', solve(None, False, False), ['enter', ['body', None], 'exit'])
check('enter failure skips exit', solve('value', True, False), ['enter', 'ValueError'])
check('body failure still exits', solve('value', False, True), ['enter', ['body', 'value'], 'exit', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Replaces a false __enter__ return value with the manager label and changes the required Python-language result.

Bind the exact __enter__ return value to the with target, including false values. A failing __enter__ prevents body and __exit__; body exceptions still call __exit__. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(payload, fail, body_error):
    log=[]
    class Manager:
        def __init__(self): self.value='manager'
        def __enter__(self):
            log.append('enter')
            if fail: raise ValueError('enter failed')
            return None
        def __exit__(self,typ,exc,tb): log.append('exit'); return False
    manager=Manager()
    try:
        with manager as bound:
            log.append(['body',bound])
            if body_error: raise KeyError('body failed')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary binding', solve('value', False, False), ['enter', ['body', 'value'], 'exit'])
check('false return remains bound', solve(0, False, False), ['enter', ['body', 0], 'exit'])
check('None return remains bound', solve(None, False, False), ['enter', ['body', None], 'exit'])
check('enter failure skips exit', solve('value', True, False), ['enter', 'ValueError'])
check('body failure still exits', solve('value', False, True), ['enter', ['body', 'value'], 'exit', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-context-enter-binding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(payload, fail, body_error):
    log=[]
    class Manager:
        def __init__(self): self.value='manager'
        def __enter__(self):
            log.append('enter')
            if fail: raise ValueError('enter failed')
            return payload
        def __exit__(self,typ,exc,tb): log.append('exit'); return False
    manager=Manager()
    try:
        with manager as bound:
            log.append(['body',manager.__enter__()])
            if body_error: raise KeyError('body failed')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary binding', solve('value', False, False), ['enter', ['body', 'value'], 'exit'])
check('false return remains bound', solve(0, False, False), ['enter', ['body', 0], 'exit'])
check('None return remains bound', solve(None, False, False), ['enter', ['body', None], 'exit'])
check('enter failure skips exit', solve('value', True, False), ['enter', 'ValueError'])
check('body failure still exits', solve('value', False, True), ['enter', ['body', 'value'], 'exit', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Returns None from __enter__ regardless of its payload and changes the required Python-language result.

Bind the exact __enter__ return value to the with target, including false values. A failing __enter__ prevents body and __exit__; body exceptions still call __exit__. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(payload, fail, body_error):
    log=[]
    class Manager:
        def __init__(self): self.value='manager'
        def __enter__(self):
            log.append('enter')
            if fail: raise ValueError('enter failed')
            return payload
        def __exit__(self,typ,exc,tb): log.append('exit'); return False
    manager=Manager()
    try:
        with manager as bound:
            log.append(['body',manager.__enter__()])
            if body_error: raise KeyError('body failed')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary binding', solve('value', False, False), ['enter', ['body', 'value'], 'exit'])
check('false return remains bound', solve(0, False, False), ['enter', ['body', 0], 'exit'])
check('None return remains bound', solve(None, False, False), ['enter', ['body', None], 'exit'])
check('enter failure skips exit', solve('value', True, False), ['enter', 'ValueError'])
check('body failure still exits', solve('value', False, True), ['enter', ['body', 'value'], 'exit', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-context-enter-binding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(payload, fail, body_error):
    log=[]
    try:
        log.append('enter')
        if fail: raise ValueError('enter failed')
        log.append(['body',payload])
        if body_error: raise KeyError('body failed')
        log.append('exit')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary binding', solve('value', False, False), ['enter', ['body', 'value'], 'exit'])
check('false return remains bound', solve(0, False, False), ['enter', ['body', 0], 'exit'])
check('None return remains bound', solve(None, False, False), ['enter', ['body', None], 'exit'])
check('enter failure skips exit', solve('value', True, False), ['enter', 'ValueError'])
check('body failure still exits', solve('value', False, True), ['enter', ['body', 'value'], 'exit', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Calls __enter__ again while constructing the with-body binding and changes the required Python-language result.

Bind the exact __enter__ return value to the with target, including false values. A failing __enter__ prevents body and __exit__; body exceptions still call __exit__. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(payload, fail, body_error):
    log=[]
    try:
        log.append('enter')
        if fail: raise ValueError('enter failed')
        log.append(['body',payload])
        if body_error: raise KeyError('body failed')
        log.append('exit')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary binding', solve('value', False, False), ['enter', ['body', 'value'], 'exit'])
check('false return remains bound', solve(0, False, False), ['enter', ['body', 0], 'exit'])
check('None return remains bound', solve(None, False, False), ['enter', ['body', None], 'exit'])
check('enter failure skips exit', solve('value', True, False), ['enter', 'ValueError'])
check('body failure still exits', solve('value', False, True), ['enter', ['body', 'value'], 'exit', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-context-enter-binding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(payload, fail, body_error):
    log=[]
    class Manager:
        def __init__(self): self.value='manager'
        def __enter__(self):
            log.append('enter')
            if fail: raise ValueError('enter failed')
            return payload
        def __exit__(self,typ,exc,tb): log.append('exit'); return False
    manager=Manager()
    try:
        with manager as bound:
            log.append(['body',manager.value])
            if body_error: raise KeyError('body failed')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary binding', solve('value', False, False), ['enter', ['body', 'value'], 'exit'])
check('false return remains bound', solve(0, False, False), ['enter', ['body', 0], 'exit'])
check('None return remains bound', solve(None, False, False), ['enter', ['body', None], 'exit'])
check('enter failure skips exit', solve('value', True, False), ['enter', 'ValueError'])
check('body failure still exits', solve('value', False, True), ['enter', ['body', 'value'], 'exit', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Calls __enter__ manually and loses guaranteed exit on body failure and changes the required Python-language result.

Bind the exact __enter__ return value to the with target, including false values. A failing __enter__ prevents body and __exit__; body exceptions still call __exit__. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fail, body_error):
    log=[]
    class Manager:
        def __init__(self,name): self.name=name
        def __enter__(self):
            log.append('enter:'+self.name)
            if fail==self.name: raise ValueError(self.name)
            return self
        def __exit__(self,typ,exc,tb): log.append('exit:'+self.name); return False
    try:
        with Manager('B'), Manager('A'):
            log.append('body')
            if body_error: raise KeyError('body')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal nesting', solve(None, False), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A'])
check('first enter fails', solve('A', False), ['enter:A', 'ValueError'])
check('second enter fails', solve('B', False), ['enter:A', 'enter:B', 'exit:A', 'ValueError'])
check('body unwinds both', solve(None, True), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-nested-context-unwinding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fail, body_error):
    log=[]
    class Manager:
        def __init__(self,name): self.name=name
        def __enter__(self):
            log.append('enter:'+self.name)
            if fail==self.name: raise ValueError(self.name)
            return self
        def __exit__(self,typ,exc,tb): log.append('exit:'+self.name); return False
    try:
        with Manager('B'):
            log.append('body')
            if body_error: raise KeyError('body')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal nesting', solve(None, False), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A'])
check('first enter fails', solve('A', False), ['enter:A', 'ValueError'])
check('second enter fails', solve('B', False), ['enter:A', 'enter:B', 'exit:A', 'ValueError'])
check('body unwinds both', solve(None, True), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Acquires managers in the reverse declaration order and changes the required Python-language result.

Enter A then B and exit acquired managers in reverse order. A manager whose __enter__ raises is not exited; previously entered managers still exit. Body errors unwind both. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fail, body_error):
    log=[]
    class Manager:
        def __init__(self,name): self.name=name
        def __enter__(self):
            log.append('enter:'+self.name)
            if fail==self.name: raise ValueError(self.name)
            return self
        def __exit__(self,typ,exc,tb): log.append('exit:'+self.name); return False
    try:
        with Manager('B'):
            log.append('body')
            if body_error: raise KeyError('body')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal nesting', solve(None, False), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A'])
check('first enter fails', solve('A', False), ['enter:A', 'ValueError'])
check('second enter fails', solve('B', False), ['enter:A', 'enter:B', 'exit:A', 'ValueError'])
check('body unwinds both', solve(None, True), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-nested-context-unwinding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fail, body_error):
    log=[]
    class Manager:
        def __init__(self,name): self.name=name
        def __enter__(self):
            log.append('enter:'+self.name)
            if fail==self.name: raise ValueError(self.name)
            return self
        def __exit__(self,typ,exc,tb): log.append('exit:'+self.name); return False
    try:
        with Manager('A'):
            log.append('body')
            if body_error: raise KeyError('body')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal nesting', solve(None, False), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A'])
check('first enter fails', solve('A', False), ['enter:A', 'ValueError'])
check('second enter fails', solve('B', False), ['enter:A', 'enter:B', 'exit:A', 'ValueError'])
check('body unwinds both', solve(None, True), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Forgets to acquire the outer manager and changes the required Python-language result.

Enter A then B and exit acquired managers in reverse order. A manager whose __enter__ raises is not exited; previously entered managers still exit. Body errors unwind both. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fail, body_error):
    log=[]
    class Manager:
        def __init__(self,name): self.name=name
        def __enter__(self):
            log.append('enter:'+self.name)
            if fail==self.name: raise ValueError(self.name)
            return self
        def __exit__(self,typ,exc,tb): log.append('exit:'+self.name); return False
    try:
        with Manager('A'):
            log.append('body')
            if body_error: raise KeyError('body')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal nesting', solve(None, False), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A'])
check('first enter fails', solve('A', False), ['enter:A', 'ValueError'])
check('second enter fails', solve('B', False), ['enter:A', 'enter:B', 'exit:A', 'ValueError'])
check('body unwinds both', solve(None, True), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-nested-context-unwinding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fail, body_error):
    log=[]; acquired=[]
    try:
        for name in ['A','B']:
            log.append('enter:'+name)
            if fail==name: raise ValueError(name)
            acquired.append(name)
        log.append('body')
        if body_error: raise KeyError('body')
    except Exception as exc:
        log.extend('exit:'+name for name in acquired)
        log.append(type(exc).__name__)
    else: log.extend('exit:'+name for name in acquired)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal nesting', solve(None, False), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A'])
check('first enter fails', solve('A', False), ['enter:A', 'ValueError'])
check('second enter fails', solve('B', False), ['enter:A', 'enter:B', 'exit:A', 'ValueError'])
check('body unwinds both', solve(None, True), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Forgets to acquire the inner manager and changes the required Python-language result.

Enter A then B and exit acquired managers in reverse order. A manager whose __enter__ raises is not exited; previously entered managers still exit. Body errors unwind both. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fail, body_error):
    log=[]; acquired=[]
    try:
        for name in ['A','B']:
            log.append('enter:'+name)
            if fail==name: raise ValueError(name)
            acquired.append(name)
        log.append('body')
        if body_error: raise KeyError('body')
    except Exception as exc:
        log.extend('exit:'+name for name in acquired)
        log.append(type(exc).__name__)
    else: log.extend('exit:'+name for name in acquired)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal nesting', solve(None, False), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A'])
check('first enter fails', solve('A', False), ['enter:A', 'ValueError'])
check('second enter fails', solve('B', False), ['enter:A', 'enter:B', 'exit:A', 'ValueError'])
check('body unwinds both', solve(None, True), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-nested-context-unwinding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fail, body_error):
    log=[]; acquired=[]
    try:
        for name in ['A','B']:
            log.append('enter:'+name); acquired.append(name)
            if fail==name: raise ValueError(name)
        log.append('body')
        if body_error: raise KeyError('body')
    except Exception as exc:
        log.extend('exit:'+name for name in reversed(acquired))
        log.append(type(exc).__name__)
    else: log.extend('exit:'+name for name in reversed(acquired))
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal nesting', solve(None, False), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A'])
check('first enter fails', solve('A', False), ['enter:A', 'ValueError'])
check('second enter fails', solve('B', False), ['enter:A', 'enter:B', 'exit:A', 'ValueError'])
check('body unwinds both', solve(None, True), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Releases nested managers in acquisition order and changes the required Python-language result.

Enter A then B and exit acquired managers in reverse order. A manager whose __enter__ raises is not exited; previously entered managers still exit. Body errors unwind both. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fail, body_error):
    log=[]; acquired=[]
    try:
        for name in ['A','B']:
            log.append('enter:'+name); acquired.append(name)
            if fail==name: raise ValueError(name)
        log.append('body')
        if body_error: raise KeyError('body')
    except Exception as exc:
        log.extend('exit:'+name for name in reversed(acquired))
        log.append(type(exc).__name__)
    else: log.extend('exit:'+name for name in reversed(acquired))
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal nesting', solve(None, False), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A'])
check('first enter fails', solve('A', False), ['enter:A', 'ValueError'])
check('second enter fails', solve('B', False), ['enter:A', 'enter:B', 'exit:A', 'ValueError'])
check('body unwinds both', solve(None, True), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-nested-context-unwinding

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(fail, body_error):
    log=[]
    class Manager:
        def __init__(self,name): self.name=name
        def __enter__(self):
            log.append('enter:'+self.name)
            if fail==self.name: raise ValueError(self.name)
            return self
        def __exit__(self,typ,exc,tb): log.append('exit:'+self.name); return False
    try:
        with Manager('B'), Manager('A'):
            log.append('body')
            if body_error: raise KeyError('body')
    except Exception as exc: log.append(type(exc).__name__)
    return log
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('normal nesting', solve(None, False), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A'])
check('first enter fails', solve('A', False), ['enter:A', 'ValueError'])
check('second enter fails', solve('B', False), ['enter:A', 'enter:B', 'exit:A', 'ValueError'])
check('body unwinds both', solve(None, True), ['enter:A', 'enter:B', 'body', 'exit:B', 'exit:A', 'KeyError'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Registers cleanup before acquisition has succeeded and changes the required Python-language result.

Enter A then B and exit acquired managers in reverse order. A manager whose __enter__ raises is not exited; previously entered managers still exit. Body errors unwind both. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind):
    try:
        kinds={'key':KeyError,'index':IndexError,'lookup':LookupError,'value':ValueError,'system':SystemExit}
        if kind in kinds: raise kinds[kind]('failure')
        return 'normal'
    except LookupError: return 'lookup'
    except KeyError: return 'key'
    except IndexError: return 'index'
    except Exception: return 'other'
    except BaseException: return 'base'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('mapping failure', solve('key'), 'key')
check('sequence failure', solve('index'), 'index')
check('generic lookup failure', solve('lookup'), 'lookup')
check('ordinary other error', solve('value'), 'other')
check('termination is not Exception', solve('system'), 'base')
check('normal path', solve('none'), 'normal')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-exception-handler-hierarchy

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind):
    try:
        kinds={'key':KeyError,'index':IndexError,'lookup':LookupError,'value':ValueError,'system':SystemExit}
        if kind in kinds: raise kinds[kind]('failure')
        return 'normal'
    except Exception: return 'other'
    except KeyError: return 'key'
    except IndexError: return 'index'
    except LookupError: return 'lookup'
    except BaseException: return 'base'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('mapping failure', solve('key'), 'key')
check('sequence failure', solve('index'), 'index')
check('generic lookup failure', solve('lookup'), 'lookup')
check('ordinary other error', solve('value'), 'other')
check('termination is not Exception', solve('system'), 'base')
check('normal path', solve('none'), 'normal')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Places a broad LookupError handler before its specific subclasses and changes the required Python-language result.

Dispatch KeyError, IndexError, other LookupError, other Exception, and BaseException separately. Handler order must preserve the specific subclass cases; SystemExit belongs outside Exception. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind):
    try:
        kinds={'key':KeyError,'index':IndexError,'lookup':LookupError,'value':ValueError,'system':SystemExit}
        if kind in kinds: raise kinds[kind]('failure')
        return 'normal'
    except Exception: return 'other'
    except KeyError: return 'key'
    except IndexError: return 'index'
    except LookupError: return 'lookup'
    except BaseException: return 'base'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('mapping failure', solve('key'), 'key')
check('sequence failure', solve('index'), 'index')
check('generic lookup failure', solve('lookup'), 'lookup')
check('ordinary other error', solve('value'), 'other')
check('termination is not Exception', solve('system'), 'base')
check('normal path', solve('none'), 'normal')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-exception-handler-hierarchy

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind):
    try:
        kinds={'key':KeyError,'index':IndexError,'lookup':LookupError,'value':ValueError,'system':SystemExit}
        if kind in kinds: raise kinds[kind]('failure')
        return 'normal'
    except TypeError: return 'key'
    except IndexError: return 'index'
    except LookupError: return 'lookup'
    except Exception: return 'other'
    except BaseException: return 'base'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('mapping failure', solve('key'), 'key')
check('sequence failure', solve('index'), 'index')
check('generic lookup failure', solve('lookup'), 'lookup')
check('ordinary other error', solve('value'), 'other')
check('termination is not Exception', solve('system'), 'base')
check('normal path', solve('none'), 'normal')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Places Exception before all specific error handlers and changes the required Python-language result.

Dispatch KeyError, IndexError, other LookupError, other Exception, and BaseException separately. Handler order must preserve the specific subclass cases; SystemExit belongs outside Exception. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access