broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind):
    try:
        kinds={'key':KeyError,'index':IndexError,'lookup':LookupError,'value':ValueError,'system':SystemExit}
        if kind in kinds: raise kinds[kind]('failure')
        return 'normal'
    except TypeError: return 'key'
    except IndexError: return 'index'
    except LookupError: return 'lookup'
    except Exception: return 'other'
    except BaseException: return 'base'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('mapping failure', solve('key'), 'key')
check('sequence failure', solve('index'), 'index')
check('generic lookup failure', solve('lookup'), 'lookup')
check('ordinary other error', solve('value'), 'other')
check('termination is not Exception', solve('system'), 'base')
check('normal path', solve('none'), 'normal')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-exception-handler-hierarchy

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind):
    try:
        kinds={'key':KeyError,'index':IndexError,'lookup':LookupError,'value':ValueError,'system':SystemExit}
        if kind in kinds: raise kinds[kind]('failure')
        return 'normal'
    except KeyError: return 'key'
    except LookupError: return 'lookup'
    except Exception: return 'other'
    except BaseException: return 'base'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('mapping failure', solve('key'), 'key')
check('sequence failure', solve('index'), 'index')
check('generic lookup failure', solve('lookup'), 'lookup')
check('ordinary other error', solve('value'), 'other')
check('termination is not Exception', solve('system'), 'base')
check('normal path', solve('none'), 'normal')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Catches TypeError in the slot intended for KeyError and changes the required Python-language result.

Dispatch KeyError, IndexError, other LookupError, other Exception, and BaseException separately. Handler order must preserve the specific subclass cases; SystemExit belongs outside Exception. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind):
    try:
        kinds={'key':KeyError,'index':IndexError,'lookup':LookupError,'value':ValueError,'system':SystemExit}
        if kind in kinds: raise kinds[kind]('failure')
        return 'normal'
    except KeyError: return 'key'
    except LookupError: return 'lookup'
    except Exception: return 'other'
    except BaseException: return 'base'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('mapping failure', solve('key'), 'key')
check('sequence failure', solve('index'), 'index')
check('generic lookup failure', solve('lookup'), 'lookup')
check('ordinary other error', solve('value'), 'other')
check('termination is not Exception', solve('system'), 'base')
check('normal path', solve('none'), 'normal')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xl-exception-handler-hierarchy

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind):
    try:
        kinds={'key':KeyError,'index':IndexError,'lookup':LookupError,'value':ValueError,'system':SystemExit}
        if kind in kinds: raise kinds[kind]('failure')
        return 'normal'
    except KeyError: return 'key'
    except IndexError: return 'index'
    except LookupError: return 'lookup'
    except BaseException: return 'other'
    except BaseException: return 'base'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('mapping failure', solve('key'), 'key')
check('sequence failure', solve('index'), 'index')
check('generic lookup failure', solve('lookup'), 'lookup')
check('ordinary other error', solve('value'), 'other')
check('termination is not Exception', solve('system'), 'base')
check('normal path', solve('none'), 'normal')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Removes the specific IndexError handler and changes the required Python-language result.

Dispatch KeyError, IndexError, other LookupError, other Exception, and BaseException separately. Handler order must preserve the specific subclass cases; SystemExit belongs outside Exception. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind):
    try:
        kinds={'key':KeyError,'index':IndexError,'lookup':LookupError,'value':ValueError,'system':SystemExit}
        if kind in kinds: raise kinds[kind]('failure')
        return 'normal'
    except KeyError: return 'key'
    except IndexError: return 'index'
    except LookupError: return 'lookup'
    except BaseException: return 'other'
    except BaseException: return 'base'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('mapping failure', solve('key'), 'key')
check('sequence failure', solve('index'), 'index')
check('generic lookup failure', solve('lookup'), 'lookup')
check('ordinary other error', solve('value'), 'other')
check('termination is not Exception', solve('system'), 'base')
check('normal path', solve('none'), 'normal')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-exception-handler-hierarchy

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(kind):
    try:
        kinds={'key':KeyError,'index':IndexError,'lookup':LookupError,'value':ValueError,'system':SystemExit}
        if kind in kinds: raise kinds[kind]('failure')
        return 'normal'
    except LookupError: return 'lookup'
    except KeyError: return 'key'
    except IndexError: return 'index'
    except Exception: return 'other'
    except BaseException: return 'base'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('mapping failure', solve('key'), 'key')
check('sequence failure', solve('index'), 'index')
check('generic lookup failure', solve('lookup'), 'lookup')
check('ordinary other error', solve('value'), 'other')
check('termination is not Exception', solve('system'), 'base')
check('normal path', solve('none'), 'normal')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Uses BaseException for the ordinary fallback and swallows control-flow exceptions and changes the required Python-language result.

Dispatch KeyError, IndexError, other LookupError, other Exception, and BaseException separately. Handler order must preserve the specific subclass cases; SystemExit belongs outside Exception. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    try:
        try:
            raise ValueError('original')
        except ValueError as original:
            error=RuntimeError('wrapper')
            if mode=='explicit':
                raise error
            elif mode=='suppressed':
                raise error from None
            else:
                raise error
    except BaseException as caught:
        pass
        return [type(caught).__name__, None if caught.__cause__ is None else type(caught.__cause__).__name__, None if caught.__context__ is None else type(caught.__context__).__name__, caught.__suppress_context__]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit cause', solve('explicit'), ['RuntimeError', 'ValueError', 'ValueError', True])
check('suppressed display', solve('suppressed'), ['RuntimeError', None, 'ValueError', True])
check('implicit context', solve('implicit'), ['RuntimeError', None, 'ValueError', False])
check('default path remains implicit', solve('other'), ['RuntimeError', None, 'ValueError', False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-exception-chaining-metadata

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    try:
        try:
            raise ValueError('original')
        except ValueError as original:
            error=RuntimeError('wrapper')
            if mode=='explicit':
                raise error from original
            elif mode=='suppressed':
                raise error
            else:
                raise error
    except BaseException as caught:
        pass
        return [type(caught).__name__, None if caught.__cause__ is None else type(caught.__cause__).__name__, None if caught.__context__ is None else type(caught.__context__).__name__, caught.__suppress_context__]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit cause', solve('explicit'), ['RuntimeError', 'ValueError', 'ValueError', True])
check('suppressed display', solve('suppressed'), ['RuntimeError', None, 'ValueError', True])
check('implicit context', solve('implicit'), ['RuntimeError', None, 'ValueError', False])
check('default path remains implicit', solve('other'), ['RuntimeError', None, 'ValueError', False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Raises the wrapper without its requested explicit cause and changes the required Python-language result.

Wrap ValueError in RuntimeError with implicit, explicit, or from-None chaining. Return [error type,cause type or None,context type or None,suppress-context flag]. Suppressing displayed context does not erase __context__. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    try:
        try:
            raise ValueError('original')
        except ValueError as original:
            error=RuntimeError('wrapper')
            if mode=='explicit':
                raise error from original
            elif mode=='suppressed':
                raise error
            else:
                raise error
    except BaseException as caught:
        pass
        return [type(caught).__name__, None if caught.__cause__ is None else type(caught.__cause__).__name__, None if caught.__context__ is None else type(caught.__context__).__name__, caught.__suppress_context__]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit cause', solve('explicit'), ['RuntimeError', 'ValueError', 'ValueError', True])
check('suppressed display', solve('suppressed'), ['RuntimeError', None, 'ValueError', True])
check('implicit context', solve('implicit'), ['RuntimeError', None, 'ValueError', False])
check('default path remains implicit', solve('other'), ['RuntimeError', None, 'ValueError', False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-exception-chaining-metadata

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    try:
        try:
            raise ValueError('original')
        except ValueError as original:
            error=RuntimeError('wrapper')
            if mode=='explicit':
                raise error from original
            elif mode=='suppressed':
                raise error from None
            else:
                raise error from original
    except BaseException as caught:
        pass
        return [type(caught).__name__, None if caught.__cause__ is None else type(caught.__cause__).__name__, None if caught.__context__ is None else type(caught.__context__).__name__, caught.__suppress_context__]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit cause', solve('explicit'), ['RuntimeError', 'ValueError', 'ValueError', True])
check('suppressed display', solve('suppressed'), ['RuntimeError', None, 'ValueError', True])
check('implicit context', solve('implicit'), ['RuntimeError', None, 'ValueError', False])
check('default path remains implicit', solve('other'), ['RuntimeError', None, 'ValueError', False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Leaves implicit context visible when from-None suppression was requested and changes the required Python-language result.

Wrap ValueError in RuntimeError with implicit, explicit, or from-None chaining. Return [error type,cause type or None,context type or None,suppress-context flag]. Suppressing displayed context does not erase __context__. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    try:
        try:
            raise ValueError('original')
        except ValueError as original:
            error=RuntimeError('wrapper')
            if mode=='explicit':
                raise error from original
            elif mode=='suppressed':
                raise error from None
            else:
                raise error from original
    except BaseException as caught:
        pass
        return [type(caught).__name__, None if caught.__cause__ is None else type(caught.__cause__).__name__, None if caught.__context__ is None else type(caught.__context__).__name__, caught.__suppress_context__]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit cause', solve('explicit'), ['RuntimeError', 'ValueError', 'ValueError', True])
check('suppressed display', solve('suppressed'), ['RuntimeError', None, 'ValueError', True])
check('implicit context', solve('implicit'), ['RuntimeError', None, 'ValueError', False])
check('default path remains implicit', solve('other'), ['RuntimeError', None, 'ValueError', False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-exception-chaining-metadata

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    try:
        try:
            raise ValueError('original')
        except ValueError as original:
            error=RuntimeError('wrapper')
            if mode=='explicit':
                raise error from original
            elif mode=='suppressed':
                raise error from None
            else:
                raise error
    except BaseException as caught:
        if caught.__suppress_context__: caught.__context__=None
        return [type(caught).__name__, None if caught.__cause__ is None else type(caught.__cause__).__name__, None if caught.__context__ is None else type(caught.__context__).__name__, caught.__suppress_context__]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit cause', solve('explicit'), ['RuntimeError', 'ValueError', 'ValueError', True])
check('suppressed display', solve('suppressed'), ['RuntimeError', None, 'ValueError', True])
check('implicit context', solve('implicit'), ['RuntimeError', None, 'ValueError', False])
check('default path remains implicit', solve('other'), ['RuntimeError', None, 'ValueError', False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Turns implicit context into an explicit cause and changes the required Python-language result.

Wrap ValueError in RuntimeError with implicit, explicit, or from-None chaining. Return [error type,cause type or None,context type or None,suppress-context flag]. Suppressing displayed context does not erase __context__. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    try:
        try:
            raise ValueError('original')
        except ValueError as original:
            error=RuntimeError('wrapper')
            if mode=='explicit':
                raise error from original
            elif mode=='suppressed':
                raise error from None
            else:
                raise error
    except BaseException as caught:
        if caught.__suppress_context__: caught.__context__=None
        return [type(caught).__name__, None if caught.__cause__ is None else type(caught.__cause__).__name__, None if caught.__context__ is None else type(caught.__context__).__name__, caught.__suppress_context__]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit cause', solve('explicit'), ['RuntimeError', 'ValueError', 'ValueError', True])
check('suppressed display', solve('suppressed'), ['RuntimeError', None, 'ValueError', True])
check('implicit context', solve('implicit'), ['RuntimeError', None, 'ValueError', False])
check('default path remains implicit', solve('other'), ['RuntimeError', None, 'ValueError', False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-exception-chaining-metadata

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    try:
        try:
            raise ValueError('original')
        except ValueError as original:
            error=RuntimeError('wrapper')
            if mode=='explicit':
                raise
            elif mode=='suppressed':
                raise
            else:
                raise
    except BaseException as caught:
        pass
        return [type(caught).__name__, None if caught.__cause__ is None else type(caught.__cause__).__name__, None if caught.__context__ is None else type(caught.__context__).__name__, caught.__suppress_context__]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit cause', solve('explicit'), ['RuntimeError', 'ValueError', 'ValueError', True])
check('suppressed display', solve('suppressed'), ['RuntimeError', None, 'ValueError', True])
check('implicit context', solve('implicit'), ['RuntimeError', None, 'ValueError', False])
check('default path remains implicit', solve('other'), ['RuntimeError', None, 'ValueError', False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Deletes the stored context when only context display should be suppressed and changes the required Python-language result.

Wrap ValueError in RuntimeError with implicit, explicit, or from-None chaining. Return [error type,cause type or None,context type or None,suppress-context flag]. Suppressing displayed context does not erase __context__. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    try:
        try:
            raise ValueError('original')
        except ValueError as original:
            error=RuntimeError('wrapper')
            if mode=='explicit':
                raise
            elif mode=='suppressed':
                raise
            else:
                raise
    except BaseException as caught:
        pass
        return [type(caught).__name__, None if caught.__cause__ is None else type(caught.__cause__).__name__, None if caught.__context__ is None else type(caught.__context__).__name__, caught.__suppress_context__]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit cause', solve('explicit'), ['RuntimeError', 'ValueError', 'ValueError', True])
check('suppressed display', solve('suppressed'), ['RuntimeError', None, 'ValueError', True])
check('implicit context', solve('implicit'), ['RuntimeError', None, 'ValueError', False])
check('default path remains implicit', solve('other'), ['RuntimeError', None, 'ValueError', False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-exception-chaining-metadata

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    try:
        try:
            raise ValueError('original')
        except ValueError as original:
            error=RuntimeError('wrapper')
            if mode=='explicit':
                raise error
            elif mode=='suppressed':
                raise error from None
            else:
                raise error
    except BaseException as caught:
        pass
        return [type(caught).__name__, None if caught.__cause__ is None else type(caught.__cause__).__name__, None if caught.__context__ is None else type(caught.__context__).__name__, caught.__suppress_context__]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('explicit cause', solve('explicit'), ['RuntimeError', 'ValueError', 'ValueError', True])
check('suppressed display', solve('suppressed'), ['RuntimeError', None, 'ValueError', True])
check('implicit context', solve('implicit'), ['RuntimeError', None, 'ValueError', False])
check('default path remains implicit', solve('other'), ['RuntimeError', None, 'ValueError', False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reraises the original exception instead of the requested wrapper and changes the required Python-language result.

Wrap ValueError in RuntimeError with implicit, explicit, or from-None chaining. Return [error type,cause type or None,context type or None,suppress-context flag]. Suppressing displayed context does not erase __context__. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(value, error, override):
    log=[]
    def f():
        try:
            log.append('try')
            if error: raise ValueError('body')
            return ['body',value]
        finally:
            log.append('finally')
            return ['cleanup',value]
    try: result=f()
    except ValueError: result='ValueError'
    return [result,log]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary body result', solve('x', False, False), [['body', 'x'], ['try', 'finally']])
check('body error preserved', solve('x', True, False), ['ValueError', ['try', 'finally']])
check('cleanup replaces successful result', solve('x', False, True), [['cleanup', 'x'], ['try', 'finally']])
check('cleanup suppresses body error', solve('x', True, True), [['cleanup', 'x'], ['try', 'finally']])
check('zero cleanup return valid', solve(0, True, True), [['cleanup', 0], ['try', 'finally']])
check('None cleanup return valid', solve(None, False, True), [['cleanup', None], ['try', 'finally']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-finally-return-precedence

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(value, error, override):
    log=[]
    def f():
        try:
            log.append('try')
            if error: raise ValueError('body')
            return ['body',value]
        finally:
            log.append('finally')
            pass
    try: result=f()
    except ValueError: result='ValueError'
    return [result,log]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary body result', solve('x', False, False), [['body', 'x'], ['try', 'finally']])
check('body error preserved', solve('x', True, False), ['ValueError', ['try', 'finally']])
check('cleanup replaces successful result', solve('x', False, True), [['cleanup', 'x'], ['try', 'finally']])
check('cleanup suppresses body error', solve('x', True, True), [['cleanup', 'x'], ['try', 'finally']])
check('zero cleanup return valid', solve(0, True, True), [['cleanup', 0], ['try', 'finally']])
check('None cleanup return valid', solve(None, False, True), [['cleanup', None], ['try', 'finally']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Returns from finally even when replacement is disabled and changes the required Python-language result.

Run cleanup for both body returns and exceptions. Only when override is true does a finally return replace the pending body result or suppress its ValueError; otherwise preserve the pending outcome. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(value, error, override):
    log=[]
    def f():
        try:
            log.append('try')
            if error: raise ValueError('body')
            return ['body',value]
        finally:
            log.append('finally')
            pass
    try: result=f()
    except ValueError: result='ValueError'
    return [result,log]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary body result', solve('x', False, False), [['body', 'x'], ['try', 'finally']])
check('body error preserved', solve('x', True, False), ['ValueError', ['try', 'finally']])
check('cleanup replaces successful result', solve('x', False, True), [['cleanup', 'x'], ['try', 'finally']])
check('cleanup suppresses body error', solve('x', True, True), [['cleanup', 'x'], ['try', 'finally']])
check('zero cleanup return valid', solve(0, True, True), [['cleanup', 0], ['try', 'finally']])
check('None cleanup return valid', solve(None, False, True), [['cleanup', None], ['try', 'finally']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-finally-return-precedence

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(value, error, override):
    log=[]
    def f():
        try:
            log.append('try')
            if error: raise ValueError('body')
            return ['body',value]
        finally:
            log.append('finally')
            if override and error: return ['cleanup',value]
    try: result=f()
    except ValueError: result='ValueError'
    return [result,log]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary body result', solve('x', False, False), [['body', 'x'], ['try', 'finally']])
check('body error preserved', solve('x', True, False), ['ValueError', ['try', 'finally']])
check('cleanup replaces successful result', solve('x', False, True), [['cleanup', 'x'], ['try', 'finally']])
check('cleanup suppresses body error', solve('x', True, True), [['cleanup', 'x'], ['try', 'finally']])
check('zero cleanup return valid', solve(0, True, True), [['cleanup', 0], ['try', 'finally']])
check('None cleanup return valid', solve(None, False, True), [['cleanup', None], ['try', 'finally']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Discards the enabled finally return and preserves the pending body outcome and changes the required Python-language result.

Run cleanup for both body returns and exceptions. Only when override is true does a finally return replace the pending body result or suppress its ValueError; otherwise preserve the pending outcome. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(value, error, override):
    log=[]
    def f():
        try:
            log.append('try')
            if error: raise ValueError('body')
            return ['body',value]
        finally:
            log.append('finally')
            if override and error: return ['cleanup',value]
    try: result=f()
    except ValueError: result='ValueError'
    return [result,log]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary body result', solve('x', False, False), [['body', 'x'], ['try', 'finally']])
check('body error preserved', solve('x', True, False), ['ValueError', ['try', 'finally']])
check('cleanup replaces successful result', solve('x', False, True), [['cleanup', 'x'], ['try', 'finally']])
check('cleanup suppresses body error', solve('x', True, True), [['cleanup', 'x'], ['try', 'finally']])
check('zero cleanup return valid', solve(0, True, True), [['cleanup', 0], ['try', 'finally']])
check('None cleanup return valid', solve(None, False, True), [['cleanup', None], ['try', 'finally']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-finally-return-precedence

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(value, error, override):
    log=[]
    def f():
        try:
            log.append('try')
            if error: raise ValueError('body')
            return ['body',value]
        finally:
            log.append('finally')
            if override and not error: return ['cleanup',value]
    try: result=f()
    except ValueError: result='ValueError'
    return [result,log]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary body result', solve('x', False, False), [['body', 'x'], ['try', 'finally']])
check('body error preserved', solve('x', True, False), ['ValueError', ['try', 'finally']])
check('cleanup replaces successful result', solve('x', False, True), [['cleanup', 'x'], ['try', 'finally']])
check('cleanup suppresses body error', solve('x', True, True), [['cleanup', 'x'], ['try', 'finally']])
check('zero cleanup return valid', solve(0, True, True), [['cleanup', 0], ['try', 'finally']])
check('None cleanup return valid', solve(None, False, True), [['cleanup', None], ['try', 'finally']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Applies the finally return only to exceptional body completion and changes the required Python-language result.

Run cleanup for both body returns and exceptions. Only when override is true does a finally return replace the pending body result or suppress its ValueError; otherwise preserve the pending outcome. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(value, error, override):
    log=[]
    def f():
        try:
            log.append('try')
            if error: raise ValueError('body')
            return ['body',value]
        finally:
            log.append('finally')
            if override and not error: return ['cleanup',value]
    try: result=f()
    except ValueError: result='ValueError'
    return [result,log]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary body result', solve('x', False, False), [['body', 'x'], ['try', 'finally']])
check('body error preserved', solve('x', True, False), ['ValueError', ['try', 'finally']])
check('cleanup replaces successful result', solve('x', False, True), [['cleanup', 'x'], ['try', 'finally']])
check('cleanup suppresses body error', solve('x', True, True), [['cleanup', 'x'], ['try', 'finally']])
check('zero cleanup return valid', solve(0, True, True), [['cleanup', 0], ['try', 'finally']])
check('None cleanup return valid', solve(None, False, True), [['cleanup', None], ['try', 'finally']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-finally-return-precedence

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(value, error, override):
    log=[]
    def f():
        try:
            log.append('try')
            if error: raise ValueError('body')
            return ['body',value]
        finally:
            log.append('finally')
            if override and value: return ['cleanup',value]
    try: result=f()
    except ValueError: result='ValueError'
    return [result,log]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary body result', solve('x', False, False), [['body', 'x'], ['try', 'finally']])
check('body error preserved', solve('x', True, False), ['ValueError', ['try', 'finally']])
check('cleanup replaces successful result', solve('x', False, True), [['cleanup', 'x'], ['try', 'finally']])
check('cleanup suppresses body error', solve('x', True, True), [['cleanup', 'x'], ['try', 'finally']])
check('zero cleanup return valid', solve(0, True, True), [['cleanup', 0], ['try', 'finally']])
check('None cleanup return valid', solve(None, False, True), [['cleanup', None], ['try', 'finally']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Refuses to let a finally return suppress a pending exception and changes the required Python-language result.

Run cleanup for both body returns and exceptions. Only when override is true does a finally return replace the pending body result or suppress its ValueError; otherwise preserve the pending outcome. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(value, error, override):
    log=[]
    def f():
        try:
            log.append('try')
            if error: raise ValueError('body')
            return ['body',value]
        finally:
            log.append('finally')
            if override and value: return ['cleanup',value]
    try: result=f()
    except ValueError: result='ValueError'
    return [result,log]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary body result', solve('x', False, False), [['body', 'x'], ['try', 'finally']])
check('body error preserved', solve('x', True, False), ['ValueError', ['try', 'finally']])
check('cleanup replaces successful result', solve('x', False, True), [['cleanup', 'x'], ['try', 'finally']])
check('cleanup suppresses body error', solve('x', True, True), [['cleanup', 'x'], ['try', 'finally']])
check('zero cleanup return valid', solve(0, True, True), [['cleanup', 0], ['try', 'finally']])
check('None cleanup return valid', solve(None, False, True), [['cleanup', None], ['try', 'finally']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-finally-return-precedence

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(value, error, override):
    log=[]
    def f():
        try:
            log.append('try')
            if error: raise ValueError('body')
            return ['body',value]
        finally:
            log.append('finally')
            return ['cleanup',value]
    try: result=f()
    except ValueError: result='ValueError'
    return [result,log]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('ordinary body result', solve('x', False, False), [['body', 'x'], ['try', 'finally']])
check('body error preserved', solve('x', True, False), ['ValueError', ['try', 'finally']])
check('cleanup replaces successful result', solve('x', False, True), [['cleanup', 'x'], ['try', 'finally']])
check('cleanup suppresses body error', solve('x', True, True), [['cleanup', 'x'], ['try', 'finally']])
check('zero cleanup return valid', solve(0, True, True), [['cleanup', 0], ['try', 'finally']])
check('None cleanup return valid', solve(None, False, True), [['cleanup', None], ['try', 'finally']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Conditions finally replacement on payload truthiness and changes the required Python-language result.

Run cleanup for both body returns and exceptions. Only when override is true does a finally return replace the pending body result or suppress its ValueError; otherwise preserve the pending outcome. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(operation, left, right):
    calls=[]
    def rhs():
        calls.append('right')
        return right
    result=bool((left and rhs()) if operation=="and" else (left or rhs()))
    return [result,calls]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('false AND returns original', solve('and', 0, 'right'), [0, []])
check('true AND evaluates right', solve('and', 'left', 'right'), ['right', ['right']])
check('true OR retains left', solve('or', 'left', 'right'), ['left', []])
check('false OR evaluates right', solve('or', None, 'right'), ['right', ['right']])
check('empty right remains empty', solve('and', 1, []), [[], ['right']])
check('empty left preserved', solve('and', [], None), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-short-circuit-operands

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(operation, left, right):
    calls=[]
    def rhs():
        calls.append('right')
        return right
    r=rhs()
    result=(left and r) if operation=='and' else (left or r)
    return [result,calls]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('false AND returns original', solve('and', 0, 'right'), [0, []])
check('true AND evaluates right', solve('and', 'left', 'right'), ['right', ['right']])
check('true OR retains left', solve('or', 'left', 'right'), ['left', []])
check('false OR evaluates right', solve('or', None, 'right'), ['right', ['right']])
check('empty right remains empty', solve('and', 1, []), [[], ['right']])
check('empty left preserved', solve('and', [], None), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Converts the selected operand into a Boolean result and changes the required Python-language result.

Evaluate left and rhs() or left or rhs(), returning [the actual selected operand,call log]. Python Boolean operators preserve operand values and call the right side only when needed. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(operation, left, right):
    calls=[]
    def rhs():
        calls.append('right')
        return right
    r=rhs()
    result=(left and r) if operation=='and' else (left or r)
    return [result,calls]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('false AND returns original', solve('and', 0, 'right'), [0, []])
check('true AND evaluates right', solve('and', 'left', 'right'), ['right', ['right']])
check('true OR retains left', solve('or', 'left', 'right'), ['left', []])
check('false OR evaluates right', solve('or', None, 'right'), ['right', ['right']])
check('empty right remains empty', solve('and', 1, []), [[], ['right']])
check('empty left preserved', solve('and', [], None), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-short-circuit-operands

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(operation, left, right):
    calls=[]
    def rhs():
        calls.append('right')
        return right
    result=left or rhs()
    return [result,calls]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('false AND returns original', solve('and', 0, 'right'), [0, []])
check('true AND evaluates right', solve('and', 'left', 'right'), ['right', ['right']])
check('true OR retains left', solve('or', 'left', 'right'), ['left', []])
check('false OR evaluates right', solve('or', None, 'right'), ['right', ['right']])
check('empty right remains empty', solve('and', 1, []), [[], ['right']])
check('empty left preserved', solve('and', [], None), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Evaluates the right operand even when the left short-circuits and changes the required Python-language result.

Evaluate left and rhs() or left or rhs(), returning [the actual selected operand,call log]. Python Boolean operators preserve operand values and call the right side only when needed. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(operation, left, right):
    calls=[]
    def rhs():
        calls.append('right')
        return right
    result=left or rhs()
    return [result,calls]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('false AND returns original', solve('and', 0, 'right'), [0, []])
check('true AND evaluates right', solve('and', 'left', 'right'), ['right', ['right']])
check('true OR retains left', solve('or', 'left', 'right'), ['left', []])
check('false OR evaluates right', solve('or', None, 'right'), ['right', ['right']])
check('empty right remains empty', solve('and', 1, []), [[], ['right']])
check('empty left preserved', solve('and', [], None), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-short-circuit-operands

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(operation, left, right):
    calls=[]
    def rhs():
        calls.append('right')
        return right
    result=left and rhs()
    return [result,calls]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('false AND returns original', solve('and', 0, 'right'), [0, []])
check('true AND evaluates right', solve('and', 'left', 'right'), ['right', ['right']])
check('true OR retains left', solve('or', 'left', 'right'), ['left', []])
check('false OR evaluates right', solve('or', None, 'right'), ['right', ['right']])
check('empty right remains empty', solve('and', 1, []), [[], ['right']])
check('empty left preserved', solve('and', [], None), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Uses OR semantics for the AND operation and changes the required Python-language result.

Evaluate left and rhs() or left or rhs(), returning [the actual selected operand,call log]. Python Boolean operators preserve operand values and call the right side only when needed. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(operation, left, right):
    calls=[]
    def rhs():
        calls.append('right')
        return right
    result=left and rhs()
    return [result,calls]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('false AND returns original', solve('and', 0, 'right'), [0, []])
check('true AND evaluates right', solve('and', 'left', 'right'), ['right', ['right']])
check('true OR retains left', solve('or', 'left', 'right'), ['left', []])
check('false OR evaluates right', solve('or', None, 'right'), ['right', ['right']])
check('empty right remains empty', solve('and', 1, []), [[], ['right']])
check('empty left preserved', solve('and', [], None), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-short-circuit-operands

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(operation, left, right):
    calls=[]
    def rhs():
        calls.append('right')
        return right
    result=(left and bool(rhs())) if operation=="and" else (left or bool(rhs()))
    return [result,calls]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('false AND returns original', solve('and', 0, 'right'), [0, []])
check('true AND evaluates right', solve('and', 'left', 'right'), ['right', ['right']])
check('true OR retains left', solve('or', 'left', 'right'), ['left', []])
check('false OR evaluates right', solve('or', None, 'right'), ['right', ['right']])
check('empty right remains empty', solve('and', 1, []), [[], ['right']])
check('empty left preserved', solve('and', [], None), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Uses AND semantics for the OR operation and changes the required Python-language result.

Evaluate left and rhs() or left or rhs(), returning [the actual selected operand,call log]. Python Boolean operators preserve operand values and call the right side only when needed. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(operation, left, right):
    calls=[]
    def rhs():
        calls.append('right')
        return right
    result=(left and bool(rhs())) if operation=="and" else (left or bool(rhs()))
    return [result,calls]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('false AND returns original', solve('and', 0, 'right'), [0, []])
check('true AND evaluates right', solve('and', 'left', 'right'), ['right', ['right']])
check('true OR retains left', solve('or', 'left', 'right'), ['left', []])
check('false OR evaluates right', solve('or', None, 'right'), ['right', ['right']])
check('empty right remains empty', solve('and', 1, []), [[], ['right']])
check('empty left preserved', solve('and', [], None), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-short-circuit-operands

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(operation, left, right):
    calls=[]
    def rhs():
        calls.append('right')
        return right
    result=bool((left and rhs()) if operation=="and" else (left or rhs()))
    return [result,calls]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('false AND returns original', solve('and', 0, 'right'), [0, []])
check('true AND evaluates right', solve('and', 'left', 'right'), ['right', ['right']])
check('true OR retains left', solve('or', 'left', 'right'), ['left', []])
check('false OR evaluates right', solve('or', None, 'right'), ['right', ['right']])
check('empty right remains empty', solve('and', 1, []), [[], ['right']])
check('empty left preserved', solve('and', [], None), [[], []])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Converts only evaluated right-hand results into Boolean values and changes the required Python-language result.

Evaluate left and rhs() or left or rhs(), returning [the actual selected operand,call log]. Python Boolean operators preserve operand values and call the right side only when needed. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, mid, right):
    trace=[]
    def lhs(): trace.append('left'); return left
    def middle(): trace.append('middle'); return mid
    def rhs(): trace.append('right'); return right
    result=lhs() < middle() and middle() < rhs()
    return [result,trace]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('increasing triple', solve(1, 2, 3), [True, ['left', 'middle', 'right']])
check('first comparison fails', solve(3, 2, 9), [False, ['left', 'middle']])
check('equal first operands', solve(2, 2, 3), [False, ['left', 'middle']])
check('equal last operands', solve(1, 2, 2), [False, ['left', 'middle', 'right']])
check('second comparison fails', solve(1, 3, 2), [False, ['left', 'middle', 'right']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-chained-comparison-evaluation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, mid, right):
    trace=[]
    def lhs(): trace.append('left'); return left
    def middle(): trace.append('middle'); return mid
    def rhs(): trace.append('right'); return right
    a,b,c=lhs(),middle(),rhs()
    return [a<b<c,trace]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('increasing triple', solve(1, 2, 3), [True, ['left', 'middle', 'right']])
check('first comparison fails', solve(3, 2, 9), [False, ['left', 'middle']])
check('equal first operands', solve(2, 2, 3), [False, ['left', 'middle']])
check('equal last operands', solve(1, 2, 2), [False, ['left', 'middle', 'right']])
check('second comparison fails', solve(1, 3, 2), [False, ['left', 'middle', 'right']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reevaluates the shared middle expression after the first comparison and changes the required Python-language result.

Evaluate lhs() < middle() < rhs(). Evaluate the middle expression once, evaluate each operand left-to-right, and skip the final expression when the first comparison fails. Both inequalities are strict. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, mid, right):
    trace=[]
    def lhs(): trace.append('left'); return left
    def middle(): trace.append('middle'); return mid
    def rhs(): trace.append('right'); return right
    a,b,c=lhs(),middle(),rhs()
    return [a<b<c,trace]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('increasing triple', solve(1, 2, 3), [True, ['left', 'middle', 'right']])
check('first comparison fails', solve(3, 2, 9), [False, ['left', 'middle']])
check('equal first operands', solve(2, 2, 3), [False, ['left', 'middle']])
check('equal last operands', solve(1, 2, 2), [False, ['left', 'middle', 'right']])
check('second comparison fails', solve(1, 3, 2), [False, ['left', 'middle', 'right']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-chained-comparison-evaluation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, mid, right):
    trace=[]
    def lhs(): trace.append('left'); return left
    def middle(): trace.append('middle'); return mid
    def rhs(): trace.append('right'); return right
    result=lhs() <= middle() < rhs()
    return [result,trace]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('increasing triple', solve(1, 2, 3), [True, ['left', 'middle', 'right']])
check('first comparison fails', solve(3, 2, 9), [False, ['left', 'middle']])
check('equal first operands', solve(2, 2, 3), [False, ['left', 'middle']])
check('equal last operands', solve(1, 2, 2), [False, ['left', 'middle', 'right']])
check('second comparison fails', solve(1, 3, 2), [False, ['left', 'middle', 'right']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Evaluates the right operand before knowing whether the first comparison passes and changes the required Python-language result.

Evaluate lhs() < middle() < rhs(). Evaluate the middle expression once, evaluate each operand left-to-right, and skip the final expression when the first comparison fails. Both inequalities are strict. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, mid, right):
    trace=[]
    def lhs(): trace.append('left'); return left
    def middle(): trace.append('middle'); return mid
    def rhs(): trace.append('right'); return right
    result=lhs() <= middle() < rhs()
    return [result,trace]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('increasing triple', solve(1, 2, 3), [True, ['left', 'middle', 'right']])
check('first comparison fails', solve(3, 2, 9), [False, ['left', 'middle']])
check('equal first operands', solve(2, 2, 3), [False, ['left', 'middle']])
check('equal last operands', solve(1, 2, 2), [False, ['left', 'middle', 'right']])
check('second comparison fails', solve(1, 3, 2), [False, ['left', 'middle', 'right']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-chained-comparison-evaluation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, mid, right):
    trace=[]
    def lhs(): trace.append('left'); return left
    def middle(): trace.append('middle'); return mid
    def rhs(): trace.append('right'); return right
    result=lhs() < middle() <= rhs()
    return [result,trace]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('increasing triple', solve(1, 2, 3), [True, ['left', 'middle', 'right']])
check('first comparison fails', solve(3, 2, 9), [False, ['left', 'middle']])
check('equal first operands', solve(2, 2, 3), [False, ['left', 'middle']])
check('equal last operands', solve(1, 2, 2), [False, ['left', 'middle', 'right']])
check('second comparison fails', solve(1, 3, 2), [False, ['left', 'middle', 'right']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Changes the first strict comparison into an inclusive one and changes the required Python-language result.

Evaluate lhs() < middle() < rhs(). Evaluate the middle expression once, evaluate each operand left-to-right, and skip the final expression when the first comparison fails. Both inequalities are strict. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, mid, right):
    trace=[]
    def lhs(): trace.append('left'); return left
    def middle(): trace.append('middle'); return mid
    def rhs(): trace.append('right'); return right
    result=lhs() < middle() <= rhs()
    return [result,trace]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('increasing triple', solve(1, 2, 3), [True, ['left', 'middle', 'right']])
check('first comparison fails', solve(3, 2, 9), [False, ['left', 'middle']])
check('equal first operands', solve(2, 2, 3), [False, ['left', 'middle']])
check('equal last operands', solve(1, 2, 2), [False, ['left', 'middle', 'right']])
check('second comparison fails', solve(1, 3, 2), [False, ['left', 'middle', 'right']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-chained-comparison-evaluation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, mid, right):
    trace=[]
    def lhs(): trace.append('left'); return left
    def middle(): trace.append('middle'); return mid
    def rhs(): trace.append('right'); return right
    result=lhs() < middle() > rhs()
    return [result,trace]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('increasing triple', solve(1, 2, 3), [True, ['left', 'middle', 'right']])
check('first comparison fails', solve(3, 2, 9), [False, ['left', 'middle']])
check('equal first operands', solve(2, 2, 3), [False, ['left', 'middle']])
check('equal last operands', solve(1, 2, 2), [False, ['left', 'middle', 'right']])
check('second comparison fails', solve(1, 3, 2), [False, ['left', 'middle', 'right']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Changes the second strict comparison into an inclusive one and changes the required Python-language result.

Evaluate lhs() < middle() < rhs(). Evaluate the middle expression once, evaluate each operand left-to-right, and skip the final expression when the first comparison fails. Both inequalities are strict. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, mid, right):
    trace=[]
    def lhs(): trace.append('left'); return left
    def middle(): trace.append('middle'); return mid
    def rhs(): trace.append('right'); return right
    result=lhs() < middle() > rhs()
    return [result,trace]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('increasing triple', solve(1, 2, 3), [True, ['left', 'middle', 'right']])
check('first comparison fails', solve(3, 2, 9), [False, ['left', 'middle']])
check('equal first operands', solve(2, 2, 3), [False, ['left', 'middle']])
check('equal last operands', solve(1, 2, 2), [False, ['left', 'middle', 'right']])
check('second comparison fails', solve(1, 3, 2), [False, ['left', 'middle', 'right']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-chained-comparison-evaluation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, mid, right):
    trace=[]
    def lhs(): trace.append('left'); return left
    def middle(): trace.append('middle'); return mid
    def rhs(): trace.append('right'); return right
    result=lhs() < middle() and middle() < rhs()
    return [result,trace]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('increasing triple', solve(1, 2, 3), [True, ['left', 'middle', 'right']])
check('first comparison fails', solve(3, 2, 9), [False, ['left', 'middle']])
check('equal first operands', solve(2, 2, 3), [False, ['left', 'middle']])
check('equal last operands', solve(1, 2, 2), [False, ['left', 'middle', 'right']])
check('second comparison fails', solve(1, 3, 2), [False, ['left', 'middle', 'right']])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reverses the direction of the second comparison and changes the required Python-language result.

Evaluate lhs() < middle() < rhs(). Evaluate the middle expression once, evaluate each operand left-to-right, and skip the final expression when the first comparison fails. Both inequalities are strict. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b):
    return a // b
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (7, 3)', solve(*(7, 3)), 3)
check('fixture 2: (-7, 3)', solve(*(-7, 3)), -2)
check('fixture 3: (6, 3)', solve(*(6, 3)), 2)
check('fixture 4: (0, 3)', solve(*(0, 3)), 0)
check('fixture 5: (-1, 3)', solve(*(-1, 3)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b):
    return (a + b - 1) // b if a >= 0 else -((abs(a)+b-1) // b)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (7, 3)', solve(*(7, 3)), 3)
check('fixture 2: (-7, 3)', solve(*(-7, 3)), -2)
check('fixture 3: (6, 3)', solve(*(6, 3)), 2)
check('fixture 4: (0, 3)', solve(*(0, 3)), 0)
check('fixture 5: (-1, 3)', solve(*(-1, 3)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Floor division is substituted for ceiling division.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. Return the least integer greater than or equal to a/b, including negative a. Exact operational definition: -(-a // b)

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b):
    return a - int(a / b) * b
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (-7, 3)', solve(*(-7, 3)), 2)
check('fixture 2: (7, 3)', solve(*(7, 3)), 1)
check('fixture 3: (-6, 3)', solve(*(-6, 3)), 0)
check('fixture 4: (0, 4)', solve(*(0, 4)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b):
    return abs(a) % b
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (-7, 3)', solve(*(-7, 3)), 2)
check('fixture 2: (7, 3)', solve(*(7, 3)), 1)
check('fixture 3: (-6, 3)', solve(*(-6, 3)), 0)
check('fixture 4: (0, 4)', solve(*(0, 4)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Truncating division leaves a negative remainder.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. Return r in [0,b) such that a-r is divisible by b. Exact operational definition: a % b

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b, m):
    return abs(a-b)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (1, 9, 10)', solve(*(1, 9, 10)), 2)
check('fixture 2: (2, 5, 10)', solve(*(2, 5, 10)), 3)
check('fixture 3: (12, 2, 10)', solve(*(12, 2, 10)), 0)
check('fixture 4: (-1, 1, 10)', solve(*(-1, 1, 10)), 2)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b, m):
    return abs(a-b)%m
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (1, 9, 10)', solve(*(1, 9, 10)), 2)
check('fixture 2: (2, 5, 10)', solve(*(2, 5, 10)), 3)
check('fixture 3: (12, 2, 10)', solve(*(12, 2, 10)), 0)
check('fixture 4: (-1, 1, 10)', solve(*(-1, 1, 10)), 2)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Linear distance ignores wraparound.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. Return the smaller nonnegative arc length on a cycle of circumference m. Exact operational definition: min((a-b)%m, (b-a)%m)

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, step):
    return round(a/step)*step
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (5, 10)', solve(*(5, 10)), 10)
check('fixture 2: (4, 10)', solve(*(4, 10)), 0)
check('fixture 3: (-5, 10)', solve(*(-5, 10)), 0)
check('fixture 4: (-6, 10)', solve(*(-6, 10)), -10)
check('fixture 5: (20, 10)', solve(*(20, 10)), 20)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, step):
    return ((a + step) // step)*step
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (5, 10)', solve(*(5, 10)), 10)
check('fixture 2: (4, 10)', solve(*(4, 10)), 0)
check('fixture 3: (-5, 10)', solve(*(-5, 10)), 0)
check('fixture 4: (-6, 10)', solve(*(-6, 10)), -10)
check('fixture 5: (20, 10)', solve(*(20, 10)), 20)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Banker rounding violates the upward midpoint convention.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. Return the nearest multiple of step; exact halfway cases choose the greater signed result. Exact operational definition: ((2*a + step) // (2*step))*step

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(n):
    return int(math.sqrt(n))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (81129638414606681695789005144063,)', solve(*(81129638414606681695789005144063,)), 9007199254740991)
check('fixture 2: (15,)', solve(*(15,)), 3)
check('fixture 3: (16,)', solve(*(16,)), 4)
check('fixture 4: (0,)', solve(*(0,)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(n):
    return round(math.sqrt(n))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (81129638414606681695789005144063,)', solve(*(81129638414606681695789005144063,)), 9007199254740991)
check('fixture 2: (15,)', solve(*(15,)), 3)
check('fixture 3: (16,)', solve(*(16,)), 4)
check('fixture 4: (0,)', solve(*(0,)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Binary float conversion erases low bits before the root.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. n is nonnegative. Return the greatest r such that r*r<=n. Exact operational definition: math.isqrt(n)

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(n):
    return math.isqrt(n)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (0,)', solve(*(0,)), 0)
check('fixture 2: (1,)', solve(*(1,)), 1)
check('fixture 3: (2,)', solve(*(2,)), 2)
check('fixture 4: (16,)', solve(*(16,)), 4)
check('fixture 5: (17,)', solve(*(17,)), 5)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(n):
    return math.isqrt(n)+1
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (0,)', solve(*(0,)), 0)
check('fixture 2: (1,)', solve(*(1,)), 1)
check('fixture 3: (2,)', solve(*(2,)), 2)
check('fixture 4: (16,)', solve(*(16,)), 4)
check('fixture 5: (17,)', solve(*(17,)), 5)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The lower root is used when an enclosing square is required.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. n is nonnegative. Return the least r>=0 such that r*r>=n. Exact operational definition: math.isqrt(n) + (math.isqrt(n)**2 < n)

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(n):
    return len(bin(n))-2
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (0,)', solve(*(0,)), 0)
check('fixture 2: (-8,)', solve(*(-8,)), 4)
check('fixture 3: (7,)', solve(*(7,)), 3)
check('fixture 4: (8,)', solve(*(8,)), 4)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(n):
    return len(bin(n).lstrip("-"))-2
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (0,)', solve(*(0,)), 0)
check('fixture 2: (-8,)', solve(*(-8,)), 4)
check('fixture 3: (7,)', solve(*(7,)), 3)
check('fixture 4: (8,)', solve(*(8,)), 4)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The minus sign is counted as a binary digit.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. Return the number of significant binary magnitude digits, with zero requiring zero digits. Exact operational definition: abs(n).bit_length()

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(n):
    return abs(n).bit_length()+1
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (-8,)', solve(*(-8,)), 4)
check('fixture 2: (7,)', solve(*(7,)), 4)
check('fixture 3: (0,)', solve(*(0,)), 1)
check('fixture 4: (-1,)', solve(*(-1,)), 1)
check('fixture 5: (8,)', solve(*(8,)), 5)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(n):
    return max(1, abs(n).bit_length())
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (-8,)', solve(*(-8,)), 4)
check('fixture 2: (7,)', solve(*(7,)), 4)
check('fixture 3: (0,)', solve(*(0,)), 1)
check('fixture 4: (-1,)', solve(*(-1,)), 1)
check('fixture 5: (8,)', solve(*(8,)), 5)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Negative powers of two are allocated an unnecessary sign bit.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. Return the smallest positive width whose signed two-complement range contains n. Exact operational definition: max(1, (n if n >= 0 else ~n).bit_length()+1)

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b, bits):
    return (a+b)% (1<<bits)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (250, 10, 8)', solve(*(250, 10, 8)), 255)
check('fixture 2: (2, 3, 8)', solve(*(2, 3, 8)), 5)
check('fixture 3: (0, 0, 1)', solve(*(0, 0, 1)), 0)
check('fixture 4: (1, 1, 1)', solve(*(1, 1, 1)), 1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b, bits):
    return min(1<<bits, a+b)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (250, 10, 8)', solve(*(250, 10, 8)), 255)
check('fixture 2: (2, 3, 8)', solve(*(2, 3, 8)), 5)
check('fixture 3: (0, 0, 1)', solve(*(0, 0, 1)), 0)
check('fixture 4: (1, 1, 1)', solve(*(1, 1, 1)), 1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Overflow wraps instead of saturating.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. a and b are nonnegative; cap their sum at 2**bits-1. Exact operational definition: min((1<<bits)-1, a+b)

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b, bits):
    return (a-b)%(1<<bits)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (1, 3, 8)', solve(*(1, 3, 8)), 0)
check('fixture 2: (3, 1, 8)', solve(*(3, 1, 8)), 2)
check('fixture 3: (0, 0, 8)', solve(*(0, 0, 8)), 0)
check('fixture 4: (255, 255, 8)', solve(*(255, 255, 8)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b, bits):
    return abs(a-b)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (1, 3, 8)', solve(*(1, 3, 8)), 0)
check('fixture 2: (3, 1, 8)', solve(*(3, 1, 8)), 2)
check('fixture 3: (0, 0, 8)', solve(*(0, 0, 8)), 0)
check('fixture 4: (255, 255, 8)', solve(*(255, 255, 8)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Underflow wraps into a large positive number.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. a and b are unsigned words; floor a-b at zero. Exact operational definition: max(0,a-b)

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b, bits):
    return min((1<<(bits-1))-1,a+b)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (-120, -20, 8)', solve(*(-120, -20, 8)), -128)
check('fixture 2: (120, 20, 8)', solve(*(120, 20, 8)), 127)
check('fixture 3: (1, -1, 8)', solve(*(1, -1, 8)), 0)
check('fixture 4: (-128, 0, 8)', solve(*(-128, 0, 8)), -128)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(a, b, bits):
    return max(-(1<<(bits-1))+1, min((1<<(bits-1))-1,a+b))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (-120, -20, 8)', solve(*(-120, -20, 8)), -128)
check('fixture 2: (120, 20, 8)', solve(*(120, 20, 8)), 127)
check('fixture 3: (1, -1, 8)', solve(*(1, -1, 8)), 0)
check('fixture 4: (-128, 0, 8)', solve(*(-128, 0, 8)), -128)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Only positive overflow is bounded.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. Cap a+b to [-2**(bits-1),2**(bits-1)-1]. Exact operational definition: max(-(1<<(bits-1)), min((1<<(bits-1))-1,a+b))

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(n, k, bits):
    return (n << k) & ((1<<bits)-1)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (129, 1, 8)', solve(*(129, 1, 8)), 3)
check('fixture 2: (3, 2, 8)', solve(*(3, 2, 8)), 12)
check('fixture 3: (128, 8, 8)', solve(*(128, 8, 8)), 128)
check('fixture 4: (15, 0, 8)', solve(*(15, 0, 8)), 15)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
import calendar
import statistics
import itertools
from fractions import Fraction
from datetime import date, datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR

N = 1
observations = []
def solve(n, k, bits):
    return ((n << k) | (n >> k)) & ((1<<bits)-1)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1: (129, 1, 8)', solve(*(129, 1, 8)), 3)
check('fixture 2: (3, 2, 8)', solve(*(3, 2, 8)), 12)
check('fixture 3: (128, 8, 8)', solve(*(128, 8, 8)), 128)
check('fixture 4: (15, 0, 8)', solve(*(15, 0, 8)), 15)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A logical shift discards bits that must rotate back in.

Integer arguments; denominators, steps, moduli, and bit widths are positive. Unsigned word inputs fit the specified width except where masking is explicitly the operation. Sequence sizes are nonnegative. n is an unsigned word and k is nonnegative; rotate by k modulo bits. Exact operational definition: ((n << (k%bits)) | (n >> ((bits-k)%bits))) & ((1<<bits)-1)

split: open-access