broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(args, kwargs):
    def f(a, **rest):
        return [a,[[k,v] for k,v in rest.items()]]
    try:
        return f(*args, **kwargs)
    except TypeError:
        return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('same spelling legal', solve([1], {'a': 2}), [1, [['a', 2]]])
check('ordered kwargs', solve([1], {'z': 0, 'a': 3, 'b': None}), [1, [['z', 0], ['a', 3], ['b', None]]])
check('no keywords', solve([None], {}), [None, []])
check('keyword cannot supply positional', solve([], {'a': 2}), 'TypeError')
check('too many positions', solve([1, 2], {}), 'TypeError')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-positional-name-in-varkw

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(args, kwargs):
    def f(a, /, **rest):
        return [a,[[k,v] for k,v in rest.items() if k!="a"]]
    try:
        return f(*args, **kwargs)
    except TypeError:
        return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('same spelling legal', solve([1], {'a': 2}), [1, [['a', 2]]])
check('ordered kwargs', solve([1], {'z': 0, 'a': 3, 'b': None}), [1, [['z', 0], ['a', 3], ['b', None]]])
check('no keywords', solve([None], {}), [None, []])
check('keyword cannot supply positional', solve([], {'a': 2}), 'TypeError')
check('too many positions', solve([1, 2], {}), 'TypeError')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Interprets the same-named keyword as a duplicate positional binding and changes the required Python-language result.

Call f(a, /, **rest), returning the positional a and the ordered [name,value] keyword pairs. A keyword named a belongs to rest. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(args, kwargs):
    def f(a, /, **rest):
        return [a,[[k,v] for k,v in rest.items() if k!="a"]]
    try:
        return f(*args, **kwargs)
    except TypeError:
        return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('same spelling legal', solve([1], {'a': 2}), [1, [['a', 2]]])
check('ordered kwargs', solve([1], {'z': 0, 'a': 3, 'b': None}), [1, [['z', 0], ['a', 3], ['b', None]]])
check('no keywords', solve([None], {}), [None, []])
check('keyword cannot supply positional', solve([], {'a': 2}), 'TypeError')
check('too many positions', solve([1, 2], {}), 'TypeError')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xl-positional-name-in-varkw

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(args, kwargs):
    def f(a=None, **rest):
        return [a,[[k,v] for k,v in rest.items()]]
    try:
        return f(*args, **kwargs)
    except TypeError:
        return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('same spelling legal', solve([1], {'a': 2}), [1, [['a', 2]]])
check('ordered kwargs', solve([1], {'z': 0, 'a': 3, 'b': None}), [1, [['z', 0], ['a', 3], ['b', None]]])
check('no keywords', solve([None], {}), [None, []])
check('keyword cannot supply positional', solve([], {'a': 2}), 'TypeError')
check('too many positions', solve([1, 2], {}), 'TypeError')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Drops the legal same-named entry from variadic keywords and changes the required Python-language result.

Call f(a, /, **rest), returning the positional a and the ordered [name,value] keyword pairs. A keyword named a belongs to rest. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(args, kwargs):
    def f(a=None, **rest):
        return [a,[[k,v] for k,v in rest.items()]]
    try:
        return f(*args, **kwargs)
    except TypeError:
        return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('same spelling legal', solve([1], {'a': 2}), [1, [['a', 2]]])
check('ordered kwargs', solve([1], {'z': 0, 'a': 3, 'b': None}), [1, [['z', 0], ['a', 3], ['b', None]]])
check('no keywords', solve([None], {}), [None, []])
check('keyword cannot supply positional', solve([], {'a': 2}), 'TypeError')
check('too many positions', solve([1, 2], {}), 'TypeError')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-positional-name-in-varkw

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(args, kwargs):
    def f(a, /, *tail, **rest):
        return [a,[[k,v] for k,v in rest.items()]]
    try:
        return f(*args, **kwargs)
    except TypeError:
        return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('same spelling legal', solve([1], {'a': 2}), [1, [['a', 2]]])
check('ordered kwargs', solve([1], {'z': 0, 'a': 3, 'b': None}), [1, [['z', 0], ['a', 3], ['b', None]]])
check('no keywords', solve([None], {}), [None, []])
check('keyword cannot supply positional', solve([], {'a': 2}), 'TypeError')
check('too many positions', solve([1, 2], {}), 'TypeError')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Allows the required positional value to be supplied through keyword rest and changes the required Python-language result.

Call f(a, /, **rest), returning the positional a and the ordered [name,value] keyword pairs. A keyword named a belongs to rest. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(args, kwargs):
    def f(a, /, *tail, **rest):
        return [a,[[k,v] for k,v in rest.items()]]
    try:
        return f(*args, **kwargs)
    except TypeError:
        return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('same spelling legal', solve([1], {'a': 2}), [1, [['a', 2]]])
check('ordered kwargs', solve([1], {'z': 0, 'a': 3, 'b': None}), [1, [['z', 0], ['a', 3], ['b', None]]])
check('no keywords', solve([None], {}), [None, []])
check('keyword cannot supply positional', solve([], {'a': 2}), 'TypeError')
check('too many positions', solve([1, 2], {}), 'TypeError')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-positional-name-in-varkw

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(args, kwargs):
    def f(a, /, **rest):
        return [rest.get("a",a),[[k,v] for k,v in rest.items()]]
    try:
        return f(*args, **kwargs)
    except TypeError:
        return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('same spelling legal', solve([1], {'a': 2}), [1, [['a', 2]]])
check('ordered kwargs', solve([1], {'z': 0, 'a': 3, 'b': None}), [1, [['z', 0], ['a', 3], ['b', None]]])
check('no keywords', solve([None], {}), [None, []])
check('keyword cannot supply positional', solve([], {'a': 2}), 'TypeError')
check('too many positions', solve([1, 2], {}), 'TypeError')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Accepts extra positional arguments despite having no variadic positional parameter and changes the required Python-language result.

Call f(a, /, **rest), returning the positional a and the ordered [name,value] keyword pairs. A keyword named a belongs to rest. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(args, kwargs):
    def f(a, /, **rest):
        return [rest.get("a",a),[[k,v] for k,v in rest.items()]]
    try:
        return f(*args, **kwargs)
    except TypeError:
        return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('same spelling legal', solve([1], {'a': 2}), [1, [['a', 2]]])
check('ordered kwargs', solve([1], {'z': 0, 'a': 3, 'b': None}), [1, [['z', 0], ['a', 3], ['b', None]]])
check('no keywords', solve([None], {}), [None, []])
check('keyword cannot supply positional', solve([], {'a': 2}), 'TypeError')
check('too many positions', solve([1, 2], {}), 'TypeError')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-positional-name-in-varkw

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(args, kwargs):
    def f(a, **rest):
        return [a,[[k,v] for k,v in rest.items()]]
    try:
        return f(*args, **kwargs)
    except TypeError:
        return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('same spelling legal', solve([1], {'a': 2}), [1, [['a', 2]]])
check('ordered kwargs', solve([1], {'z': 0, 'a': 3, 'b': None}), [1, [['z', 0], ['a', 3], ['b', None]]])
check('no keywords', solve([None], {}), [None, []])
check('keyword cannot supply positional', solve([], {'a': 2}), 'TypeError')
check('too many positions', solve([1, 2], {}), 'TypeError')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Lets the keyword named a overwrite the positional-only value and changes the required Python-language result.

Call f(a, /, **rest), returning the positional a and the ordered [name,value] keyword pairs. A keyword named a belongs to rest. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, right):
    def f(**kw): return [[k,v] for k,v in kw.items()]
    try: return f(**{**left,**right})
    except TypeError: return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate equal rejected', solve({'a': 1}, {'a': 1}), 'TypeError')
check('duplicate different rejected', solve({'a': 1}, {'a': 2}), 'TypeError')
check('integer key rejected', solve({1: 'v'}, {}), 'TypeError')
check('empty name legal', solve({'': 2}, {}), [['', 2]])
check('hyphenated name legal', solve({'a-b': 3}, {}), [['a-b', 3]])
check('left then right order', solve({'z': 1}, {'a': 2}), [['z', 1], ['a', 2]])
check('empty expansions', solve({}, {}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-keyword-unpacking-collisions

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, right):
    if any(k in left and left[k]!=v for k,v in right.items()): return "TypeError"
    kw={**left,**right}
    return [[k,v] for k,v in kw.items()] if all(isinstance(k,str) for k in kw) else "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate equal rejected', solve({'a': 1}, {'a': 1}), 'TypeError')
check('duplicate different rejected', solve({'a': 1}, {'a': 2}), 'TypeError')
check('integer key rejected', solve({1: 'v'}, {}), 'TypeError')
check('empty name legal', solve({'': 2}, {}), [['', 2]])
check('hyphenated name legal', solve({'a-b': 3}, {}), [['a-b', 3]])
check('left then right order', solve({'z': 1}, {'a': 2}), [['z', 1], ['a', 2]])
check('empty expansions', solve({}, {}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Merges mappings before call binding and overwrites duplicate keywords and changes the required Python-language result.

Invoke f(**left, **right), returning ordered keyword pairs. Reject any duplicate keyword across mappings even when values agree; keys must be strings. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, right):
    if any(k in left and left[k]!=v for k,v in right.items()): return "TypeError"
    kw={**left,**right}
    return [[k,v] for k,v in kw.items()] if all(isinstance(k,str) for k in kw) else "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate equal rejected', solve({'a': 1}, {'a': 1}), 'TypeError')
check('duplicate different rejected', solve({'a': 1}, {'a': 2}), 'TypeError')
check('integer key rejected', solve({1: 'v'}, {}), 'TypeError')
check('empty name legal', solve({'': 2}, {}), [['', 2]])
check('hyphenated name legal', solve({'a-b': 3}, {}), [['a-b', 3]])
check('left then right order', solve({'z': 1}, {'a': 2}), [['z', 1], ['a', 2]])
check('empty expansions', solve({}, {}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-keyword-unpacking-collisions

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, right):
    def f(**kw): return [[k,v] for k,v in kw.items()]
    try: return f(**{str(k):v for k,v in left.items()},**{str(k):v for k,v in right.items()})
    except TypeError: return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate equal rejected', solve({'a': 1}, {'a': 1}), 'TypeError')
check('duplicate different rejected', solve({'a': 1}, {'a': 2}), 'TypeError')
check('integer key rejected', solve({1: 'v'}, {}), 'TypeError')
check('empty name legal', solve({'': 2}, {}), [['', 2]])
check('hyphenated name legal', solve({'a-b': 3}, {}), [['a-b', 3]])
check('left then right order', solve({'z': 1}, {'a': 2}), [['z', 1], ['a', 2]])
check('empty expansions', solve({}, {}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Accepts repeated keywords when their values compare equal and changes the required Python-language result.

Invoke f(**left, **right), returning ordered keyword pairs. Reject any duplicate keyword across mappings even when values agree; keys must be strings. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, right):
    def f(**kw): return [[k,v] for k,v in kw.items()]
    try: return f(**{str(k):v for k,v in left.items()},**{str(k):v for k,v in right.items()})
    except TypeError: return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate equal rejected', solve({'a': 1}, {'a': 1}), 'TypeError')
check('duplicate different rejected', solve({'a': 1}, {'a': 2}), 'TypeError')
check('integer key rejected', solve({1: 'v'}, {}), 'TypeError')
check('empty name legal', solve({'': 2}, {}), [['', 2]])
check('hyphenated name legal', solve({'a-b': 3}, {}), [['a-b', 3]])
check('left then right order', solve({'z': 1}, {'a': 2}), [['z', 1], ['a', 2]])
check('empty expansions', solve({}, {}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-keyword-unpacking-collisions

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, right):
    if "" in left or "" in right: return "TypeError"
    def f(**kw): return [[k,v] for k,v in kw.items()]
    try: return f(**left, **right)
    except TypeError: return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate equal rejected', solve({'a': 1}, {'a': 1}), 'TypeError')
check('duplicate different rejected', solve({'a': 1}, {'a': 2}), 'TypeError')
check('integer key rejected', solve({1: 'v'}, {}), 'TypeError')
check('empty name legal', solve({'': 2}, {}), [['', 2]])
check('hyphenated name legal', solve({'a-b': 3}, {}), [['a-b', 3]])
check('left then right order', solve({'z': 1}, {'a': 2}), [['z', 1], ['a', 2]])
check('empty expansions', solve({}, {}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Coerces non-string unpacked keyword names into strings and changes the required Python-language result.

Invoke f(**left, **right), returning ordered keyword pairs. Reject any duplicate keyword across mappings even when values agree; keys must be strings. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, right):
    if "" in left or "" in right: return "TypeError"
    def f(**kw): return [[k,v] for k,v in kw.items()]
    try: return f(**left, **right)
    except TypeError: return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate equal rejected', solve({'a': 1}, {'a': 1}), 'TypeError')
check('duplicate different rejected', solve({'a': 1}, {'a': 2}), 'TypeError')
check('integer key rejected', solve({1: 'v'}, {}), 'TypeError')
check('empty name legal', solve({'': 2}, {}), [['', 2]])
check('hyphenated name legal', solve({'a-b': 3}, {}), [['a-b', 3]])
check('left then right order', solve({'z': 1}, {'a': 2}), [['z', 1], ['a', 2]])
check('empty expansions', solve({}, {}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-keyword-unpacking-collisions

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, right):
    if any(not isinstance(k,str) or not k.isidentifier() for k in list(left)+list(right)): return "TypeError"
    def f(**kw): return [[k,v] for k,v in kw.items()]
    try: return f(**left, **right)
    except TypeError: return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate equal rejected', solve({'a': 1}, {'a': 1}), 'TypeError')
check('duplicate different rejected', solve({'a': 1}, {'a': 2}), 'TypeError')
check('integer key rejected', solve({1: 'v'}, {}), 'TypeError')
check('empty name legal', solve({'': 2}, {}), [['', 2]])
check('hyphenated name legal', solve({'a-b': 3}, {}), [['a-b', 3]])
check('left then right order', solve({'z': 1}, {'a': 2}), [['z', 1], ['a', 2]])
check('empty expansions', solve({}, {}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Rejects an empty string although variadic keywords accept it and changes the required Python-language result.

Invoke f(**left, **right), returning ordered keyword pairs. Reject any duplicate keyword across mappings even when values agree; keys must be strings. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, right):
    if any(not isinstance(k,str) or not k.isidentifier() for k in list(left)+list(right)): return "TypeError"
    def f(**kw): return [[k,v] for k,v in kw.items()]
    try: return f(**left, **right)
    except TypeError: return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate equal rejected', solve({'a': 1}, {'a': 1}), 'TypeError')
check('duplicate different rejected', solve({'a': 1}, {'a': 2}), 'TypeError')
check('integer key rejected', solve({1: 'v'}, {}), 'TypeError')
check('empty name legal', solve({'': 2}, {}), [['', 2]])
check('hyphenated name legal', solve({'a-b': 3}, {}), [['a-b', 3]])
check('left then right order', solve({'z': 1}, {'a': 2}), [['z', 1], ['a', 2]])
check('empty expansions', solve({}, {}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-keyword-unpacking-collisions

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(left, right):
    def f(**kw): return [[k,v] for k,v in kw.items()]
    try: return f(**{**left,**right})
    except TypeError: return "TypeError"
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('duplicate equal rejected', solve({'a': 1}, {'a': 1}), 'TypeError')
check('duplicate different rejected', solve({'a': 1}, {'a': 2}), 'TypeError')
check('integer key rejected', solve({1: 'v'}, {}), 'TypeError')
check('empty name legal', solve({'': 2}, {}), [['', 2]])
check('hyphenated name legal', solve({'a-b': 3}, {}), [['a-b', 3]])
check('left then right order', solve({'z': 1}, {'a': 2}), [['z', 1], ['a', 2]])
check('empty expansions', solve({}, {}), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Requires unpacked keyword names to be lexical identifiers and changes the required Python-language result.

Invoke f(**left, **right), returning ordered keyword pairs. Reject any duplicate keyword across mappings even when values agree; keys must be strings. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self, obj, owner):
            if obj is None: return 'class-descriptor'
            return obj.__dict__.get('field',obj.__dict__.get('_value',7))
        def __set__(self, obj, value):
            obj.__dict__['_value']=value
        def __delete__(self, obj):
            obj.__dict__.pop('_value',None)
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='class': out.append(C.field)
        elif op=='set': c.field=value
        elif op=='delete': del c.field
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('descriptor beats shadow', solve({'field': 99, '_value': 2}, [['get', None], ['class', None]]), [[2, 'class-descriptor'], {'field': 99, '_value': 2}])
check('set and delete dispatch', solve({}, [['set', 5], ['get', None], ['delete', None], ['get', None]]), [[5, 7], {}])
check('zero is stored', solve({'_value': 0}, [['get', None]]), [[0], {'_value': 0}])
check('shadow survives descriptor delete', solve({'field': 4, '_value': 8}, [['delete', None], ['get', None]]), [[7], {'field': 4}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-data-descriptor-storage

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self, obj, owner):
            if obj is None: return 'class-descriptor'
            return obj.__dict__.get('_value',7)
        def __set__(self, obj, value):
            obj.__dict__['field']=value
        def __delete__(self, obj):
            obj.__dict__.pop('_value',None)
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='class': out.append(C.field)
        elif op=='set': c.field=value
        elif op=='delete': del c.field
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('descriptor beats shadow', solve({'field': 99, '_value': 2}, [['get', None], ['class', None]]), [[2, 'class-descriptor'], {'field': 99, '_value': 2}])
check('set and delete dispatch', solve({}, [['set', 5], ['get', None], ['delete', None], ['get', None]]), [[5, 7], {}])
check('zero is stored', solve({'_value': 0}, [['get', None]]), [[0], {'_value': 0}])
check('shadow survives descriptor delete', solve({'field': 4, '_value': 8}, [['delete', None], ['get', None]]), [[7], {'field': 4}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Lets an instance shadow override a data descriptor and changes the required Python-language result.

A data descriptor field takes precedence over an instance field shadow. Its backing key is _value, missing reads return 7, writes update that key, deletion removes it, and class access returns class-descriptor. Return [read results,final instance dictionary]. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self, obj, owner):
            if obj is None: return 'class-descriptor'
            return obj.__dict__.get('_value',7)
        def __set__(self, obj, value):
            obj.__dict__['field']=value
        def __delete__(self, obj):
            obj.__dict__.pop('_value',None)
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='class': out.append(C.field)
        elif op=='set': c.field=value
        elif op=='delete': del c.field
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('descriptor beats shadow', solve({'field': 99, '_value': 2}, [['get', None], ['class', None]]), [[2, 'class-descriptor'], {'field': 99, '_value': 2}])
check('set and delete dispatch', solve({}, [['set', 5], ['get', None], ['delete', None], ['get', None]]), [[5, 7], {}])
check('zero is stored', solve({'_value': 0}, [['get', None]]), [[0], {'_value': 0}])
check('shadow survives descriptor delete', solve({'field': 4, '_value': 8}, [['delete', None], ['get', None]]), [[7], {'field': 4}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-data-descriptor-storage

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self, obj, owner):
            if obj is None: return 'class-descriptor'
            return obj.__dict__.get('_value',7)
        def __set__(self, obj, value):
            obj.__dict__['_value']=value
        def __delete__(self, obj):
            obj.__dict__.pop('field',None)
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='class': out.append(C.field)
        elif op=='set': c.field=value
        elif op=='delete': del c.field
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('descriptor beats shadow', solve({'field': 99, '_value': 2}, [['get', None], ['class', None]]), [[2, 'class-descriptor'], {'field': 99, '_value': 2}])
check('set and delete dispatch', solve({}, [['set', 5], ['get', None], ['delete', None], ['get', None]]), [[5, 7], {}])
check('zero is stored', solve({'_value': 0}, [['get', None]]), [[0], {'_value': 0}])
check('shadow survives descriptor delete', solve({'field': 4, '_value': 8}, [['delete', None], ['get', None]]), [[7], {'field': 4}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Writes to the shadow field instead of the descriptor backing key and changes the required Python-language result.

A data descriptor field takes precedence over an instance field shadow. Its backing key is _value, missing reads return 7, writes update that key, deletion removes it, and class access returns class-descriptor. Return [read results,final instance dictionary]. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self, obj, owner):
            if obj is None: return 'class-descriptor'
            return obj.__dict__.get('_value',7)
        def __set__(self, obj, value):
            obj.__dict__['_value']=value
        def __delete__(self, obj):
            obj.__dict__.pop('field',None)
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='class': out.append(C.field)
        elif op=='set': c.field=value
        elif op=='delete': del c.field
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('descriptor beats shadow', solve({'field': 99, '_value': 2}, [['get', None], ['class', None]]), [[2, 'class-descriptor'], {'field': 99, '_value': 2}])
check('set and delete dispatch', solve({}, [['set', 5], ['get', None], ['delete', None], ['get', None]]), [[5, 7], {}])
check('zero is stored', solve({'_value': 0}, [['get', None]]), [[0], {'_value': 0}])
check('shadow survives descriptor delete', solve({'field': 4, '_value': 8}, [['delete', None], ['get', None]]), [[7], {'field': 4}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-data-descriptor-storage

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self, obj, owner):
            if obj is None: return 7
            return obj.__dict__.get('_value',7)
        def __set__(self, obj, value):
            obj.__dict__['_value']=value
        def __delete__(self, obj):
            obj.__dict__.pop('_value',None)
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='class': out.append(C.field)
        elif op=='set': c.field=value
        elif op=='delete': del c.field
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('descriptor beats shadow', solve({'field': 99, '_value': 2}, [['get', None], ['class', None]]), [[2, 'class-descriptor'], {'field': 99, '_value': 2}])
check('set and delete dispatch', solve({}, [['set', 5], ['get', None], ['delete', None], ['get', None]]), [[5, 7], {}])
check('zero is stored', solve({'_value': 0}, [['get', None]]), [[0], {'_value': 0}])
check('shadow survives descriptor delete', solve({'field': 4, '_value': 8}, [['delete', None], ['get', None]]), [[7], {'field': 4}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Deletes the shadow key instead of the descriptor backing key and changes the required Python-language result.

A data descriptor field takes precedence over an instance field shadow. Its backing key is _value, missing reads return 7, writes update that key, deletion removes it, and class access returns class-descriptor. Return [read results,final instance dictionary]. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self, obj, owner):
            if obj is None: return 7
            return obj.__dict__.get('_value',7)
        def __set__(self, obj, value):
            obj.__dict__['_value']=value
        def __delete__(self, obj):
            obj.__dict__.pop('_value',None)
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='class': out.append(C.field)
        elif op=='set': c.field=value
        elif op=='delete': del c.field
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('descriptor beats shadow', solve({'field': 99, '_value': 2}, [['get', None], ['class', None]]), [[2, 'class-descriptor'], {'field': 99, '_value': 2}])
check('set and delete dispatch', solve({}, [['set', 5], ['get', None], ['delete', None], ['get', None]]), [[5, 7], {}])
check('zero is stored', solve({'_value': 0}, [['get', None]]), [[0], {'_value': 0}])
check('shadow survives descriptor delete', solve({'field': 4, '_value': 8}, [['delete', None], ['get', None]]), [[7], {'field': 4}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-data-descriptor-storage

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self, obj, owner):
            if obj is None: return 'class-descriptor'
            return obj.__dict__.get('_value') or 7
        def __set__(self, obj, value):
            obj.__dict__['_value']=value
        def __delete__(self, obj):
            obj.__dict__.pop('_value',None)
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='class': out.append(C.field)
        elif op=='set': c.field=value
        elif op=='delete': del c.field
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('descriptor beats shadow', solve({'field': 99, '_value': 2}, [['get', None], ['class', None]]), [[2, 'class-descriptor'], {'field': 99, '_value': 2}])
check('set and delete dispatch', solve({}, [['set', 5], ['get', None], ['delete', None], ['get', None]]), [[5, 7], {}])
check('zero is stored', solve({'_value': 0}, [['get', None]]), [[0], {'_value': 0}])
check('shadow survives descriptor delete', solve({'field': 4, '_value': 8}, [['delete', None], ['get', None]]), [[7], {'field': 4}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Returns an instance default for descriptor access on the class and changes the required Python-language result.

A data descriptor field takes precedence over an instance field shadow. Its backing key is _value, missing reads return 7, writes update that key, deletion removes it, and class access returns class-descriptor. Return [read results,final instance dictionary]. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self, obj, owner):
            if obj is None: return 'class-descriptor'
            return obj.__dict__.get('_value') or 7
        def __set__(self, obj, value):
            obj.__dict__['_value']=value
        def __delete__(self, obj):
            obj.__dict__.pop('_value',None)
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='class': out.append(C.field)
        elif op=='set': c.field=value
        elif op=='delete': del c.field
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('descriptor beats shadow', solve({'field': 99, '_value': 2}, [['get', None], ['class', None]]), [[2, 'class-descriptor'], {'field': 99, '_value': 2}])
check('set and delete dispatch', solve({}, [['set', 5], ['get', None], ['delete', None], ['get', None]]), [[5, 7], {}])
check('zero is stored', solve({'_value': 0}, [['get', None]]), [[0], {'_value': 0}])
check('shadow survives descriptor delete', solve({'field': 4, '_value': 8}, [['delete', None], ['get', None]]), [[7], {'field': 4}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-data-descriptor-storage

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self, obj, owner):
            if obj is None: return 'class-descriptor'
            return obj.__dict__.get('field',obj.__dict__.get('_value',7))
        def __set__(self, obj, value):
            obj.__dict__['_value']=value
        def __delete__(self, obj):
            obj.__dict__.pop('_value',None)
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='class': out.append(C.field)
        elif op=='set': c.field=value
        elif op=='delete': del c.field
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('descriptor beats shadow', solve({'field': 99, '_value': 2}, [['get', None], ['class', None]]), [[2, 'class-descriptor'], {'field': 99, '_value': 2}])
check('set and delete dispatch', solve({}, [['set', 5], ['get', None], ['delete', None], ['get', None]]), [[5, 7], {}])
check('zero is stored', solve({'_value': 0}, [['get', None]]), [[0], {'_value': 0}])
check('shadow survives descriptor delete', solve({'field': 4, '_value': 8}, [['delete', None], ['get', None]]), [[7], {'field': 4}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Replaces a known zero backing value with the missing-value default and changes the required Python-language result.

A data descriptor field takes precedence over an instance field shadow. Its backing key is _value, missing reads return 7, writes update that key, deletion removes it, and class access returns class-descriptor. Return [read results,final instance dictionary]. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self,obj,owner):
            return 'class' if obj is None else 'computed'
        def __set__(self,obj,value): obj.__dict__['field']=value
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='set': c.field=value
        elif op=='delete': c.__dict__.pop('field',None)
        elif op=='class': out.append(C.field)
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial shadow', solve({'field': 'local'}, [['get', None]]), [['local'], {'field': 'local'}])
check('assign then unshadow', solve({}, [['get', None], ['set', 'local'], ['get', None], ['delete', None], ['get', None], ['class', None]]), [['computed', 'local', 'computed', 'class'], {}])
check('false shadow', solve({'field': 0}, [['get', None]]), [[0], {'field': 0}])
check('None shadow', solve({'field': None}, [['get', None], ['class', None]]), [[None, 'class'], {'field': None}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-nondata-descriptor-shadow

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self,obj,owner):
            return 'class' if obj is None else 'computed'
        pass
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(C.__dict__['field'].__get__(c,C))
        elif op=='set': c.field=value
        elif op=='delete': c.__dict__.pop('field',None)
        elif op=='class': out.append(C.field)
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial shadow', solve({'field': 'local'}, [['get', None]]), [['local'], {'field': 'local'}])
check('assign then unshadow', solve({}, [['get', None], ['set', 'local'], ['get', None], ['delete', None], ['get', None], ['class', None]]), [['computed', 'local', 'computed', 'class'], {}])
check('false shadow', solve({'field': 0}, [['get', None]]), [[0], {'field': 0}])
check('None shadow', solve({'field': None}, [['get', None], ['class', None]]), [[None, 'class'], {'field': None}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Adds a setter and unexpectedly converts the non-data descriptor into a data descriptor and changes the required Python-language result.

An instance dictionary field shadows a non-data descriptor. Assignment creates a shadow, deleting the shadow restores computed descriptor reads, and class access stays class. Return reads and final instance dictionary. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self,obj,owner):
            return 'class' if obj is None else 'computed'
        pass
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(C.__dict__['field'].__get__(c,C))
        elif op=='set': c.field=value
        elif op=='delete': c.__dict__.pop('field',None)
        elif op=='class': out.append(C.field)
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial shadow', solve({'field': 'local'}, [['get', None]]), [['local'], {'field': 'local'}])
check('assign then unshadow', solve({}, [['get', None], ['set', 'local'], ['get', None], ['delete', None], ['get', None], ['class', None]]), [['computed', 'local', 'computed', 'class'], {}])
check('false shadow', solve({'field': 0}, [['get', None]]), [[0], {'field': 0}])
check('None shadow', solve({'field': None}, [['get', None], ['class', None]]), [[None, 'class'], {'field': None}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-nondata-descriptor-shadow

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self,obj,owner):
            return 'class' if obj is None else 'computed'
        pass
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='set': C.field=value
        elif op=='delete': c.__dict__.pop('field',None)
        elif op=='class': out.append(C.field)
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial shadow', solve({'field': 'local'}, [['get', None]]), [['local'], {'field': 'local'}])
check('assign then unshadow', solve({}, [['get', None], ['set', 'local'], ['get', None], ['delete', None], ['get', None], ['class', None]]), [['computed', 'local', 'computed', 'class'], {}])
check('false shadow', solve({'field': 0}, [['get', None]]), [[0], {'field': 0}])
check('None shadow', solve({'field': None}, [['get', None], ['class', None]]), [[None, 'class'], {'field': None}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Manually calls the descriptor and ignores legal instance shadows and changes the required Python-language result.

An instance dictionary field shadows a non-data descriptor. Assignment creates a shadow, deleting the shadow restores computed descriptor reads, and class access stays class. Return reads and final instance dictionary. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self,obj,owner):
            return 'class' if obj is None else 'computed'
        pass
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='set': C.field=value
        elif op=='delete': c.__dict__.pop('field',None)
        elif op=='class': out.append(C.field)
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial shadow', solve({'field': 'local'}, [['get', None]]), [['local'], {'field': 'local'}])
check('assign then unshadow', solve({}, [['get', None], ['set', 'local'], ['get', None], ['delete', None], ['get', None], ['class', None]]), [['computed', 'local', 'computed', 'class'], {}])
check('false shadow', solve({'field': 0}, [['get', None]]), [[0], {'field': 0}])
check('None shadow', solve({'field': None}, [['get', None], ['class', None]]), [[None, 'class'], {'field': None}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-nondata-descriptor-shadow

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self,obj,owner):
            return 'class' if obj is None else 'computed'
        pass
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='set': c.field=value
        elif op=='delete': c.__dict__['field']=None
        elif op=='class': out.append(C.field)
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial shadow', solve({'field': 'local'}, [['get', None]]), [['local'], {'field': 'local'}])
check('assign then unshadow', solve({}, [['get', None], ['set', 'local'], ['get', None], ['delete', None], ['get', None], ['class', None]]), [['computed', 'local', 'computed', 'class'], {}])
check('false shadow', solve({'field': 0}, [['get', None]]), [[0], {'field': 0}])
check('None shadow', solve({'field': None}, [['get', None], ['class', None]]), [[None, 'class'], {'field': None}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Assigns the instance value to the class and replaces its descriptor and changes the required Python-language result.

An instance dictionary field shadows a non-data descriptor. Assignment creates a shadow, deleting the shadow restores computed descriptor reads, and class access stays class. Return reads and final instance dictionary. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self,obj,owner):
            return 'class' if obj is None else 'computed'
        pass
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='set': c.field=value
        elif op=='delete': c.__dict__['field']=None
        elif op=='class': out.append(C.field)
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial shadow', solve({'field': 'local'}, [['get', None]]), [['local'], {'field': 'local'}])
check('assign then unshadow', solve({}, [['get', None], ['set', 'local'], ['get', None], ['delete', None], ['get', None], ['class', None]]), [['computed', 'local', 'computed', 'class'], {}])
check('false shadow', solve({'field': 0}, [['get', None]]), [[0], {'field': 0}])
check('None shadow', solve({'field': None}, [['get', None], ['class', None]]), [[None, 'class'], {'field': None}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-nondata-descriptor-shadow

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self,obj,owner):
            return 'class' if obj is None else 'computed'
        pass
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.__dict__.get('field') or C.__dict__['field'].__get__(c,C))
        elif op=='set': c.field=value
        elif op=='delete': c.__dict__.pop('field',None)
        elif op=='class': out.append(C.field)
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial shadow', solve({'field': 'local'}, [['get', None]]), [['local'], {'field': 'local'}])
check('assign then unshadow', solve({}, [['get', None], ['set', 'local'], ['get', None], ['delete', None], ['get', None], ['class', None]]), [['computed', 'local', 'computed', 'class'], {}])
check('false shadow', solve({'field': 0}, [['get', None]]), [[0], {'field': 0}])
check('None shadow', solve({'field': None}, [['get', None], ['class', None]]), [[None, 'class'], {'field': None}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Writes None instead of removing the shadow and changes the required Python-language result.

An instance dictionary field shadows a non-data descriptor. Assignment creates a shadow, deleting the shadow restores computed descriptor reads, and class access stays class. Return reads and final instance dictionary. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self,obj,owner):
            return 'class' if obj is None else 'computed'
        pass
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.__dict__.get('field') or C.__dict__['field'].__get__(c,C))
        elif op=='set': c.field=value
        elif op=='delete': c.__dict__.pop('field',None)
        elif op=='class': out.append(C.field)
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial shadow', solve({'field': 'local'}, [['get', None]]), [['local'], {'field': 'local'}])
check('assign then unshadow', solve({}, [['get', None], ['set', 'local'], ['get', None], ['delete', None], ['get', None], ['class', None]]), [['computed', 'local', 'computed', 'class'], {}])
check('false shadow', solve({'field': 0}, [['get', None]]), [[0], {'field': 0}])
check('None shadow', solve({'field': None}, [['get', None], ['class', None]]), [[None, 'class'], {'field': None}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-nondata-descriptor-shadow

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(initial, commands):
    class D:
        def __get__(self,obj,owner):
            return 'class' if obj is None else 'computed'
        def __set__(self,obj,value): obj.__dict__['field']=value
    class C:
        field=D()
    c=C(); c.__dict__.update(initial); out=[]
    for op,value in commands:
        if op=='get': out.append(c.field)
        elif op=='set': c.field=value
        elif op=='delete': c.__dict__.pop('field',None)
        elif op=='class': out.append(C.field)
    return [out,dict(c.__dict__)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('initial shadow', solve({'field': 'local'}, [['get', None]]), [['local'], {'field': 'local'}])
check('assign then unshadow', solve({}, [['get', None], ['set', 'local'], ['get', None], ['delete', None], ['get', None], ['class', None]]), [['computed', 'local', 'computed', 'class'], {}])
check('false shadow', solve({'field': 0}, [['get', None]]), [[0], {'field': 0}])
check('None shadow', solve({'field': None}, [['get', None], ['class', None]]), [[None, 'class'], {'field': None}])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Treats false instance shadows as absent and changes the required Python-language result.

An instance dictionary field shadows a non-data descriptor. Assignment creates a shadow, deleting the shadow restores computed descriptor reads, and class access stays class. Return reads and final instance dictionary. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(values, names):
    class C:
        present='class-value'
        @property
        def trouble(self):
            raise AttributeError('getter failed')
        def __getattr__(self,name):
            return 'missing:'+name
    c=C(); c.__dict__.update(values)
    out=[]
    for name in names:
        try: out.append(c.__getattr__(name))
        except Exception as exc: out.append(type(exc).__name__)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('existing class and missing', solve({}, ['present', 'other']), ['class-value', 'missing:other'])
check('instance override', solve({'present': 0}, ['present']), [0])
check('property AttributeError falls back', solve({}, ['trouble']), ['missing:trouble'])
check('stored None is present', solve({'other': None}, ['other', 'absent']), [None, 'missing:absent'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-getattr-fallback

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(values, names):
    class C:
        present='class-value'
        @property
        def trouble(self):
            raise AttributeError('getter failed')
        def __getattr__(self,name):
            return 'missing:'+name
    c=C(); c.__dict__.update(values)
    out=[]
    for name in names:
        try: out.append(object.__getattribute__(c,name))
        except Exception as exc: out.append(type(exc).__name__)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('existing class and missing', solve({}, ['present', 'other']), ['class-value', 'missing:other'])
check('instance override', solve({'present': 0}, ['present']), [0])
check('property AttributeError falls back', solve({}, ['trouble']), ['missing:trouble'])
check('stored None is present', solve({'other': None}, ['other', 'absent']), [None, 'missing:absent'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Invokes __getattr__ before trying actual attribute lookup and changes the required Python-language result.

Read attributes normally: instance attributes override plain class values; __getattr__ supplies missing:name only after AttributeError, including AttributeError from a property getter. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(values, names):
    class C:
        present='class-value'
        @property
        def trouble(self):
            raise AttributeError('getter failed')
        def __getattr__(self,name):
            return 'missing:'+name
    c=C(); c.__dict__.update(values)
    out=[]
    for name in names:
        try: out.append(object.__getattribute__(c,name))
        except Exception as exc: out.append(type(exc).__name__)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('existing class and missing', solve({}, ['present', 'other']), ['class-value', 'missing:other'])
check('instance override', solve({'present': 0}, ['present']), [0])
check('property AttributeError falls back', solve({}, ['trouble']), ['missing:trouble'])
check('stored None is present', solve({'other': None}, ['other', 'absent']), [None, 'missing:absent'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-getattr-fallback

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(values, names):
    class C:
        present='class-value'
        @property
        def trouble(self):
            raise AttributeError('getter failed')
        def __getattr__(self,name):
            return 'missing:'+name
    c=C(); c.__dict__.update(values)
    out=[]
    for name in names:
        try: out.append(c.__dict__.get(name,'missing:'+name))
        except Exception as exc: out.append(type(exc).__name__)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('existing class and missing', solve({}, ['present', 'other']), ['class-value', 'missing:other'])
check('instance override', solve({'present': 0}, ['present']), [0])
check('property AttributeError falls back', solve({}, ['trouble']), ['missing:trouble'])
check('stored None is present', solve({'other': None}, ['other', 'absent']), [None, 'missing:absent'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Calls object.__getattribute__ directly and bypasses __getattr__ fallback and changes the required Python-language result.

Read attributes normally: instance attributes override plain class values; __getattr__ supplies missing:name only after AttributeError, including AttributeError from a property getter. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(values, names):
    class C:
        present='class-value'
        @property
        def trouble(self):
            raise AttributeError('getter failed')
        def __getattr__(self,name):
            return 'missing:'+name
    c=C(); c.__dict__.update(values)
    out=[]
    for name in names:
        try: out.append(c.__dict__.get(name,'missing:'+name))
        except Exception as exc: out.append(type(exc).__name__)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('existing class and missing', solve({}, ['present', 'other']), ['class-value', 'missing:other'])
check('instance override', solve({'present': 0}, ['present']), [0])
check('property AttributeError falls back', solve({}, ['trouble']), ['missing:trouble'])
check('stored None is present', solve({'other': None}, ['other', 'absent']), [None, 'missing:absent'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-getattr-fallback

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(values, names):
    class C:
        present='class-value'
        @property
        def trouble(self):
            raise ValueError('getter failed')
        def __getattr__(self,name):
            return 'missing:'+name
    c=C(); c.__dict__.update(values)
    out=[]
    for name in names:
        try: out.append(getattr(c,name))
        except Exception as exc: out.append(type(exc).__name__)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('existing class and missing', solve({}, ['present', 'other']), ['class-value', 'missing:other'])
check('instance override', solve({'present': 0}, ['present']), [0])
check('property AttributeError falls back', solve({}, ['trouble']), ['missing:trouble'])
check('stored None is present', solve({'other': None}, ['other', 'absent']), [None, 'missing:absent'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Looks only in the instance dictionary and skips class descriptors and changes the required Python-language result.

Read attributes normally: instance attributes override plain class values; __getattr__ supplies missing:name only after AttributeError, including AttributeError from a property getter. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(values, names):
    class C:
        present='class-value'
        @property
        def trouble(self):
            raise ValueError('getter failed')
        def __getattr__(self,name):
            return 'missing:'+name
    c=C(); c.__dict__.update(values)
    out=[]
    for name in names:
        try: out.append(getattr(c,name))
        except Exception as exc: out.append(type(exc).__name__)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('existing class and missing', solve({}, ['present', 'other']), ['class-value', 'missing:other'])
check('instance override', solve({'present': 0}, ['present']), [0])
check('property AttributeError falls back', solve({}, ['trouble']), ['missing:trouble'])
check('stored None is present', solve({'other': None}, ['other', 'absent']), [None, 'missing:absent'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-getattr-fallback

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(values, names):
    class C:
        present='class-value'
        @property
        def trouble(self):
            raise AttributeError('getter failed')
        def __getattr__(self,name):
            return 'missing'
    c=C(); c.__dict__.update(values)
    out=[]
    for name in names:
        try: out.append(getattr(c,name))
        except Exception as exc: out.append(type(exc).__name__)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('existing class and missing', solve({}, ['present', 'other']), ['class-value', 'missing:other'])
check('instance override', solve({'present': 0}, ['present']), [0])
check('property AttributeError falls back', solve({}, ['trouble']), ['missing:trouble'])
check('stored None is present', solve({'other': None}, ['other', 'absent']), [None, 'missing:absent'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Changes an AttributeError getter into ValueError so fallback no longer runs and changes the required Python-language result.

Read attributes normally: instance attributes override plain class values; __getattr__ supplies missing:name only after AttributeError, including AttributeError from a property getter. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(values, names):
    class C:
        present='class-value'
        @property
        def trouble(self):
            raise AttributeError('getter failed')
        def __getattr__(self,name):
            return 'missing'
    c=C(); c.__dict__.update(values)
    out=[]
    for name in names:
        try: out.append(getattr(c,name))
        except Exception as exc: out.append(type(exc).__name__)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('existing class and missing', solve({}, ['present', 'other']), ['class-value', 'missing:other'])
check('instance override', solve({'present': 0}, ['present']), [0])
check('property AttributeError falls back', solve({}, ['trouble']), ['missing:trouble'])
check('stored None is present', solve({'other': None}, ['other', 'absent']), [None, 'missing:absent'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-getattr-fallback

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(values, names):
    class C:
        present='class-value'
        @property
        def trouble(self):
            raise AttributeError('getter failed')
        def __getattr__(self,name):
            return 'missing:'+name
    c=C(); c.__dict__.update(values)
    out=[]
    for name in names:
        try: out.append(c.__getattr__(name))
        except Exception as exc: out.append(type(exc).__name__)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('existing class and missing', solve({}, ['present', 'other']), ['class-value', 'missing:other'])
check('instance override', solve({'present': 0}, ['present']), [0])
check('property AttributeError falls back', solve({}, ['trouble']), ['missing:trouble'])
check('stored None is present', solve({'other': None}, ['other', 'absent']), [None, 'missing:absent'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Returns a constant fallback instead of the requested missing attribute name and changes the required Python-language result.

Read attributes normally: instance attributes override plain class values; __getattr__ supplies missing:name only after AttributeError, including AttributeError from a property getter. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    class D:
        def __get__(self,obj,owner): return [obj is None,'Base']
    class Base:
        field=D()
    class Child(Base): pass
    obj=Child()
    if mode=='base-class': return Base.field
    if mode=='child-class': return Child.field
    if mode=='instance': return obj.field
    if mode=='super': return super(Child,obj).field
    return Base().field
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('defining class access', solve('base-class'), [True, 'Base'])
check('inherited class access', solve('child-class'), [True, 'Child'])
check('subclass instance', solve('instance'), [False, 'Child'])
check('super keeps dynamic owner', solve('super'), [False, 'Child'])
check('base instance', solve('base-instance'), [False, 'Base'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-descriptor-owner-argument

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    class D:
        def __get__(self,obj,owner): return [obj is None,type(obj).__name__]
    class Base:
        field=D()
    class Child(Base): pass
    obj=Child()
    if mode=='base-class': return Base.field
    if mode=='child-class': return Child.field
    if mode=='instance': return obj.field
    if mode=='super': return super(Child,obj).field
    return Base().field
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('defining class access', solve('base-class'), [True, 'Base'])
check('inherited class access', solve('child-class'), [True, 'Child'])
check('subclass instance', solve('instance'), [False, 'Child'])
check('super keeps dynamic owner', solve('super'), [False, 'Child'])
check('base instance', solve('base-instance'), [False, 'Base'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Hardcodes the descriptor-defining class as owner and changes the required Python-language result.

Return [instance_is_None,owner_class_name] observed by an inherited descriptor for class, instance and super access. Owner is the accessing subclass, not necessarily the descriptor-defining class. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    class D:
        def __get__(self,obj,owner): return [obj is None,type(obj).__name__]
    class Base:
        field=D()
    class Child(Base): pass
    obj=Child()
    if mode=='base-class': return Base.field
    if mode=='child-class': return Child.field
    if mode=='instance': return obj.field
    if mode=='super': return super(Child,obj).field
    return Base().field
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('defining class access', solve('base-class'), [True, 'Base'])
check('inherited class access', solve('child-class'), [True, 'Child'])
check('subclass instance', solve('instance'), [False, 'Child'])
check('super keeps dynamic owner', solve('super'), [False, 'Child'])
check('base instance', solve('base-instance'), [False, 'Base'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-descriptor-owner-argument

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    class D:
        def __get__(self,obj,owner): return [False,owner.__name__]
    class Base:
        field=D()
    class Child(Base): pass
    obj=Child()
    if mode=='base-class': return Base.field
    if mode=='child-class': return Child.field
    if mode=='instance': return obj.field
    if mode=='super': return super(Child,obj).field
    return Base().field
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('defining class access', solve('base-class'), [True, 'Base'])
check('inherited class access', solve('child-class'), [True, 'Child'])
check('subclass instance', solve('instance'), [False, 'Child'])
check('super keeps dynamic owner', solve('super'), [False, 'Child'])
check('base instance', solve('base-instance'), [False, 'Base'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Uses type(None) as owner for class-level access and changes the required Python-language result.

Return [instance_is_None,owner_class_name] observed by an inherited descriptor for class, instance and super access. Owner is the accessing subclass, not necessarily the descriptor-defining class. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    class D:
        def __get__(self,obj,owner): return [False,owner.__name__]
    class Base:
        field=D()
    class Child(Base): pass
    obj=Child()
    if mode=='base-class': return Base.field
    if mode=='child-class': return Child.field
    if mode=='instance': return obj.field
    if mode=='super': return super(Child,obj).field
    return Base().field
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('defining class access', solve('base-class'), [True, 'Base'])
check('inherited class access', solve('child-class'), [True, 'Child'])
check('subclass instance', solve('instance'), [False, 'Child'])
check('super keeps dynamic owner', solve('super'), [False, 'Child'])
check('base instance', solve('base-instance'), [False, 'Base'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-descriptor-owner-argument

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    class D:
        def __get__(self,obj,owner): return [True,owner.__name__]
    class Base:
        field=D()
    class Child(Base): pass
    obj=Child()
    if mode=='base-class': return Base.field
    if mode=='child-class': return Child.field
    if mode=='instance': return obj.field
    if mode=='super': return super(Child,obj).field
    return Base().field
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('defining class access', solve('base-class'), [True, 'Base'])
check('inherited class access', solve('child-class'), [True, 'Child'])
check('subclass instance', solve('instance'), [False, 'Child'])
check('super keeps dynamic owner', solve('super'), [False, 'Child'])
check('base instance', solve('base-instance'), [False, 'Base'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Reports a non-None instance during class descriptor access and changes the required Python-language result.

Return [instance_is_None,owner_class_name] observed by an inherited descriptor for class, instance and super access. Owner is the accessing subclass, not necessarily the descriptor-defining class. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    class D:
        def __get__(self,obj,owner): return [True,owner.__name__]
    class Base:
        field=D()
    class Child(Base): pass
    obj=Child()
    if mode=='base-class': return Base.field
    if mode=='child-class': return Child.field
    if mode=='instance': return obj.field
    if mode=='super': return super(Child,obj).field
    return Base().field
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('defining class access', solve('base-class'), [True, 'Base'])
check('inherited class access', solve('child-class'), [True, 'Child'])
check('subclass instance', solve('instance'), [False, 'Child'])
check('super keeps dynamic owner', solve('super'), [False, 'Child'])
check('base instance', solve('base-instance'), [False, 'Base'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-descriptor-owner-argument

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    class D:
        def __get__(self,obj,owner): return [obj is None,type(owner).__name__]
    class Base:
        field=D()
    class Child(Base): pass
    obj=Child()
    if mode=='base-class': return Base.field
    if mode=='child-class': return Child.field
    if mode=='instance': return obj.field
    if mode=='super': return super(Child,obj).field
    return Base().field
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('defining class access', solve('base-class'), [True, 'Base'])
check('inherited class access', solve('child-class'), [True, 'Child'])
check('subclass instance', solve('instance'), [False, 'Child'])
check('super keeps dynamic owner', solve('super'), [False, 'Child'])
check('base instance', solve('base-instance'), [False, 'Base'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Drops instance binding for ordinary instance access and changes the required Python-language result.

Return [instance_is_None,owner_class_name] observed by an inherited descriptor for class, instance and super access. Owner is the accessing subclass, not necessarily the descriptor-defining class. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    class D:
        def __get__(self,obj,owner): return [obj is None,type(owner).__name__]
    class Base:
        field=D()
    class Child(Base): pass
    obj=Child()
    if mode=='base-class': return Base.field
    if mode=='child-class': return Child.field
    if mode=='instance': return obj.field
    if mode=='super': return super(Child,obj).field
    return Base().field
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('defining class access', solve('base-class'), [True, 'Base'])
check('inherited class access', solve('child-class'), [True, 'Child'])
check('subclass instance', solve('instance'), [False, 'Child'])
check('super keeps dynamic owner', solve('super'), [False, 'Child'])
check('base instance', solve('base-instance'), [False, 'Base'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-descriptor-owner-argument

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(mode):
    class D:
        def __get__(self,obj,owner): return [obj is None,'Base']
    class Base:
        field=D()
    class Child(Base): pass
    obj=Child()
    if mode=='base-class': return Base.field
    if mode=='child-class': return Child.field
    if mode=='instance': return obj.field
    if mode=='super': return super(Child,obj).field
    return Base().field
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('defining class access', solve('base-class'), [True, 'Base'])
check('inherited class access', solve('child-class'), [True, 'Child'])
check('subclass instance', solve('instance'), [False, 'Child'])
check('super keeps dynamic owner', solve('super'), [False, 'Child'])
check('base instance', solve('base-instance'), [False, 'Base'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Passes the metaclass rather than the owner class identity and changes the required Python-language result.

Return [instance_is_None,owner_class_name] observed by an inherited descriptor for class, instance and super access. Owner is the accessing subclass, not necessarily the descriptor-defining class. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declared, shadow, operation):
    class C:
        def __len__(self): return declared
    obj=C()
    obj.__len__=lambda: shadow
    if operation=='builtin': return obj.__len__()
    if operation=='direct': return obj.__len__()
    if operation=='class': return C.__len__(obj)
    return bool(obj)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('implicit ignores shadow', solve(2, 9, 'builtin'), 2)
check('direct sees shadow', solve(2, 9, 'direct'), 9)
check('class explicit sees declared', solve(2, 9, 'class'), 2)
check('zero native length false', solve(0, 9, 'truth'), False)
check('positive native length true', solve(2, 0, 'truth'), True)
check('zero shadow direct', solve(2, 0, 'direct'), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-special-method-type-lookup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declared, shadow, operation):
    class C:
        def __len__(self): return declared
    obj=C()
    obj.__len__=lambda: shadow
    if operation=='builtin': return len(obj)
    if operation=='direct': return C.__len__(obj)
    if operation=='class': return C.__len__(obj)
    return bool(obj)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('implicit ignores shadow', solve(2, 9, 'builtin'), 2)
check('direct sees shadow', solve(2, 9, 'direct'), 9)
check('class explicit sees declared', solve(2, 9, 'class'), 2)
check('zero native length false', solve(0, 9, 'truth'), False)
check('positive native length true', solve(2, 0, 'truth'), True)
check('zero shadow direct', solve(2, 0, 'direct'), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Uses direct instance lookup for the implicit len protocol and changes the required Python-language result.

A per-instance __len__ shadows direct attribute calls but does not change implicit len or truth testing. Return the selected native result; declared is an integer from 0 through sys.maxsize, and shadow is any nonnegative integer. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declared, shadow, operation):
    class C:
        def __len__(self): return declared
    obj=C()
    obj.__len__=lambda: shadow
    if operation=='builtin': return len(obj)
    if operation=='direct': return C.__len__(obj)
    if operation=='class': return C.__len__(obj)
    return bool(obj)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('implicit ignores shadow', solve(2, 9, 'builtin'), 2)
check('direct sees shadow', solve(2, 9, 'direct'), 9)
check('class explicit sees declared', solve(2, 9, 'class'), 2)
check('zero native length false', solve(0, 9, 'truth'), False)
check('positive native length true', solve(2, 0, 'truth'), True)
check('zero shadow direct', solve(2, 0, 'direct'), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xl-special-method-type-lookup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(declared, shadow, operation):
    class C:
        def __len__(self): return declared
    obj=C()
    obj.__len__=lambda: shadow
    if operation=='builtin': return len(obj)
    if operation=='direct': return obj.__len__()
    if operation=='class': return obj.__len__()
    return bool(obj)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('implicit ignores shadow', solve(2, 9, 'builtin'), 2)
check('direct sees shadow', solve(2, 9, 'direct'), 9)
check('class explicit sees declared', solve(2, 9, 'class'), 2)
check('zero native length false', solve(0, 9, 'truth'), False)
check('positive native length true', solve(2, 0, 'truth'), True)
check('zero shadow direct', solve(2, 0, 'direct'), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Bypasses the instance shadow for an explicit __len__ call and changes the required Python-language result.

A per-instance __len__ shadows direct attribute calls but does not change implicit len or truth testing. Return the selected native result; declared is an integer from 0 through sys.maxsize, and shadow is any nonnegative integer. Inputs use the finite Python types shown in the fixtures. An error string denotes the specified caught exception, not an unhandled process failure.

split: open-access