broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, success):
    return [entry[0],entry[1],False]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 3, True], False)), ['a', 3, True])
check('fixture 2', solve(*(['a', 3, True], True)), ['a', 3, False])
check('fixture 3', solve(*(['a', 3, False], False)), ['a', 3, False])
check('fixture 4', solve(*(['a', None, False], True)), ['a', None, False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-failed-writeback-retains-dirty

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, success):
    return [entry[0],entry[1],not success]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 3, True], False)), ['a', 3, True])
check('fixture 2', solve(*(['a', 3, True], True)), ['a', 3, False])
check('fixture 3', solve(*(['a', 3, False], False)), ['a', 3, False])
check('fixture 4', solve(*(['a', None, False], True)), ['a', None, False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A failed flush marks its cache entry clean.

Entry is [key,value,dirty]. Successful flush clears dirty; failed flush preserves prior dirty state and never changes key/value.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, flushed_version, success):
    return [entry[0],entry[1],entry[2],False if success else entry[3]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'new', 2, True], 1, True)), ['a', 'new', 2, True])
check('fixture 2', solve(*(['a', 'new', 2, True], 2, False)), ['a', 'new', 2, True])
check('fixture 3', solve(*(['a', 'new', 2, True], 2, True)), ['a', 'new', 2, False])
check('fixture 4', solve(*(['a', 'new', 2, False], 1, False)), ['a', 'new', 2, False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xc-writeback-version-match

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, flushed_version, success):
    return [entry[0],entry[1],entry[2],entry[3] and entry[2]!=flushed_version]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'new', 2, True], 1, True)), ['a', 'new', 2, True])
check('fixture 2', solve(*(['a', 'new', 2, True], 2, False)), ['a', 'new', 2, True])
check('fixture 3', solve(*(['a', 'new', 2, True], 2, True)), ['a', 'new', 2, False])
check('fixture 4', solve(*(['a', 'new', 2, False], 1, False)), ['a', 'new', 2, False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Completion of an old flush clears a newer dirty update.

Entry is [key,value,version,dirty]. Clear dirty only when flush succeeded for this exact current version; preserve all other fields.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, fetched):
    return [fetched,False]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['local', True], 'remote')), ['local', True])
check('fixture 2', solve(*(['old', False], 'remote')), ['remote', False])
check('fixture 3', solve(*([None, True], 'remote')), [None, True])
check('fixture 4', solve(*([None, False], 0)), [0, False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-dirty-replacement-stays-dirty

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, fetched):
    return [entry[0],entry[1]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['local', True], 'remote')), ['local', True])
check('fixture 2', solve(*(['old', False], 'remote')), ['remote', False])
check('fixture 3', solve(*([None, True], 'remote')), [None, True])
check('fixture 4', solve(*([None, False], 0)), [0, False])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Replacing a dirty value with a read result silently loses pending writes.

Entry is [value,dirty]. A fetched value replaces only clean state; a dirty local write remains authoritative until flushed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, tags):
    return [x for x in entries if not set(tags)<=set(x[1])]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['a', ['red']], ['b', ['blue']], ['c', []]], ['red', 'green'])), [['b', ['blue']], ['c', []]])
check('fixture 2', solve(*([['a', ['red', 'blue']]], ['red'])), [])
check('fixture 3', solve(*([['a', ['red']]], [])), [['a', ['red']]])
check('fixture 4', solve(*([], ['red'])), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-tag-invalidation-any

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, tags):
    return [x for x in entries if set(x[1])!=set(tags)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['a', ['red']], ['b', ['blue']], ['c', []]], ['red', 'green'])), [['b', ['blue']], ['c', []]])
check('fixture 2', solve(*([['a', ['red', 'blue']]], ['red'])), [])
check('fixture 3', solve(*([['a', ['red']]], [])), [['a', ['red']]])
check('fixture 4', solve(*([], ['red'])), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Tag invalidation keeps entries sharing only one requested tag.

Entries are [key,tags]. Remove every entry having any tag in the requested set, preserving input order; empty requested tags remove nothing.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(index, key, new_tags):
    out={tag:list(keys) for tag,keys in index.items()}
    for tag in new_tags: out.setdefault(tag,[]).append(key)
    return {tag:sorted(set(keys)) for tag,keys in out.items() if keys}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'old': ['a'], 'shared': ['a', 'b']}, 'a', ['new'])), {'shared': ['b'], 'new': ['a']})
check('fixture 2', solve(*({'x': ['a', 'b']}, 'a', ['x'])), {'x': ['a', 'b']})
check('fixture 3', solve(*({}, 'a', [])), {})
check('fixture 4', solve(*({'x': ['a'], 'y': ['b']}, 'a', [])), {'y': ['b']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-tag-index-replacement

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(index, key, new_tags):
    return {tag:[key] for tag in sorted(set(new_tags))}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'old': ['a'], 'shared': ['a', 'b']}, 'a', ['new'])), {'shared': ['b'], 'new': ['a']})
check('fixture 2', solve(*({'x': ['a', 'b']}, 'a', ['x'])), {'x': ['a', 'b']})
check('fixture 3', solve(*({}, 'a', [])), {})
check('fixture 4', solve(*({'x': ['a'], 'y': ['b']}, 'a', [])), {'y': ['b']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Replacing entry tags leaves obsolete reverse-index membership.

Return tag-to-sorted-unique-keys index after replacing one key tags; drop empty buckets and preserve associations belonging to other keys.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, namespace):
    return [x for x in entries if not x[0].startswith(namespace)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['user', 1], ['users', 2], ['xuser', 3]], 'user')), [['users', 2], ['xuser', 3]])
check('fixture 2', solve(*([['a', 1]], 'z')), [['a', 1]])
check('fixture 3', solve(*([['', 1], ['x', 2]], '')), [['x', 2]])
check('fixture 4', solve(*([], 'a')), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-namespace-invalidation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, namespace):
    return [x for x in entries if namespace not in x[0]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['user', 1], ['users', 2], ['xuser', 3]], 'user')), [['users', 2], ['xuser', 3]])
check('fixture 2', solve(*([['a', 1]], 'z')), [['a', 1]])
check('fixture 3', solve(*([['', 1], ['x', 2]], '')), [['x', 2]])
check('fixture 4', solve(*([], 'a')), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A cache namespace purge deletes similarly prefixed namespaces.

Entries are [namespace,value]. Purge exact matching namespace identities only; namespaces are already separately parsed strings.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cache, key, value, started, current):
    out=dict(cache)
    out[key]=value
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({}, 'a', 'stale', 2, 3)), {})
check('fixture 2', solve(*({'a': 'new'}, 'a', 'old', 2, 3)), {'a': 'new'})
check('fixture 3', solve(*({}, 'a', 'fresh', 3, 3)), {'a': 'fresh'})
check('fixture 4', solve(*({'b': 0}, 'a', None, 0, 0)), {'b': 0, 'a': None})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-generation-invalidation-race

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cache, key, value, started, current):
    out=dict(cache)
    if started<=current: out[key]=value
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({}, 'a', 'stale', 2, 3)), {})
check('fixture 2', solve(*({'a': 'new'}, 'a', 'old', 2, 3)), {'a': 'new'})
check('fixture 3', solve(*({}, 'a', 'fresh', 3, 3)), {'a': 'fresh'})
check('fixture 4', solve(*({'b': 0}, 'a', None, 0, 0)), {'b': 0, 'a': None})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An old fetch repopulates a cache after invalidation.

Publish fetched value only when its captured generation equals the current generation; otherwise preserve the complete cache.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(vary, headers):
    return [[name,headers.get(name)] for name in vary[:1]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['accept', 'lang'], {'accept': 'json', 'lang': 'fr', 'trace': 'x'})), [['accept', 'json'], ['lang', 'fr']])
check('fixture 2', solve(*(['lang'], {'lang': 'en', 'trace': 'y'})), [['lang', 'en']])
check('fixture 3', solve(*([], {'trace': 'x'})), [])
check('fixture 4', solve(*(['lang', 'lang'], {})), [['lang', None]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-vary-field-values

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(vary, headers):
    return [[name,headers.get(name)] for name in sorted(headers)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['accept', 'lang'], {'accept': 'json', 'lang': 'fr', 'trace': 'x'})), [['accept', 'json'], ['lang', 'fr']])
check('fixture 2', solve(*(['lang'], {'lang': 'en', 'trace': 'y'})), [['lang', 'en']])
check('fixture 3', solve(*([], {'trace': 'x'})), [])
check('fixture 4', solve(*(['lang', 'lang'], {})), [['lang', None]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Representation identity ignores one of the selected request headers.

Header names are already normalized lowercase and values are strings. Return sorted unique requested Vary names paired with value or None for missing fields; ignore unselected headers.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, names, request):
    return [x for x in entries if not all(x[1].get(k,'')==request.get(k,'') for k in names)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['missing', {}], ['empty', {'x': ''}]], ['x'], {'x': ''})), [['missing', {}]])
check('fixture 2', solve(*([['one', {'x': 'one'}], ['two', {'x': 'two'}]], ['x'], {'x': 'one'})), [['two', {'x': 'two'}]])
check('fixture 3', solve(*([['missing', {}], ['empty', {'x': ''}]], ['x'], {})), [['empty', {'x': ''}]])
check('fixture 4', solve(*([], ['x'], {})), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-vary-missing-variant-invalidation

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, names, request):
    return [x for x in entries if not all(bool(x[1].get(k))==bool(request.get(k)) for k in names)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['missing', {}], ['empty', {'x': ''}]], ['x'], {'x': ''})), [['missing', {}]])
check('fixture 2', solve(*([['one', {'x': 'one'}], ['two', {'x': 'two'}]], ['x'], {'x': 'one'})), [['two', {'x': 'two'}]])
check('fixture 3', solve(*([['missing', {}], ['empty', {'x': ''}]], ['x'], {})), [['empty', {'x': ''}]])
check('fixture 4', solve(*([], ['x'], {})), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Invalidating an empty-header variant also removes the absent-header variant.

Entries are [variant-ID,normalized request-header mapping]. Remove entries matching every selected header in both presence and exact string value; preserve all other variants and their order. Empty selected names match every variant.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, resource, variant, value):
    return [x for x in entries if x[0]!=resource]+[[resource,variant,value]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['r', 'en', 1], ['r', 'fr', 2]], 'r', 'en', 3)), [['r', 'fr', 2], ['r', 'en', 3]])
check('fixture 2', solve(*([['r', 'en', 1], ['s', 'en', 2]], 'r', 'en', 3)), [['s', 'en', 2], ['r', 'en', 3]])
check('fixture 3', solve(*([], 'r', 'en', 0)), [['r', 'en', 0]])
check('fixture 4', solve(*([['r', 'fr', 2]], 'r', 'en', 3)), [['r', 'fr', 2], ['r', 'en', 3]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-variant-replacement-isolated

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, resource, variant, value):
    return [x for x in entries if x[1]!=variant]+[[resource,variant,value]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['r', 'en', 1], ['r', 'fr', 2]], 'r', 'en', 3)), [['r', 'fr', 2], ['r', 'en', 3]])
check('fixture 2', solve(*([['r', 'en', 1], ['s', 'en', 2]], 'r', 'en', 3)), [['s', 'en', 2], ['r', 'en', 3]])
check('fixture 3', solve(*([], 'r', 'en', 0)), [['r', 'en', 0]])
check('fixture 4', solve(*([['r', 'fr', 2]], 'r', 'en', 3)), [['r', 'fr', 2], ['r', 'en', 3]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Updating one cached representation removes sibling variants.

Unique entries are [resource,variant,value]. Replace only the exact pair and append its updated representation, preserving every other entry in order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, response):
    return [response[1],response[2]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['body', 1], ['not-modified', None, 2])), ['body', 2])
check('fixture 2', solve(*(['old', 1], ['modified', 'new', 2])), ['new', 2])
check('fixture 3', solve(*(['', 1], ['not-modified', None, 3])), ['', 3])
check('fixture 4', solve(*(['old', 1], ['modified', '', 4])), ['', 4])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-stale-revalidation-retains-body

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, response):
    return [entry[0],response[2]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['body', 1], ['not-modified', None, 2])), ['body', 2])
check('fixture 2', solve(*(['old', 1], ['modified', 'new', 2])), ['new', 2])
check('fixture 3', solve(*(['', 1], ['not-modified', None, 3])), ['', 3])
check('fixture 4', solve(*(['old', 1], ['modified', '', 4])), ['', 4])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A not-modified revalidation response replaces the cached body with an empty payload.

Entry is [body,validated-at]. Response is [modified-or-not-modified,body-or-None,validated-at]. Preserve old body only for not-modified; always update validation time.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, outcome, body):
    return None if outcome=='error' else body
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*('stale', 'error', None)), 'stale')
check('fixture 2', solve(*('old', 'success', 'new')), 'new')
check('fixture 3', solve(*(None, 'success', 0)), 0)
check('fixture 4', solve(*('', 'error', None)), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-revalidation-error-keeps-stale

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, outcome, body):
    return entry
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*('stale', 'error', None)), 'stale')
check('fixture 2', solve(*('old', 'success', 'new')), 'new')
check('fixture 3', solve(*(None, 'success', 0)), 0)
check('fixture 4', solve(*('', 'error', None)), '')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A refresh failure destroys a still-usable stale cache entry.

For this stale-allowed cache policy, error preserves the prior value; success replaces it with body, including zero or None. Expiration eligibility is handled before this operation.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(active, key, outcome):
    return sorted(set(active)-({key} if outcome=='success' else set()))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'b'], 'a', 'error')), ['b'])
check('fixture 2', solve(*(['a', 'b'], 'a', 'success')), ['b'])
check('fixture 3', solve(*(['b'], 'a', 'error')), ['b'])
check('fixture 4', solve(*([], 'a', 'success')), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-refresh-singleflight-cleanup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(active, key, outcome):
    return []
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(['a', 'b'], 'a', 'error')), ['b'])
check('fixture 2', solve(*(['a', 'b'], 'a', 'success')), ['b'])
check('fixture 3', solve(*(['b'], 'a', 'error')), ['b'])
check('fixture 4', solve(*([], 'a', 'success')), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Failed background refresh leaves a permanent in-flight marker.

Completion, successful or failed, removes only its own refresh key from the in-flight registry; output is a sorted unique key list.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, incoming):
    return list(incoming)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([3, 'new'], [2, 'old'])), [3, 'new'])
check('fixture 2', solve(*([3, 'old'], [4, 'new'])), [4, 'new'])
check('fixture 3', solve(*([3, 'kept'], [3, 'duplicate'])), [3, 'kept'])
check('fixture 4', solve(*([0, None], [5, 'value'])), [5, 'value'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-refresh-version-order

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entry, incoming):
    return list(incoming) if incoming[0]>entry[0]+1 else list(entry)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([3, 'new'], [2, 'old'])), [3, 'new'])
check('fixture 2', solve(*([3, 'old'], [4, 'new'])), [4, 'new'])
check('fixture 3', solve(*([3, 'kept'], [3, 'duplicate'])), [3, 'kept'])
check('fixture 4', solve(*([0, None], [5, 'value'])), [5, 'value'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An older refresh completion overwrites a newer cached representation.

Entries are [globally comparable integer version,value]. Install incoming only when its version is strictly greater; equal or older completions preserve current state.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, now):
    out=list(entries)
    while out and out[0][1]<=now:
        out.pop(0)
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['a', 20], ['b', 5], ['c', 30]], 10)), [['a', 20], ['c', 30]])
check('fixture 2', solve(*([['a', 10], ['b', 11]], 10)), [['b', 11]])
check('fixture 3', solve(*([], 10)), [])
check('fixture 4', solve(*([['a', 1], ['b', 2]], 10)), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-expired-sweep-mixed-order

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(entries, now):
    return [x for x in entries if x[1]>=now]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([['a', 20], ['b', 5], ['c', 30]], 10)), [['a', 20], ['c', 30]])
check('fixture 2', solve(*([['a', 10], ['b', 11]], 10)), [['b', 11]])
check('fixture 3', solve(*([], 10)), [])
check('fixture 4', solve(*([['a', 1], ['b', 2]], 10)), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An expiration sweep stops at a live entry and leaves later expired entries.

Entries have arbitrary order and are [key,exclusive expiration timestamp]. Sweep every expired entry while preserving survivor order; do not assume expiration and recency share order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(expires, now, ttl, hit):
    return now+ttl if hit else expires
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(10, 11, 5, True)), 10)
check('fixture 2', solve(*(10, 8, 5, True)), 13)
check('fixture 3', solve(*(10, 8, 5, False)), 10)
check('fixture 4', solve(*(10, 10, 5, True)), 10)
check('fixture 5', solve(*(10, 8, 0, True)), 8)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-sliding-expiry-touch

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(expires, now, ttl, hit):
    return expires+ttl if hit and now<expires else expires
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(10, 11, 5, True)), 10)
check('fixture 2', solve(*(10, 8, 5, True)), 13)
check('fixture 3', solve(*(10, 8, 5, False)), 10)
check('fixture 4', solve(*(10, 10, 5, True)), 10)
check('fixture 5', solve(*(10, 8, 0, True)), 8)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A sliding cache read moves the deadline even when the entry has already expired.

For a physical hit strictly before exclusive expiration, set deadline to now plus nonnegative TTL. Misses and already expired records preserve their old deadline; this operation only computes the deadline.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cache, keys):
    return [[k,"hit",cache[k]] for k in cache if k in keys]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'a': 1, 'b': 2}, ['b', 'a', 'b'])), [['b', 'hit', 2], ['a', 'hit', 1], ['b', 'hit', 2]])
check('fixture 2', solve(*({'a': None}, ['a', 'z'])), [['a', 'hit', None], ['z', 'miss', None]])
check('fixture 3', solve(*({}, [])), [])
check('fixture 4', solve(*({}, ['a'])), [['a', 'miss', None]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-batch-get-position

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(cache, keys):
    return [[k,"hit",cache[k]] if k in cache else [k,"miss",None] for k in dict.fromkeys(keys)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'a': 1, 'b': 2}, ['b', 'a', 'b'])), [['b', 'hit', 2], ['a', 'hit', 1], ['b', 'hit', 2]])
check('fixture 2', solve(*({'a': None}, ['a', 'z'])), [['a', 'hit', None], ['z', 'miss', None]])
check('fixture 3', solve(*({}, [])), [])
check('fixture 4', solve(*({}, ['a'])), [['a', 'miss', None]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Batch cache retrieval loses request order and duplicate positions.

Return one [key,hit-or-miss,value-or-None] result for every requested key in original order, including repeated keys and cached negative values.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(l1, l2, key):
    return [dict(l1),l1.get(key,l2.get(key))]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({}, {'a': 4}, 'a')), [{'a': 4}, 4])
check('fixture 2', solve(*({}, {'a': 0}, 'a')), [{'a': 0}, 0])
check('fixture 3', solve(*({'a': 'fresh'}, {'a': 'old'}, 'a')), [{'a': 'fresh'}, 'fresh'])
check('fixture 4', solve(*({'b': 2}, {}, 'a')), [{'b': 2}, None])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xc-multi-tier-promotion

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(l1, l2, key):
    out=dict(l1)
    if l2.get(key): out[key]=l2[key]
    return [out,out.get(key)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({}, {'a': 4}, 'a')), [{'a': 4}, 4])
check('fixture 2', solve(*({}, {'a': 0}, 'a')), [{'a': 0}, 0])
check('fixture 3', solve(*({'a': 'fresh'}, {'a': 'old'}, 'a')), [{'a': 'fresh'}, 'fresh'])
check('fixture 4', solve(*({'b': 2}, {}, 'a')), [{'b': 2}, None])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An L2 hit fails to populate the faster cache tier.

On an L1 miss and L2 hit, copy the L2 value into L1. Existing L1 data wins. Return [new-L1,selected value or None]; this model output does not distinguish a terminal miss from a cached None.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, alpha, destination):
    return source+destination*(1-alpha)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.8, 0.5, 0.2)), 0.5)
check('fixture 2', solve(*(1, 0, 0.25)), 0.25)
check('fixture 3', solve(*(0.25, 1, 0.8)), 0.25)
check('fixture 4', solve(*(0, 0.5, 1)), 0.5)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-straight-alpha-source-over-channel

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, alpha, destination):
    return source*alpha+destination
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.8, 0.5, 0.2)), 0.5)
check('fixture 2', solve(*(1, 0, 0.25)), 0.25)
check('fixture 3', solve(*(0.25, 1, 0.8)), 0.25)
check('fixture 4', solve(*(0, 0.5, 1)), 0.5)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A straight source channel is treated as already premultiplied.

Return a normalized straight source channel composited over an opaque destination channel. All three inputs lie in [0,1].

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, alpha, destination):
    return source*alpha+destination*(1-alpha)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.4, 0.5, 0.2)), 0.5)
check('fixture 2', solve(*(0, 0, 0.25)), 0.25)
check('fixture 3', solve(*(0.25, 1, 0.8)), 0.25)
check('fixture 4', solve(*(0.125, 0.5, 0.5)), 0.375)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-premultiplied-source-over-channel

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, alpha, destination):
    return source+destination
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.4, 0.5, 0.2)), 0.5)
check('fixture 2', solve(*(0, 0, 0.25)), 0.25)
check('fixture 3', solve(*(0.25, 1, 0.8)), 0.25)
check('fixture 4', solve(*(0.125, 0.5, 0.5)), 0.375)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Premultiplied foreground is multiplied by opacity a second time.

Source is a premultiplied channel in [0,alpha]; destination is a normalized premultiplied channel. Return source-over premultiplied channel.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source_alpha, destination_alpha):
    return source_alpha+destination_alpha
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.5, 0.5)), 0.75)
check('fixture 2', solve(*(0, 0.5)), 0.5)
check('fixture 3', solve(*(1, 0.25)), 1)
check('fixture 4', solve(*(0.25, 0.5)), 0.625)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-source-over-alpha-union

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source_alpha, destination_alpha):
    return max(source_alpha,destination_alpha)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.5, 0.5)), 0.75)
check('fixture 2', solve(*(0, 0.5)), 0.5)
check('fixture 3', solve(*(1, 0.25)), 1)
check('fixture 4', solve(*(0.25, 0.5)), 0.625)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Adding opacities treats overlap as disjoint coverage.

Return source-over output alpha for two normalized opacities.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgba):
    return list(rgba)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 0.5, 0.25, 0.5],)), [0.5, 0.25, 0.125, 0.5])
check('fixture 2', solve(*([1, 1, 1, 0],)), [0, 0, 0, 0])
check('fixture 3', solve(*([0.25, 0.5, 1, 1],)), [0.25, 0.5, 1, 1])
check('fixture 4', solve(*([0, 0, 0, 0.5],)), [0, 0, 0, 0.5])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-premultiply-all-color-channels

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgba):
    return [x*rgba[3] for x in rgba]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 0.5, 0.25, 0.5],)), [0.5, 0.25, 0.125, 0.5])
check('fixture 2', solve(*([1, 1, 1, 0],)), [0, 0, 0, 0])
check('fixture 3', solve(*([0.25, 0.5, 1, 1],)), [0.25, 0.5, 1, 1])
check('fixture 4', solve(*([0, 0, 0, 0.5],)), [0, 0, 0, 0.5])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Straight RGB channels enter a pipeline that expects premultiplied channels.

Convert normalized straight RGBA to premultiplied RGBA, leaving alpha itself unchanged.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgba):
    return list(rgba)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0.5, 0.25, 0.125, 0.5],)), [1, 0.5, 0.25, 0.5])
check('fixture 2', solve(*([0, 0, 0, 0],)), [0, 0, 0, 0])
check('fixture 3', solve(*([0.25, 0.5, 1, 1],)), [0.25, 0.5, 1, 1])
check('fixture 4', solve(*([0, 0.125, 0, 0.25],)), [0, 0.5, 0, 0.25])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-unpremultiply-transparent-color

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgba):
    return [x/max(rgba[3],1) for x in rgba[:3]]+[rgba[3]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0.5, 0.25, 0.125, 0.5],)), [1, 0.5, 0.25, 0.5])
check('fixture 2', solve(*([0, 0, 0, 0],)), [0, 0, 0, 0])
check('fixture 3', solve(*([0.25, 0.5, 1, 1],)), [0.25, 0.5, 1, 1])
check('fixture 4', solve(*([0, 0.125, 0, 0.25],)), [0, 0.5, 0, 0.25])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Premultiplied channels are exposed as straight channels and darken translucent pixels.

Input is valid premultiplied normalized RGBA. Divide RGB by nonzero alpha; represent fully transparent output as transparent black.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgba, opacity):
    return rgba[:3]+[rgba[3]*opacity]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0.5, 0.25, 0, 0.5], 0.5)), [0.25, 0.125, 0, 0.25])
check('fixture 2', solve(*([1, 1, 1, 1], 0)), [0, 0, 0, 0])
check('fixture 3', solve(*([0.25, 0, 0, 0.5], 1)), [0.25, 0, 0, 0.5])
check('fixture 4', solve(*([0, 0, 0, 0], 0.5)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-global-opacity-premultiplied

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgba, opacity):
    return [x*opacity for x in rgba[:3]]+[rgba[3]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0.5, 0.25, 0, 0.5], 0.5)), [0.25, 0.125, 0, 0.25])
check('fixture 2', solve(*([1, 1, 1, 1], 0)), [0, 0, 0, 0])
check('fixture 3', solve(*([0.25, 0, 0, 0.5], 1)), [0.25, 0, 0, 0.5])
check('fixture 4', solve(*([0, 0, 0, 0], 0.5)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Layer opacity reduces alpha while leaving premultiplied color unchanged.

Apply normalized layer opacity to premultiplied RGBA by scaling both RGB and alpha.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(destination, source_alpha):
    return [x*source_alpha for x in destination]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0.5, 0.25, 0, 0.5], 0.25)), [0.375, 0.1875, 0, 0.375])
check('fixture 2', solve(*([1, 1, 1, 1], 1)), [0, 0, 0, 0])
check('fixture 3', solve(*([0.5, 0.5, 0.5, 0.5], 0)), [0.5, 0.5, 0.5, 0.5])
check('fixture 4', solve(*([0, 0, 0, 0], 0.5)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-destination-out-preserves-uncovered

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(destination, source_alpha):
    return destination[:3]+[destination[3]*(1-source_alpha)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0.5, 0.25, 0, 0.5], 0.25)), [0.375, 0.1875, 0, 0.375])
check('fixture 2', solve(*([1, 1, 1, 1], 1)), [0, 0, 0, 0])
check('fixture 3', solve(*([0.5, 0.5, 0.5, 0.5], 0)), [0.5, 0.5, 0.5, 0.5])
check('fixture 4', solve(*([0, 0, 0, 0], 0.5)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The erased coverage is retained instead of its complement.

Destination is premultiplied RGBA. Destination-out retains each channel times one minus normalized source alpha.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, destination_alpha):
    return list(source)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0.5, 0.25, 0, 0.5], 0.5)), [0.25, 0.125, 0, 0.25])
check('fixture 2', solve(*([1, 1, 1, 1], 0)), [0, 0, 0, 0])
check('fixture 3', solve(*([0.5, 0, 0, 0.5], 1)), [0.5, 0, 0, 0.5])
check('fixture 4', solve(*([0, 0, 0, 0], 0.5)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-source-in-clips-premultiplied

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, destination_alpha):
    return source[:3]+[source[3]*destination_alpha]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0.5, 0.25, 0, 0.5], 0.5)), [0.25, 0.125, 0, 0.25])
check('fixture 2', solve(*([1, 1, 1, 1], 0)), [0, 0, 0, 0])
check('fixture 3', solve(*([0.5, 0, 0, 0.5], 1)), [0.5, 0, 0, 0.5])
check('fixture 4', solve(*([0, 0, 0, 0], 0.5)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Source-in ignores the destination coverage mask.

Source is premultiplied RGBA. Source-in multiplies every channel by normalized destination alpha.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, sa, destination, da):
    return source+destination
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.5, 0.5, 0.25, 0.25)), 0.5)
check('fixture 2', solve(*(1, 1, 1, 1)), 0)
check('fixture 3', solve(*(0.25, 0.25, 0, 0)), 0.25)
check('fixture 4', solve(*(0, 0, 0.5, 0.5)), 0.5)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-xor-disjoint-coverage

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, sa, destination, da):
    return source*(1-sa)+destination*(1-da)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.5, 0.5, 0.25, 0.25)), 0.5)
check('fixture 2', solve(*(1, 1, 1, 1)), 0)
check('fixture 3', solve(*(0.25, 0.25, 0, 0)), 0.25)
check('fixture 4', solve(*(0, 0, 0.5, 0.5)), 0.5)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Adding both channels retains their overlap during exclusive compositing.

Source and destination are premultiplied channel values with corresponding normalized alphas. Return Porter-Duff XOR channel.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, destination):
    return source+destination
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.75, 0.5)), 1)
check('fixture 2', solve(*(0.25, 0.5)), 0.75)
check('fixture 3', solve(*(0, 0)), 0)
check('fixture 4', solve(*(1, 0)), 1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-plus-blend-clamps-channel

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, destination):
    return max(source,destination)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.75, 0.5)), 1)
check('fixture 2', solve(*(0.25, 0.5)), 0.75)
check('fixture 3', solve(*(0, 0)), 0)
check('fixture 4', solve(*(1, 0)), 1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Additive blend writes values above the normalized render-target maximum.

Return saturated additive blending for normalized nonnegative channel inputs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, destination):
    return source*destination
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.5, 0.5)), 0.75)
check('fixture 2', solve(*(0, 0.25)), 0.25)
check('fixture 3', solve(*(1, 0.25)), 1)
check('fixture 4', solve(*(0.25, 0.5)), 0.625)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-screen-blend-complement-product

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, destination):
    return min(1,source+destination)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.5, 0.5)), 0.75)
check('fixture 2', solve(*(0, 0.25)), 0.25)
check('fixture 3', solve(*(1, 0.25)), 1)
check('fixture 4', solve(*(0.25, 0.5)), 0.625)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Multiplication computes a darkening blend rather than screen.

Return screen blending for two normalized straight color channels.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, destination):
    return 2*source*destination if source<=.5 else 1-2*(1-source)*(1-destination)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.75, 0.25)), 0.375)
check('fixture 2', solve(*(0.25, 0.75)), 0.625)
check('fixture 3', solve(*(0.5, 0.5)), 0.5)
check('fixture 4', solve(*(0, 1)), 1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-overlay-branches-on-backdrop

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(source, destination):
    return source*destination
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.75, 0.25)), 0.375)
check('fixture 2', solve(*(0.25, 0.75)), 0.625)
check('fixture 3', solve(*(0.5, 0.5)), 0.5)
check('fixture 4', solve(*(0, 1)), 1)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Overlay uses source brightness to choose its branch, yielding hard-light instead.

Return overlay blending for normalized straight channels; the backdrop destination chooses the dark or light branch.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgba):
    return list(rgba)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 2, 3, 4],)), [3, 2, 1, 4])
check('fixture 2', solve(*([255, 0, 0, 128],)), [0, 0, 255, 128])
check('fixture 3', solve(*([0, 0, 0, 0],)), [0, 0, 0, 0])
check('fixture 4', solve(*([7, 7, 7, 9],)), [7, 7, 7, 9])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-rgba-byte-channel-reorder

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgba):
    return [rgba[3],rgba[2],rgba[1],rgba[0]]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 2, 3, 4],)), [3, 2, 1, 4])
check('fixture 2', solve(*([255, 0, 0, 128],)), [0, 0, 255, 128])
check('fixture 3', solve(*([0, 0, 0, 0],)), [0, 0, 0, 0])
check('fixture 4', solve(*([7, 7, 7, 9],)), [7, 7, 7, 9])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
An RGBA source buffer is uploaded unchanged to a BGRA consumer.

Convert RGBA byte channel order into BGRA without changing channel values or alpha position.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgba):
    return (rgba[3]<<24)|(rgba[2]<<16)|(rgba[1]<<8)|rgba[0]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 2, 3, 4],)), 16909060)
check('fixture 2', solve(*([0, 0, 0, 255],)), 255)
check('fixture 3', solve(*([255, 0, 0, 0],)), 4278190080)
check('fixture 4', solve(*([0, 0, 0, 0],)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-pack-rgba-big-endian-word

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgba):
    return (rgba[0]<<24)|(rgba[1]<<16)|(rgba[2]<<8)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 2, 3, 4],)), 16909060)
check('fixture 2', solve(*([0, 0, 0, 255],)), 255)
check('fixture 3', solve(*([255, 0, 0, 0],)), 4278190080)
check('fixture 4', solve(*([0, 0, 0, 0],)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Packing assumes little-endian channel significance instead of the stated integer layout.

Pack four integer bytes RGBA into an unsigned word with red as most significant byte and alpha as least significant byte.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(word):
    return [word>>shift for shift in (24,16,8,0)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(16909060,)), [1, 2, 3, 4])
check('fixture 2', solve(*(255,)), [0, 0, 0, 255])
check('fixture 3', solve(*(4278190080,)), [255, 0, 0, 0])
check('fixture 4', solve(*(0,)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-unpack-rgba-byte-mask

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(word):
    return [(word>>shift)&255 for shift in (0,8,16,24)]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(16909060,)), [1, 2, 3, 4])
check('fixture 2', solve(*(255,)), [0, 0, 0, 255])
check('fixture 3', solve(*(4278190080,)), [255, 0, 0, 0])
check('fixture 4', solve(*(0,)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Shifted higher-order channels leak into lower byte values.

Unpack a 32-bit unsigned big-endian RGBA integer into four bytes, masking each shifted field.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(value):
    return int(value*255)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.5,)), 128)
check('fixture 2', solve(*(0.3,)), 77)
check('fixture 3', solve(*(-0.5,)), 0)
check('fixture 4', solve(*(1.5,)), 255)
check('fixture 5', solve(*(0,)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-normalized-channel-to-byte-half-up

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(value):
    return min(255,max(0,round(value*255)))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.5,)), 128)
check('fixture 2', solve(*(0.3,)), 77)
check('fixture 3', solve(*(-0.5,)), 0)
check('fixture 4', solve(*(1.5,)), 255)
check('fixture 5', solve(*(0,)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Truncating scaled color introduces a downward quantization bias.

Clamp a finite normalized channel to byte range and round the scaled value to nearest with half-up ties.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(byte):
    return byte/256
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(255,)), 1)
check('fixture 2', solve(*(0,)), 0)
check('fixture 3', solve(*(85,)), 0.3333333333333333)
check('fixture 4', solve(*(128,)), 0.5019607843137255)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-byte-channel-normalization

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(byte):
    return byte//255
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(255,)), 1)
check('fixture 2', solve(*(0,)), 0)
check('fixture 3', solve(*(85,)), 0.3333333333333333)
check('fixture 4', solve(*(128,)), 0.5019607843137255)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Dividing by 256 makes the maximum representable white less than one.

Map integer byte values in [0,255] to normalized [0,1] with both endpoints exact.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(r, g, b):
    return ((r>>3)<<11)|((g>>3)<<5)|(b>>3)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(255, 255, 255)), 65535)
check('fixture 2', solve(*(0, 255, 0)), 2016)
check('fixture 3', solve(*(255, 0, 0)), 63488)
check('fixture 4', solve(*(0, 0, 255)), 31)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(r, g, b):
    return ((r>>3)<<10)|((g>>3)<<5)|(b>>3)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(255, 255, 255)), 65535)
check('fixture 2', solve(*(0, 255, 0)), 2016)
check('fixture 3', solve(*(255, 0, 0)), 63488)
check('fixture 4', solve(*(0, 0, 255)), 31)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Green is quantized to five bits in a six-bit field.

Quantize three RGB bytes into RGB565 using high five red bits, high six green bits and high five blue bits.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(word):
    return [((word>>11)&31)<<3,((word>>5)&63)<<2,(word&31)<<3]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(65535,)), [255, 255, 255])
check('fixture 2', solve(*(0,)), [0, 0, 0])
check('fixture 3', solve(*(63488,)), [255, 0, 0])
check('fixture 4', solve(*(2016,)), [0, 255, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(word):
    return [((word>>11)&31)*8,((word>>5)&31)*8,(word&31)*8]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(65535,)), [255, 255, 255])
check('fixture 2', solve(*(0,)), [0, 0, 0])
check('fixture 3', solve(*(63488,)), [255, 0, 0])
check('fixture 4', solve(*(2016,)), [0, 255, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Zero-padding low bits leaves decoded maximum channels below 255.

Expand RGB565 into RGB bytes by replicating high bits into low bits, preserving full-scale white.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(indices, palette):
    return [list(palette[i][:3])+[255] for i in indices]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 0], [[255, 0, 0, 255], [0, 0, 255, 0]])), [[0, 0, 255, 0], [255, 0, 0, 255]])
check('fixture 2', solve(*([], [[0, 0, 0, 0]])), [])
check('fixture 3', solve(*([0], [[1, 2, 3, 128]])), [[1, 2, 3, 128]])
check('fixture 4', solve(*([1, 1], [[0, 0, 0, 255], [5, 6, 7, 64]])), [[5, 6, 7, 64], [5, 6, 7, 64]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-palette-alpha-lookup

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(indices, palette):
    return [list(palette[0]) for i in indices]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 0], [[255, 0, 0, 255], [0, 0, 255, 0]])), [[0, 0, 255, 0], [255, 0, 0, 255]])
check('fixture 2', solve(*([], [[0, 0, 0, 0]])), [])
check('fixture 3', solve(*([0], [[1, 2, 3, 128]])), [[1, 2, 3, 128]])
check('fixture 4', solve(*([1, 1], [[0, 0, 0, 255], [5, 6, 7, 64]])), [[5, 6, 7, 64], [5, 6, 7, 64]])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Palette expansion makes every transparent palette entry opaque.

Expand valid palette indices into independent RGBA byte lists, retaining per-entry transparency.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(gray, alpha):
    return [gray,gray,gray,gray]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(20, 128)), [20, 20, 20, 128])
check('fixture 2', solve(*(0, 255)), [0, 0, 0, 255])
check('fixture 3', solve(*(255, 0)), [255, 255, 255, 0])
check('fixture 4', solve(*(0, 0)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-grayscale-opacity-independent

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(gray, alpha):
    return [gray,gray,gray,255]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(20, 128)), [20, 20, 20, 128])
check('fixture 2', solve(*(0, 255)), [0, 0, 0, 255])
check('fixture 3', solve(*(255, 0)), [255, 255, 255, 0])
check('fixture 4', solve(*(0, 0)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The grayscale luminance value is incorrectly copied into alpha.

Expand a straight grayscale byte and independent alpha byte into straight RGBA.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgb, key, alpha):
    return list(rgb)+[alpha]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 2, 3], [1, 2, 3], 255)), [1, 2, 3, 0])
check('fixture 2', solve(*([1, 2, 4], [1, 2, 3], 128)), [1, 2, 4, 128])
check('fixture 3', solve(*([0, 0, 0], [1, 2, 3], 0)), [0, 0, 0, 0])
check('fixture 4', solve(*([9, 8, 7], [1, 2, 3], 255)), [9, 8, 7, 255])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-color-key-before-opacity

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(rgb, key, alpha):
    return list(rgb)+[0 if any(a==b for a,b in zip(rgb,key)) else alpha]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 2, 3], [1, 2, 3], 255)), [1, 2, 3, 0])
check('fixture 2', solve(*([1, 2, 4], [1, 2, 3], 128)), [1, 2, 4, 128])
check('fixture 3', solve(*([0, 0, 0], [1, 2, 3], 0)), [0, 0, 0, 0])
check('fixture 4', solve(*([9, 8, 7], [1, 2, 3], 255)), [9, 8, 7, 255])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Color-key transparency is ignored during RGBA conversion.

For straight RGB bytes, matching all three color-key channels sets alpha to zero; other pixels retain supplied alpha.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(row, start, end):
    return row[start:end]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0, 1, 2, 3], -2, 2)), [0, 1])
check('fixture 2', solve(*([0, 1, 2, 3], 1, 3)), [1, 2])
check('fixture 3', solve(*([0, 1], 4, 8)), [])
check('fixture 4', solve(*([0, 1], 0, 0)), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-clip-row-both-ends

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(row, start, end):
    return row[max(0,start):min(len(row),end)+1]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0, 1, 2, 3], -2, 2)), [0, 1])
check('fixture 2', solve(*([0, 1, 2, 3], 1, 3)), [1, 2])
check('fixture 3', solve(*([0, 1], 4, 8)), [])
check('fixture 4', solve(*([0, 1], 0, 0)), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Negative clipping coordinates are interpreted as Python indices from the row end.

Clip integer half-open horizontal coverage [start,end) against a row extent [0,len(row)); return covered pixels in order.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(destination, source, x):
    out=list(destination)
    for i,pixel in enumerate(source):
        if 0<=max(0,x)+i<len(out): out[max(0,x)+i]=pixel
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0, 0, 0], [7, 8, 9], -1)), [8, 9, 0])
check('fixture 2', solve(*([0, 0, 0], [7, 8], 2)), [0, 0, 7])
check('fixture 3', solve(*([0, 0], [7], 0)), [7, 0])
check('fixture 4', solve(*([0, 0], [7], 3)), [0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-clipped-blit-source-offset

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(destination, source, x):
    out=list(destination)
    for i,pixel in enumerate(source):
        if 0<x+i<len(out): out[x+i]=pixel
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0, 0, 0], [7, 8, 9], -1)), [8, 9, 0])
check('fixture 2', solve(*([0, 0, 0], [7, 8], 2)), [0, 0, 7])
check('fixture 3', solve(*([0, 0], [7], 0)), [7, 0])
check('fixture 4', solve(*([0, 0], [7], 3)), [0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Clamping the destination origin shifts discarded source pixels back into view.

Blit source row at signed destination x, discarding out-of-range destination pixels while retaining original source alignment.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(a, b):
    return [min(a[0],b[0]),min(a[1],b[1]),max(a[2],b[2]),max(a[3],b[3])]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0, 0, 4, 4], [2, 1, 6, 3])), [2, 1, 4, 3])
check('fixture 2', solve(*([0, 0, 1, 1], [3, 3, 4, 4])), [3, 3, 3, 3])
check('fixture 3', solve(*([0, 0, 2, 2], [2, 0, 3, 2])), [2, 0, 2, 2])
check('fixture 4', solve(*([0, 0, 4, 4], [1, 1, 2, 2])), [1, 1, 2, 2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-scissor-intersection-empty

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(a, b):
    return [max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([0, 0, 4, 4], [2, 1, 6, 3])), [2, 1, 4, 3])
check('fixture 2', solve(*([0, 0, 1, 1], [3, 3, 4, 4])), [3, 3, 3, 3])
check('fixture 3', solve(*([0, 0, 2, 2], [2, 0, 3, 2])), [2, 0, 2, 2])
check('fixture 4', solve(*([0, 0, 4, 4], [1, 1, 2, 2])), [1, 1, 2, 2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Combining scissor bounds as a union enlarges the drawable region.

Intersect two valid integer half-open scissor rectangles [left,top,right,bottom]. Clamp negative overlap width/height to zero at the maximum start coordinate.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(top, height, framebuffer_height):
    return framebuffer_height-top
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(2, 3, 10)), 5)
check('fixture 2', solve(*(0, 10, 10)), 0)
check('fixture 3', solve(*(0, 1, 10)), 9)
check('fixture 4', solve(*(9, 1, 10)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-viewport-bottom-origin-flip

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(top, height, framebuffer_height):
    return top
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(2, 3, 10)), 5)
check('fixture 2', solve(*(0, 10, 10)), 0)
check('fixture 3', solve(*(0, 1, 10)), 9)
check('fixture 4', solve(*(9, 1, 10)), 0)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Origin conversion flips the top edge without accounting for rectangle height.

Convert the top coordinate of an in-bounds top-left-origin rectangle to its bottom-left-origin viewport y using the full rectangle height.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(width, left, right):
    return [i for i in range(width) if left<=i<right]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(4, 0.5, 2.5)), [0, 1])
check('fixture 2', solve(*(4, 0.6, 2.6)), [1, 2])
check('fixture 3', solve(*(3, 0, 3)), [0, 1, 2])
check('fixture 4', solve(*(3, 1, 1)), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-pixel-center-coverage

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(width, left, right):
    return [i for i in range(width) if left<i+.5<right]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(4, 0.5, 2.5)), [0, 1])
check('fixture 2', solve(*(4, 0.6, 2.6)), [1, 2])
check('fixture 3', solve(*(3, 0, 3)), [0, 1, 2])
check('fixture 4', solve(*(3, 1, 1)), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Coverage samples pixel corners instead of pixel centers.

List integer pixel indices whose centers i+0.5 are covered by finite half-open span [left,right); width is a nonnegative integer.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(start, end, tile):
    return list(range(start//tile,end//tile+1))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0, 8, 4)), [0, 1])
check('fixture 2', solve(*(1, 5, 4)), [0, 1])
check('fixture 3', solve(*(4, 4, 4)), [])
check('fixture 4', solve(*(4, 8, 4)), [1])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-damage-tiles-exclusive-end

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(start, end, tile):
    return list(range(start//tile,end//tile))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0, 8, 4)), [0, 1])
check('fixture 2', solve(*(1, 5, 4)), [0, 1])
check('fixture 3', solve(*(4, 4, 4)), [])
check('fixture 4', solve(*(4, 8, 4)), [1])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A damage interval ending on a tile boundary invalidates the following clean tile.

For nonnegative integer damaged span [start,end) with end>=start and positive tile size, list every intersected tile index once.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(enabled, scissor, width, height):
    return list(scissor)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(False, [1, 1, 2, 2], 4, 3)), [0, 0, 4, 3])
check('fixture 2', solve(*(True, [1, 1, 2, 2], 4, 3)), [1, 1, 2, 2])
check('fixture 3', solve(*(False, [9, 9, 10, 10], 0, 0)), [0, 0, 0, 0])
check('fixture 4', solve(*(True, [0, 0, 0, 0], 4, 3)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-scissor-disabled-full-target

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(enabled, scissor, width, height):
    return [0,0,width,height]
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(False, [1, 1, 2, 2], 4, 3)), [0, 0, 4, 3])
check('fixture 2', solve(*(True, [1, 1, 2, 2], 4, 3)), [1, 1, 2, 2])
check('fixture 3', solve(*(False, [9, 9, 10, 10], 0, 0)), [0, 0, 0, 0])
check('fixture 4', solve(*(True, [0, 0, 0, 0], 4, 3)), [0, 0, 0, 0])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A remembered scissor rectangle still clips rendering after scissor testing is disabled.

Resolve effective scissor bounds: enabled uses the supplied rectangle, disabled uses the complete nonnegative-sized render target.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(spans):
    out=[]
    for left,right in sorted(spans):
        if right<=left: continue
        if out and left<out[-1][1]: out[-1][1]=max(out[-1][1],right)
        else: out.append([left,right])
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([[0, 2], [2, 4]],)), [[0, 4]])
check('fixture 2', solve(*([[0, 5], [1, 2]],)), [[0, 5]])
check('fixture 3', solve(*([[3, 3], [1, 2]],)), [[1, 2]])
check('fixture 4', solve(*([],)), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-span-merge-touching

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(spans):
    out=[]
    for left,right in sorted(spans):
        if out and left<=out[-1][1]: out[-1][1]=right
        else: out.append([left,right])
    return out
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([[0, 2], [2, 4]],)), [[0, 4]])
check('fixture 2', solve(*([[0, 5], [1, 2]],)), [[0, 5]])
check('fixture 3', solve(*([[3, 3], [1, 2]],)), [[1, 2]])
check('fixture 4', solve(*([],)), [])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Touching coverage spans remain separate and create redundant raster work.

Return sorted disjoint half-open raster spans after dropping zero/negative lengths and coalescing both overlapping and adjacent spans.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(state, channel, value):
    saved=dict(state)
    current=dict(state)
    current['color']=list(current['color'])
    saved['color']=current['color']
    current['color'][channel]=value
    return saved
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'color': [1, 2, 3, 4], 'viewport': [0, 0, 8, 8]}, 0, 9)), {'color': [1, 2, 3, 4], 'viewport': [0, 0, 8, 8]})
check('fixture 2', solve(*({'color': [0, 0, 0, 0]}, 3, 5)), {'color': [0, 0, 0, 0]})
check('fixture 3', solve(*({'color': [1, 1, 1, 1], 'other': [2]}, 1, 7)), {'color': [1, 1, 1, 1], 'other': [2]})
check('fixture 4', solve(*({'color': [4, 3, 2, 1]}, 0, 4)), {'color': [4, 3, 2, 1]})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-state-push-deep-copy

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(state, channel, value):
    return {'color':list(state['color'])}
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*({'color': [1, 2, 3, 4], 'viewport': [0, 0, 8, 8]}, 0, 9)), {'color': [1, 2, 3, 4], 'viewport': [0, 0, 8, 8]})
check('fixture 2', solve(*({'color': [0, 0, 0, 0]}, 3, 5)), {'color': [0, 0, 0, 0]})
check('fixture 3', solve(*({'color': [1, 1, 1, 1], 'other': [2]}, 1, 7)), {'color': [1, 1, 1, 1], 'other': [2]})
check('fixture 4', solve(*({'color': [4, 3, 2, 1]}, 0, 4)), {'color': [4, 3, 2, 1]})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The saved state aliases a mutable color list and changes when current state changes.

State maps names to flat lists and contains four-channel color. Save a snapshot, mutate a current color channel, and return the untouched full snapshot; do not mutate input.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(old, new, mask):
    return list(new) if any(mask) else list(old)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 2, 3, 4], [9, 8, 7, 6], [True, False, True, False])), [9, 2, 7, 4])
check('fixture 2', solve(*([1, 2, 3, 4], [9, 8, 7, 6], [False, False, False, False])), [1, 2, 3, 4])
check('fixture 3', solve(*([1, 2, 3, 4], [9, 8, 7, 6], [True, True, True, True])), [9, 8, 7, 6])
check('fixture 4', solve(*([0, 0, 0, 0], [1, 2, 3, 4], [False, False, False, True])), [0, 0, 0, 4])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-color-write-mask-channelwise

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(old, new, mask):
    return list(new) if all(mask) else list(old)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*([1, 2, 3, 4], [9, 8, 7, 6], [True, False, True, False])), [9, 2, 7, 4])
check('fixture 2', solve(*([1, 2, 3, 4], [9, 8, 7, 6], [False, False, False, False])), [1, 2, 3, 4])
check('fixture 3', solve(*([1, 2, 3, 4], [9, 8, 7, 6], [True, True, True, True])), [9, 8, 7, 6])
check('fixture 4', solve(*([0, 0, 0, 0], [1, 2, 3, 4], [False, False, False, True])), [0, 0, 0, 4])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Any enabled color channel causes the entire pixel to be overwritten.

Old/new are four channels and mask contains four Booleans. Replace exactly the enabled channels.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(old, incoming, test_passed, write_enabled):
    return incoming if test_passed else old
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.5, 0.25, True, False)), 0.5)
check('fixture 2', solve(*(0.5, 0.25, False, True)), 0.5)
check('fixture 3', solve(*(0.5, 0.25, True, True)), 0.25)
check('fixture 4', solve(*(0.5, 0.25, False, False)), 0.5)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-depth-write-independent-test

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(old, incoming, test_passed, write_enabled):
    return incoming if write_enabled else old
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(0.5, 0.25, True, False)), 0.5)
check('fixture 2', solve(*(0.5, 0.25, False, True)), 0.5)
check('fixture 3', solve(*(0.5, 0.25, True, True)), 0.25)
check('fixture 4', solve(*(0.5, 0.25, False, False)), 0.5)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Passing depth testing overrides a disabled depth write mask.

Update a scalar depth buffer only if fragment depth testing passed and depth writes are enabled.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(old, new, mask):
    return new & mask
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(240, 15, 15)), 255)
check('fixture 2', solve(*(170, 85, 0)), 170)
check('fixture 3', solve(*(170, 85, 255)), 85)
check('fixture 4', solve(*(0, 255, 240)), 240)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-stencil-write-mask-preserves-bits

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(old, new, mask):
    return (old & mask) | (new & (255^mask))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(240, 15, 15)), 255)
check('fixture 2', solve(*(170, 85, 0)), 170)
check('fixture 3', solve(*(170, 85, 255)), 85)
check('fixture 4', solve(*(0, 255, 240)), 240)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Masking only the new stencil value clears protected existing bits.

Old/new/mask are bytes. Replace only stencil bits enabled by the write mask; preserve every masked-off old bit.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(value):
    return (value+1)&255
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(255,)), 255)
check('fixture 2', solve(*(254,)), 255)
check('fixture 3', solve(*(0,)), 1)
check('fixture 4', solve(*(100,)), 101)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xg-stencil-increment-saturates

source:
"""Failure Map reference implementation. Python standard library only."""
import json
import math
N = 1
observations = []
def solve(value):
    return min(254,value+1)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(*(255,)), 255)
check('fixture 2', solve(*(254,)), 255)
check('fixture 3', solve(*(0,)), 1)
check('fixture 4', solve(*(100,)), 101)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Stencil increment wraps a saturated 255 value back to zero.

Increment a stencil byte using the saturating increment operation, distinct from increment-and-wrap.

split: open-access