broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return (False or state['digest'] != state['old_digest'] or state['tool'] != state['old_tool'] or state['recipe'] != state['old_recipe'] or state['failed'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 2', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 3', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 4', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 5', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 6', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 7', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 8', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 9', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 10', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 11', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 12', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 13', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 14', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 15', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 16', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 17', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), False)
check('fixture 18', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 19', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 20', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 21', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 22', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 23', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 24', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 25', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 26', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 27', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 28', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 29', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 30', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 31', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 32', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rebuild

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return (state['exists'] or state['digest'] != state['old_digest'] or state['tool'] != state['old_tool'] or state['recipe'] != state['old_recipe'] or state['failed'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 2', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 3', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 4', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 5', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 6', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 7', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 8', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 9', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 10', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 11', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 12', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 13', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 14', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 15', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 16', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 17', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), False)
check('fixture 18', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 19', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 20', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 21', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 22', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 23', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 24', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 25', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 26', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 27', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 28', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 29', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 30', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 31', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 32', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Missing outputs are accepted as cached.

Rebuild if output absent, content digest changed, tool changed, recipe changed, or a prerequisite failed; timestamps alone are irrelevant.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return (not state['exists'] or False or state['tool'] != state['old_tool'] or state['recipe'] != state['old_recipe'] or state['failed'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 2', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 3', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 4', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 5', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 6', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 7', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 8', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 9', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 10', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 11', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 12', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 13', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 14', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 15', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 16', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 17', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), False)
check('fixture 18', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 19', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 20', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 21', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 22', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 23', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 24', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 25', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 26', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 27', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 28', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 29', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 30', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 31', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 32', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xt-rebuild

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return (not state['exists'] or state['digest'] == state['old_digest'] or state['tool'] != state['old_tool'] or state['recipe'] != state['old_recipe'] or state['failed'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 2', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 3', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 4', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 5', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 6', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 7', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 8', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 9', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 10', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 11', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 12', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 13', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 14', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 15', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 16', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 17', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), False)
check('fixture 18', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 19', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 20', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 21', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 22', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 23', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 24', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 25', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 26', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 27', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 28', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 29', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 30', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 31', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 32', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Content changes with unchanged metadata are missed.

Rebuild if output absent, content digest changed, tool changed, recipe changed, or a prerequisite failed; timestamps alone are irrelevant.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return (not state['exists'] or state['digest'] != state['old_digest'] or False or state['recipe'] != state['old_recipe'] or state['failed'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 2', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 3', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 4', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 5', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 6', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 7', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 8', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 9', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 10', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 11', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 12', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 13', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 14', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 15', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 16', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 17', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), False)
check('fixture 18', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 19', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 20', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 21', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 22', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 23', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 24', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 25', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 26', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 27', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 28', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 29', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 30', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 31', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 32', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rebuild

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return (not state['exists'] or state['digest'] != state['old_digest'] or state['tool'] == state['old_tool'] or state['recipe'] != state['old_recipe'] or state['failed'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 2', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 3', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 4', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 5', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 6', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 7', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 8', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 9', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 10', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 11', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 12', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 13', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 14', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 15', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 16', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 17', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), False)
check('fixture 18', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 19', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 20', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 21', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 22', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 23', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 24', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 25', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 26', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 27', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 28', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 29', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 30', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 31', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 32', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Compiler identity is absent from invalidation.

Rebuild if output absent, content digest changed, tool changed, recipe changed, or a prerequisite failed; timestamps alone are irrelevant.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return (not state['exists'] or state['digest'] != state['old_digest'] or state['tool'] != state['old_tool'] or False or state['failed'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 2', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 3', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 4', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 5', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 6', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 7', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 8', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 9', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 10', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 11', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 12', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 13', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 14', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 15', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 16', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 17', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), False)
check('fixture 18', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 19', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 20', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 21', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 22', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 23', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 24', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 25', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 26', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 27', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 28', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 29', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 30', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 31', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 32', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rebuild

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return (not state['exists'] or state['digest'] != state['old_digest'] or state['tool'] != state['old_tool'] or state['recipe'] == state['old_recipe'] or state['failed'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 2', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 3', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 4', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 5', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 6', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 7', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 8', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 9', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 10', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 11', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 12', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 13', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 14', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 15', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 16', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 17', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), False)
check('fixture 18', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 19', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 20', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 21', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 22', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 23', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 24', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 25', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 26', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 27', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 28', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 29', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 30', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 31', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 32', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Recipe changes reuse obsolete outputs.

Rebuild if output absent, content digest changed, tool changed, recipe changed, or a prerequisite failed; timestamps alone are irrelevant.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return (not state['exists'] or state['digest'] != state['old_digest'] or state['tool'] != state['old_tool'] or state['recipe'] != state['old_recipe'] or False)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 2', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 3', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 4', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 5', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 6', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 7', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 8', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 9', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 10', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 11', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 12', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 13', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 14', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 15', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 16', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 17', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), False)
check('fixture 18', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 19', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 20', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 21', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 22', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 23', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 24', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 25', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 26', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 27', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 28', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 29', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 30', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 31', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 32', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rebuild

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(state):
    return (not state['exists'] or state['digest'] != state['old_digest'] or state['tool'] != state['old_tool'] or state['recipe'] != state['old_recipe'] or not state['failed'])
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 2', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 3', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 4', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 5', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 6', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 7', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 8', solve({'exists': False, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 9', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 10', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 11', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 12', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 13', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 14', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 15', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 16', solve({'exists': False, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 17', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), False)
check('fixture 18', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 19', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 20', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 21', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 22', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 23', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 24', solve({'exists': True, 'digest': 'a', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 25', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 26', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 27', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 28', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc1', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 29', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 30', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r1', 'old_recipe': 'r1', 'failed': True}), True)
check('fixture 31', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': False}), True)
check('fixture 32', solve({'exists': True, 'digest': 'b', 'old_digest': 'a', 'tool': 'cc2', 'old_tool': 'cc1', 'recipe': 'r2', 'old_recipe': 'r1', 'failed': True}), True)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Failed prerequisites leave cached outputs eligible.

Rebuild if output absent, content digest changed, tool changed, recipe changed, or a prerequisite failed; timestamps alone are irrelevant.

split: open-access


evaluation_group: xt-cache-key

prompt:
Different compilers collide in the action key.

Return a tuple of compiler, ordered arguments, sorted environment pairs, sorted input digest pairs, and target platform; argument order and duplicate arguments matter.

split: open-access


evaluation_group: xt-cache-key

prompt:
Sorting arguments merges order-sensitive commands.

Return a tuple of compiler, ordered arguments, sorted environment pairs, sorted input digest pairs, and target platform; argument order and duplicate arguments matter.

split: open-access


evaluation_group: xt-cache-key

prompt:
Environment insertion order fragments the action key.

Return a tuple of compiler, ordered arguments, sorted environment pairs, sorted input digest pairs, and target platform; argument order and duplicate arguments matter.

split: open-access


evaluation_group: xt-cache-key

prompt:
Input names replace input content identities.

Return a tuple of compiler, ordered arguments, sorted environment pairs, sorted input digest pairs, and target platform; argument order and duplicate arguments matter.

split: open-access


evaluation_group: xt-cache-key

prompt:
Cross-platform actions collide.

Return a tuple of compiler, ordered arguments, sorted environment pairs, sorted input digest pairs, and target platform; argument order and duplicate arguments matter.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(roots, edges):
    seen = set()
    stack = list(roots[:1])
    while stack:
        node = stack.pop()
        if node in seen:
            continue
        seen.add(node)
        stack.extend(dep for dep, kind in edges.get(node, []) if kind == 'normal')
    return sorted(seen)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['a'], {}), ['a'])
check('fixture 3', solve(['a'], {'a': [('b', 'normal'), ('c', 'order')]}), ['a', 'b'])
check('fixture 4', solve(['a'], {'a': [('b', 'normal')], 'b': [('c', 'normal')]}), ['a', 'b', 'c'])
check('fixture 5', solve(['z', 'a'], {}), ['a', 'z'])
check('fixture 6', solve(['a', 'a'], {'a': [('b', 'normal')]}), ['a', 'b'])
check('fixture 7', solve(['a'], {'a': [('b', 'normal')], 'b': [('a', 'normal')]}), ['a', 'b'])
check('fixture 8', solve(['a'], {'a': [('b', 'order')]}), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-target-closure

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(roots, edges):
    seen = set()
    stack = list(roots[1:])
    while stack:
        node = stack.pop()
        if node in seen:
            continue
        seen.add(node)
        stack.extend(dep for dep, kind in edges.get(node, []) if kind == 'normal')
    return sorted(seen)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['a'], {}), ['a'])
check('fixture 3', solve(['a'], {'a': [('b', 'normal'), ('c', 'order')]}), ['a', 'b'])
check('fixture 4', solve(['a'], {'a': [('b', 'normal')], 'b': [('c', 'normal')]}), ['a', 'b', 'c'])
check('fixture 5', solve(['z', 'a'], {}), ['a', 'z'])
check('fixture 6', solve(['a', 'a'], {'a': [('b', 'normal')]}), ['a', 'b'])
check('fixture 7', solve(['a'], {'a': [('b', 'normal')], 'b': [('a', 'normal')]}), ['a', 'b'])
check('fixture 8', solve(['a'], {'a': [('b', 'order')]}), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Only the first requested target enters the closure.

Return sorted unique reachable target names including roots. Traverse normal prerequisites, exclude order-only prerequisites, retain unknown leaves, and stop cycles.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(roots, edges):
    seen = set()
    stack = list(roots)
    while stack:
        node = stack.pop()
        if node in seen:
            continue
        seen.add(node)
        stack.extend(dep for dep, kind in edges.get(node, []) if kind == 'normal')
    return sorted(seen - set(roots))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['a'], {}), ['a'])
check('fixture 3', solve(['a'], {'a': [('b', 'normal'), ('c', 'order')]}), ['a', 'b'])
check('fixture 4', solve(['a'], {'a': [('b', 'normal')], 'b': [('c', 'normal')]}), ['a', 'b', 'c'])
check('fixture 5', solve(['z', 'a'], {}), ['a', 'z'])
check('fixture 6', solve(['a', 'a'], {'a': [('b', 'normal')]}), ['a', 'b'])
check('fixture 7', solve(['a'], {'a': [('b', 'normal')], 'b': [('a', 'normal')]}), ['a', 'b'])
check('fixture 8', solve(['a'], {'a': [('b', 'order')]}), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-target-closure

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(roots, edges):
    seen = set()
    stack = list(roots)
    while stack:
        node = stack.pop()
        if node in seen:
            continue
        seen.add(node)
        stack.extend(dep for dep, kind in edges.get(node, []) if kind == 'normal')
    return sorted(set(roots))
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['a'], {}), ['a'])
check('fixture 3', solve(['a'], {'a': [('b', 'normal'), ('c', 'order')]}), ['a', 'b'])
check('fixture 4', solve(['a'], {'a': [('b', 'normal')], 'b': [('c', 'normal')]}), ['a', 'b', 'c'])
check('fixture 5', solve(['z', 'a'], {}), ['a', 'z'])
check('fixture 6', solve(['a', 'a'], {'a': [('b', 'normal')]}), ['a', 'b'])
check('fixture 7', solve(['a'], {'a': [('b', 'normal')], 'b': [('a', 'normal')]}), ['a', 'b'])
check('fixture 8', solve(['a'], {'a': [('b', 'order')]}), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Requested root targets disappear from results.

Return sorted unique reachable target names including roots. Traverse normal prerequisites, exclude order-only prerequisites, retain unknown leaves, and stop cycles.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(roots, edges):
    seen = set()
    stack = list(roots)
    while stack:
        node = stack.pop()
        if node in seen:
            continue
        seen.add(node)
        stack.extend(dep for dep, kind in edges.get(node, []) if True)
    return sorted(seen)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['a'], {}), ['a'])
check('fixture 3', solve(['a'], {'a': [('b', 'normal'), ('c', 'order')]}), ['a', 'b'])
check('fixture 4', solve(['a'], {'a': [('b', 'normal')], 'b': [('c', 'normal')]}), ['a', 'b', 'c'])
check('fixture 5', solve(['z', 'a'], {}), ['a', 'z'])
check('fixture 6', solve(['a', 'a'], {'a': [('b', 'normal')]}), ['a', 'b'])
check('fixture 7', solve(['a'], {'a': [('b', 'normal')], 'b': [('a', 'normal')]}), ['a', 'b'])
check('fixture 8', solve(['a'], {'a': [('b', 'order')]}), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-target-closure

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(roots, edges):
    seen = set()
    stack = list(roots)
    while stack:
        node = stack.pop()
        if node in seen:
            continue
        seen.add(node)
        stack.extend(dep for dep, kind in edges.get(node, []) if kind == 'order')
    return sorted(seen)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['a'], {}), ['a'])
check('fixture 3', solve(['a'], {'a': [('b', 'normal'), ('c', 'order')]}), ['a', 'b'])
check('fixture 4', solve(['a'], {'a': [('b', 'normal')], 'b': [('c', 'normal')]}), ['a', 'b', 'c'])
check('fixture 5', solve(['z', 'a'], {}), ['a', 'z'])
check('fixture 6', solve(['a', 'a'], {'a': [('b', 'normal')]}), ['a', 'b'])
check('fixture 7', solve(['a'], {'a': [('b', 'normal')], 'b': [('a', 'normal')]}), ['a', 'b'])
check('fixture 8', solve(['a'], {'a': [('b', 'order')]}), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Order-only edges contaminate content prerequisites.

Return sorted unique reachable target names including roots. Traverse normal prerequisites, exclude order-only prerequisites, retain unknown leaves, and stop cycles.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(roots, edges):
    seen = set()
    stack = list(roots)
    while stack:
        node = stack.pop()
        if node in seen:
            continue
        seen.update([node] if node in edges else [])
        if node not in edges:
            continue
        stack.extend(dep for dep, kind in edges.get(node, []) if kind == 'normal')
    return sorted(seen)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['a'], {}), ['a'])
check('fixture 3', solve(['a'], {'a': [('b', 'normal'), ('c', 'order')]}), ['a', 'b'])
check('fixture 4', solve(['a'], {'a': [('b', 'normal')], 'b': [('c', 'normal')]}), ['a', 'b', 'c'])
check('fixture 5', solve(['z', 'a'], {}), ['a', 'z'])
check('fixture 6', solve(['a', 'a'], {'a': [('b', 'normal')]}), ['a', 'b'])
check('fixture 7', solve(['a'], {'a': [('b', 'normal')], 'b': [('a', 'normal')]}), ['a', 'b'])
check('fixture 8', solve(['a'], {'a': [('b', 'order')]}), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-target-closure

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(roots, edges):
    seen = set()
    stack = list(roots)
    while stack:
        node = stack.pop()
        if node in seen:
            continue
        seen.add(node)
        if node in edges:
            continue
        stack.extend(dep for dep, kind in edges.get(node, []) if kind == 'normal')
    return sorted(seen)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['a'], {}), ['a'])
check('fixture 3', solve(['a'], {'a': [('b', 'normal'), ('c', 'order')]}), ['a', 'b'])
check('fixture 4', solve(['a'], {'a': [('b', 'normal')], 'b': [('c', 'normal')]}), ['a', 'b', 'c'])
check('fixture 5', solve(['z', 'a'], {}), ['a', 'z'])
check('fixture 6', solve(['a', 'a'], {'a': [('b', 'normal')]}), ['a', 'b'])
check('fixture 7', solve(['a'], {'a': [('b', 'normal')], 'b': [('a', 'normal')]}), ['a', 'b'])
check('fixture 8', solve(['a'], {'a': [('b', 'order')]}), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Undeclared leaf targets disappear.

Return sorted unique reachable target names including roots. Traverse normal prerequisites, exclude order-only prerequisites, retain unknown leaves, and stop cycles.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(roots, edges):
    seen = set()
    stack = list(roots)
    while stack:
        node = stack.pop()
        if node in seen:
            continue
        seen.add(node)
        stack.extend(dep for dep, kind in edges.get(node, []) if kind == 'normal')
    return sorted(seen, reverse=True)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['a'], {}), ['a'])
check('fixture 3', solve(['a'], {'a': [('b', 'normal'), ('c', 'order')]}), ['a', 'b'])
check('fixture 4', solve(['a'], {'a': [('b', 'normal')], 'b': [('c', 'normal')]}), ['a', 'b', 'c'])
check('fixture 5', solve(['z', 'a'], {}), ['a', 'z'])
check('fixture 6', solve(['a', 'a'], {'a': [('b', 'normal')]}), ['a', 'b'])
check('fixture 7', solve(['a'], {'a': [('b', 'normal')], 'b': [('a', 'normal')]}), ['a', 'b'])
check('fixture 8', solve(['a'], {'a': [('b', 'order')]}), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-target-closure

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(roots, edges):
    seen = set()
    stack = list(roots)
    while stack:
        node = stack.pop()
        if node in seen:
            continue
        seen.add(node)
        stack.extend(dep for dep, kind in edges.get(node, []) if kind == 'normal')
    return sorted(seen, key=lambda x: (-len(x), x), reverse=True)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], {}), [])
check('fixture 2', solve(['a'], {}), ['a'])
check('fixture 3', solve(['a'], {'a': [('b', 'normal'), ('c', 'order')]}), ['a', 'b'])
check('fixture 4', solve(['a'], {'a': [('b', 'normal')], 'b': [('c', 'normal')]}), ['a', 'b', 'c'])
check('fixture 5', solve(['z', 'a'], {}), ['a', 'z'])
check('fixture 6', solve(['a', 'a'], {'a': [('b', 'normal')]}), ['a', 'b'])
check('fixture 7', solve(['a'], {'a': [('b', 'normal')], 'b': [('a', 'normal')]}), ['a', 'b'])
check('fixture 8', solve(['a'], {'a': [('b', 'order')]}), ['a'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Closure output follows reverse lexical order.

Return sorted unique reachable target names including roots. Traverse normal prerequisites, exclude order-only prerequisites, retain unknown leaves, and stop cycles.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(objects, archives, frameworks, directories, output):
    result = sorted(objects)
    result += list(archives)
    result += [part for name in frameworks for part in ['-framework', name]]
    result += [part for name in directories for part in ['-L', name]]
    result += ['-o', output]
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], 'app'), ['-o', 'app'])
check('fixture 2', solve(['b.o', 'a.o'], ['x.a', 'y.a', 'x.a'], ['Core', 'UI'], ['/lib a', '/lib b'], 'my app'), ['b.o', 'a.o', 'x.a', 'y.a', 'x.a', '-framework', 'Core', '-framework', 'UI', '-L', '/lib a', '-L', '/lib b', '-o', 'my app'])
check('fixture 3', solve(['a.o'], ['b.a'], [], [], 'x'), ['a.o', 'b.a', '-o', 'x'])
check('fixture 4', solve([], [], ['Core'], [], 'x'), ['-framework', 'Core', '-o', 'x'])
check('fixture 5', solve([], [], [], ['space dir'], 'x'), ['-L', 'space dir', '-o', 'x'])
check('fixture 6', solve(['z.o'], [], ['A'], ['D'], 'z'), ['z.o', '-framework', 'A', '-L', 'D', '-o', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-link-line

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(objects, archives, frameworks, directories, output):
    result = list(reversed(objects))
    result += list(archives)
    result += [part for name in frameworks for part in ['-framework', name]]
    result += [part for name in directories for part in ['-L', name]]
    result += ['-o', output]
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], 'app'), ['-o', 'app'])
check('fixture 2', solve(['b.o', 'a.o'], ['x.a', 'y.a', 'x.a'], ['Core', 'UI'], ['/lib a', '/lib b'], 'my app'), ['b.o', 'a.o', 'x.a', 'y.a', 'x.a', '-framework', 'Core', '-framework', 'UI', '-L', '/lib a', '-L', '/lib b', '-o', 'my app'])
check('fixture 3', solve(['a.o'], ['b.a'], [], [], 'x'), ['a.o', 'b.a', '-o', 'x'])
check('fixture 4', solve([], [], ['Core'], [], 'x'), ['-framework', 'Core', '-o', 'x'])
check('fixture 5', solve([], [], [], ['space dir'], 'x'), ['-L', 'space dir', '-o', 'x'])
check('fixture 6', solve(['z.o'], [], ['A'], ['D'], 'z'), ['z.o', '-framework', 'A', '-L', 'D', '-o', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Object order is alphabetized.

Emit object arguments first, then each archive in declared order preserving repeats, then frameworks as paired flags, then search directories as paired flags, then the output pair.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(objects, archives, frameworks, directories, output):
    result = list(objects)
    result += list(dict.fromkeys(archives))
    result += [part for name in frameworks for part in ['-framework', name]]
    result += [part for name in directories for part in ['-L', name]]
    result += ['-o', output]
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], 'app'), ['-o', 'app'])
check('fixture 2', solve(['b.o', 'a.o'], ['x.a', 'y.a', 'x.a'], ['Core', 'UI'], ['/lib a', '/lib b'], 'my app'), ['b.o', 'a.o', 'x.a', 'y.a', 'x.a', '-framework', 'Core', '-framework', 'UI', '-L', '/lib a', '-L', '/lib b', '-o', 'my app'])
check('fixture 3', solve(['a.o'], ['b.a'], [], [], 'x'), ['a.o', 'b.a', '-o', 'x'])
check('fixture 4', solve([], [], ['Core'], [], 'x'), ['-framework', 'Core', '-o', 'x'])
check('fixture 5', solve([], [], [], ['space dir'], 'x'), ['-L', 'space dir', '-o', 'x'])
check('fixture 6', solve(['z.o'], [], ['A'], ['D'], 'z'), ['z.o', '-framework', 'A', '-L', 'D', '-o', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-link-line

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(objects, archives, frameworks, directories, output):
    result = list(objects)
    result += sorted(set(archives))
    result += [part for name in frameworks for part in ['-framework', name]]
    result += [part for name in directories for part in ['-L', name]]
    result += ['-o', output]
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], 'app'), ['-o', 'app'])
check('fixture 2', solve(['b.o', 'a.o'], ['x.a', 'y.a', 'x.a'], ['Core', 'UI'], ['/lib a', '/lib b'], 'my app'), ['b.o', 'a.o', 'x.a', 'y.a', 'x.a', '-framework', 'Core', '-framework', 'UI', '-L', '/lib a', '-L', '/lib b', '-o', 'my app'])
check('fixture 3', solve(['a.o'], ['b.a'], [], [], 'x'), ['a.o', 'b.a', '-o', 'x'])
check('fixture 4', solve([], [], ['Core'], [], 'x'), ['-framework', 'Core', '-o', 'x'])
check('fixture 5', solve([], [], [], ['space dir'], 'x'), ['-L', 'space dir', '-o', 'x'])
check('fixture 6', solve(['z.o'], [], ['A'], ['D'], 'z'), ['z.o', '-framework', 'A', '-L', 'D', '-o', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Repeated archives are dropped.

Emit object arguments first, then each archive in declared order preserving repeats, then frameworks as paired flags, then search directories as paired flags, then the output pair.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(objects, archives, frameworks, directories, output):
    result = list(objects)
    result += list(archives)
    result += [part for name in frameworks for part in [name]]
    result += [part for name in directories for part in ['-L', name]]
    result += ['-o', output]
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], 'app'), ['-o', 'app'])
check('fixture 2', solve(['b.o', 'a.o'], ['x.a', 'y.a', 'x.a'], ['Core', 'UI'], ['/lib a', '/lib b'], 'my app'), ['b.o', 'a.o', 'x.a', 'y.a', 'x.a', '-framework', 'Core', '-framework', 'UI', '-L', '/lib a', '-L', '/lib b', '-o', 'my app'])
check('fixture 3', solve(['a.o'], ['b.a'], [], [], 'x'), ['a.o', 'b.a', '-o', 'x'])
check('fixture 4', solve([], [], ['Core'], [], 'x'), ['-framework', 'Core', '-o', 'x'])
check('fixture 5', solve([], [], [], ['space dir'], 'x'), ['-L', 'space dir', '-o', 'x'])
check('fixture 6', solve(['z.o'], [], ['A'], ['D'], 'z'), ['z.o', '-framework', 'A', '-L', 'D', '-o', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-link-line

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(objects, archives, frameworks, directories, output):
    result = list(objects)
    result += list(archives)
    result += [part for name in frameworks for part in [name, '-framework']]
    result += [part for name in directories for part in ['-L', name]]
    result += ['-o', output]
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], 'app'), ['-o', 'app'])
check('fixture 2', solve(['b.o', 'a.o'], ['x.a', 'y.a', 'x.a'], ['Core', 'UI'], ['/lib a', '/lib b'], 'my app'), ['b.o', 'a.o', 'x.a', 'y.a', 'x.a', '-framework', 'Core', '-framework', 'UI', '-L', '/lib a', '-L', '/lib b', '-o', 'my app'])
check('fixture 3', solve(['a.o'], ['b.a'], [], [], 'x'), ['a.o', 'b.a', '-o', 'x'])
check('fixture 4', solve([], [], ['Core'], [], 'x'), ['-framework', 'Core', '-o', 'x'])
check('fixture 5', solve([], [], [], ['space dir'], 'x'), ['-L', 'space dir', '-o', 'x'])
check('fixture 6', solve(['z.o'], [], ['A'], ['D'], 'z'), ['z.o', '-framework', 'A', '-L', 'D', '-o', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A framework flag is omitted.

Emit object arguments first, then each archive in declared order preserving repeats, then frameworks as paired flags, then search directories as paired flags, then the output pair.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(objects, archives, frameworks, directories, output):
    result = list(objects)
    result += list(archives)
    result += [part for name in frameworks for part in ['-framework', name]]
    result += [part for name in directories for part in ['-L' + name]]
    result += ['-o', output]
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], 'app'), ['-o', 'app'])
check('fixture 2', solve(['b.o', 'a.o'], ['x.a', 'y.a', 'x.a'], ['Core', 'UI'], ['/lib a', '/lib b'], 'my app'), ['b.o', 'a.o', 'x.a', 'y.a', 'x.a', '-framework', 'Core', '-framework', 'UI', '-L', '/lib a', '-L', '/lib b', '-o', 'my app'])
check('fixture 3', solve(['a.o'], ['b.a'], [], [], 'x'), ['a.o', 'b.a', '-o', 'x'])
check('fixture 4', solve([], [], ['Core'], [], 'x'), ['-framework', 'Core', '-o', 'x'])
check('fixture 5', solve([], [], [], ['space dir'], 'x'), ['-L', 'space dir', '-o', 'x'])
check('fixture 6', solve(['z.o'], [], ['A'], ['D'], 'z'), ['z.o', '-framework', 'A', '-L', 'D', '-o', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-link-line

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(objects, archives, frameworks, directories, output):
    result = list(objects)
    result += list(archives)
    result += [part for name in frameworks for part in ['-framework', name]]
    result += [part for name in directories for part in [name]]
    result += ['-o', output]
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], 'app'), ['-o', 'app'])
check('fixture 2', solve(['b.o', 'a.o'], ['x.a', 'y.a', 'x.a'], ['Core', 'UI'], ['/lib a', '/lib b'], 'my app'), ['b.o', 'a.o', 'x.a', 'y.a', 'x.a', '-framework', 'Core', '-framework', 'UI', '-L', '/lib a', '-L', '/lib b', '-o', 'my app'])
check('fixture 3', solve(['a.o'], ['b.a'], [], [], 'x'), ['a.o', 'b.a', '-o', 'x'])
check('fixture 4', solve([], [], ['Core'], [], 'x'), ['-framework', 'Core', '-o', 'x'])
check('fixture 5', solve([], [], [], ['space dir'], 'x'), ['-L', 'space dir', '-o', 'x'])
check('fixture 6', solve(['z.o'], [], ['A'], ['D'], 'z'), ['z.o', '-framework', 'A', '-L', 'D', '-o', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Search path arguments are collapsed.

Emit object arguments first, then each archive in declared order preserving repeats, then frameworks as paired flags, then search directories as paired flags, then the output pair.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(objects, archives, frameworks, directories, output):
    result = list(objects)
    result += list(archives)
    result += [part for name in frameworks for part in ['-framework', name]]
    result += [part for name in directories for part in ['-L', name]]
    result += [output]
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], 'app'), ['-o', 'app'])
check('fixture 2', solve(['b.o', 'a.o'], ['x.a', 'y.a', 'x.a'], ['Core', 'UI'], ['/lib a', '/lib b'], 'my app'), ['b.o', 'a.o', 'x.a', 'y.a', 'x.a', '-framework', 'Core', '-framework', 'UI', '-L', '/lib a', '-L', '/lib b', '-o', 'my app'])
check('fixture 3', solve(['a.o'], ['b.a'], [], [], 'x'), ['a.o', 'b.a', '-o', 'x'])
check('fixture 4', solve([], [], ['Core'], [], 'x'), ['-framework', 'Core', '-o', 'x'])
check('fixture 5', solve([], [], [], ['space dir'], 'x'), ['-L', 'space dir', '-o', 'x'])
check('fixture 6', solve(['z.o'], [], ['A'], ['D'], 'z'), ['z.o', '-framework', 'A', '-L', 'D', '-o', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-link-line

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(objects, archives, frameworks, directories, output):
    result = list(objects)
    result += list(archives)
    result += [part for name in frameworks for part in ['-framework', name]]
    result += [part for name in directories for part in ['-L', name]]
    result += [output, '-o']
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], [], [], [], 'app'), ['-o', 'app'])
check('fixture 2', solve(['b.o', 'a.o'], ['x.a', 'y.a', 'x.a'], ['Core', 'UI'], ['/lib a', '/lib b'], 'my app'), ['b.o', 'a.o', 'x.a', 'y.a', 'x.a', '-framework', 'Core', '-framework', 'UI', '-L', '/lib a', '-L', '/lib b', '-o', 'my app'])
check('fixture 3', solve(['a.o'], ['b.a'], [], [], 'x'), ['a.o', 'b.a', '-o', 'x'])
check('fixture 4', solve([], [], ['Core'], [], 'x'), ['-framework', 'Core', '-o', 'x'])
check('fixture 5', solve([], [], [], ['space dir'], 'x'), ['-L', 'space dir', '-o', 'x'])
check('fixture 6', solve(['z.o'], [], ['A'], ['D'], 'z'), ['z.o', '-framework', 'A', '-L', 'D', '-o', 'z'])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Output names lose the output selector.

Emit object arguments first, then each archive in declared order preserving repeats, then frameworks as paired flags, then search directories as paired flags, then the output pair.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(previous, updates):
    result = {}
    for target, deps in updates:
        result.setdefault(target, [])
        for dep in deps:
            if dep != target and dep not in result[target]:
                result[target].append(dep)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': ['b']}, [('a', ['c'])]), {'a': ['b', 'c']})
check('fixture 3', solve({}, [('a', ['a', 'b', 'b', 'c', 'b'])]), {'a': ['b', 'c']})
check('fixture 4', solve({}, [('z', [])]), {'z': []})
check('fixture 5', solve({'a': ['b']}, [('a', ['b', 'c']), ('a', ['d'])]), {'a': ['b', 'c', 'd']})
check('fixture 6', solve({'old': ['x']}, [('new', ['z', 'a'])]), {'old': ['x'], 'new': ['z', 'a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-depfile-merge

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(previous, updates):
    result = {name: [] for name in previous}
    for target, deps in updates:
        result.setdefault(target, [])
        for dep in deps:
            if dep != target and dep not in result[target]:
                result[target].append(dep)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': ['b']}, [('a', ['c'])]), {'a': ['b', 'c']})
check('fixture 3', solve({}, [('a', ['a', 'b', 'b', 'c', 'b'])]), {'a': ['b', 'c']})
check('fixture 4', solve({}, [('z', [])]), {'z': []})
check('fixture 5', solve({'a': ['b']}, [('a', ['b', 'c']), ('a', ['d'])]), {'a': ['b', 'c', 'd']})
check('fixture 6', solve({'old': ['x']}, [('new', ['z', 'a'])]), {'old': ['x'], 'new': ['z', 'a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Existing targets are erased during updates.

Merge prerequisite lists by target, preserving first occurrence order, retaining previous prerequisites, ignoring self references, and retaining empty targets.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(previous, updates):
    result = {name: list(deps) for name, deps in previous.items()}
    for target, deps in updates:
        result[target] = []
        for dep in deps:
            if dep != target and dep not in result[target]:
                result[target].append(dep)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': ['b']}, [('a', ['c'])]), {'a': ['b', 'c']})
check('fixture 3', solve({}, [('a', ['a', 'b', 'b', 'c', 'b'])]), {'a': ['b', 'c']})
check('fixture 4', solve({}, [('z', [])]), {'z': []})
check('fixture 5', solve({'a': ['b']}, [('a', ['b', 'c']), ('a', ['d'])]), {'a': ['b', 'c', 'd']})
check('fixture 6', solve({'old': ['x']}, [('new', ['z', 'a'])]), {'old': ['x'], 'new': ['z', 'a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-depfile-merge

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(previous, updates):
    result = {name: list(deps) for name, deps in previous.items()}
    for target, deps in updates:
        result[target] = list(result.get(target, []))[-1:]
        for dep in deps:
            if dep != target and dep not in result[target]:
                result[target].append(dep)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, []), {})
check('fixture 2', solve({'a': ['b']}, [('a', ['c'])]), {'a': ['b', 'c']})
check('fixture 3', solve({}, [('a', ['a', 'b', 'b', 'c', 'b'])]), {'a': ['b', 'c']})
check('fixture 4', solve({}, [('z', [])]), {'z': []})
check('fixture 5', solve({'a': ['b']}, [('a', ['b', 'c']), ('a', ['d'])]), {'a': ['b', 'c', 'd']})
check('fixture 6', solve({'old': ['x']}, [('new', ['z', 'a'])]), {'old': ['x'], 'new': ['z', 'a']})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Later rules reset an earlier target list.

Merge prerequisite lists by target, preserving first occurrence order, retaining previous prerequisites, ignoring self references, and retaining empty targets.

split: open-access