broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(valid, failed, observed, generation, updated, available, desired):
    if False: return 'blocked'
    if failed: return 'degraded'
    if observed < generation: return 'progressing'
    if updated < desired: return 'progressing'
    if available < desired: return 'progressing'
    return 'ready'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, False, 1, 1, 1, 1, 1), 'blocked')
check('fixture 2', solve(True, True, 1, 1, 1, 1, 1), 'degraded')
check('fixture 3', solve(True, False, 0, 1, 1, 1, 1), 'progressing')
check('fixture 4', solve(True, False, 1, 1, 0, 1, 1), 'progressing')
check('fixture 5', solve(True, False, 1, 1, 1, 0, 1), 'progressing')
check('fixture 6', solve(True, False, 1, 1, 1, 1, 1), 'ready')
check('fixture 7', solve(True, False, 1, 1, 0, 0, 0), 'ready')
check('fixture 8', solve(True, False, 2, 1, 2, 2, 1), 'ready')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollout-summary

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(valid, failed, observed, generation, updated, available, desired):
    if valid: return 'blocked'
    if failed: return 'degraded'
    if observed < generation: return 'progressing'
    if updated < desired: return 'progressing'
    if available < desired: return 'progressing'
    return 'ready'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, False, 1, 1, 1, 1, 1), 'blocked')
check('fixture 2', solve(True, True, 1, 1, 1, 1, 1), 'degraded')
check('fixture 3', solve(True, False, 0, 1, 1, 1, 1), 'progressing')
check('fixture 4', solve(True, False, 1, 1, 0, 1, 1), 'progressing')
check('fixture 5', solve(True, False, 1, 1, 1, 0, 1), 'progressing')
check('fixture 6', solve(True, False, 1, 1, 1, 1, 1), 'ready')
check('fixture 7', solve(True, False, 1, 1, 0, 0, 0), 'ready')
check('fixture 8', solve(True, False, 2, 1, 2, 2, 1), 'ready')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Invalid specifications appear ready.

Report blocked on invalid specification, degraded on failed required replicas, progressing when observed generation lags, progressing when updated or available counts are below desired, otherwise ready. Zero desired is ready when observed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(valid, failed, observed, generation, updated, available, desired):
    if not valid: return 'blocked'
    if False: return 'degraded'
    if observed < generation: return 'progressing'
    if updated < desired: return 'progressing'
    if available < desired: return 'progressing'
    return 'ready'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, False, 1, 1, 1, 1, 1), 'blocked')
check('fixture 2', solve(True, True, 1, 1, 1, 1, 1), 'degraded')
check('fixture 3', solve(True, False, 0, 1, 1, 1, 1), 'progressing')
check('fixture 4', solve(True, False, 1, 1, 0, 1, 1), 'progressing')
check('fixture 5', solve(True, False, 1, 1, 1, 0, 1), 'progressing')
check('fixture 6', solve(True, False, 1, 1, 1, 1, 1), 'ready')
check('fixture 7', solve(True, False, 1, 1, 0, 0, 0), 'ready')
check('fixture 8', solve(True, False, 2, 1, 2, 2, 1), 'ready')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

meaning: Fixture-count band, not measured model difficulty

deps: stdlib

evaluation_group: xt-rollout-summary

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(valid, failed, observed, generation, updated, available, desired):
    if not valid: return 'blocked'
    if not failed: return 'degraded'
    if observed < generation: return 'progressing'
    if updated < desired: return 'progressing'
    if available < desired: return 'progressing'
    return 'ready'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, False, 1, 1, 1, 1, 1), 'blocked')
check('fixture 2', solve(True, True, 1, 1, 1, 1, 1), 'degraded')
check('fixture 3', solve(True, False, 0, 1, 1, 1, 1), 'progressing')
check('fixture 4', solve(True, False, 1, 1, 0, 1, 1), 'progressing')
check('fixture 5', solve(True, False, 1, 1, 1, 0, 1), 'progressing')
check('fixture 6', solve(True, False, 1, 1, 1, 1, 1), 'ready')
check('fixture 7', solve(True, False, 1, 1, 0, 0, 0), 'ready')
check('fixture 8', solve(True, False, 2, 1, 2, 2, 1), 'ready')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Failed replicas do not degrade rollout.

Report blocked on invalid specification, degraded on failed required replicas, progressing when observed generation lags, progressing when updated or available counts are below desired, otherwise ready. Zero desired is ready when observed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(valid, failed, observed, generation, updated, available, desired):
    if not valid: return 'blocked'
    if failed: return 'degraded'
    if False: return 'progressing'
    if updated < desired: return 'progressing'
    if available < desired: return 'progressing'
    return 'ready'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, False, 1, 1, 1, 1, 1), 'blocked')
check('fixture 2', solve(True, True, 1, 1, 1, 1, 1), 'degraded')
check('fixture 3', solve(True, False, 0, 1, 1, 1, 1), 'progressing')
check('fixture 4', solve(True, False, 1, 1, 0, 1, 1), 'progressing')
check('fixture 5', solve(True, False, 1, 1, 1, 0, 1), 'progressing')
check('fixture 6', solve(True, False, 1, 1, 1, 1, 1), 'ready')
check('fixture 7', solve(True, False, 1, 1, 0, 0, 0), 'ready')
check('fixture 8', solve(True, False, 2, 1, 2, 2, 1), 'ready')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollout-summary

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(valid, failed, observed, generation, updated, available, desired):
    if not valid: return 'blocked'
    if failed: return 'degraded'
    if observed <= generation: return 'progressing'
    if updated < desired: return 'progressing'
    if available < desired: return 'progressing'
    return 'ready'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, False, 1, 1, 1, 1, 1), 'blocked')
check('fixture 2', solve(True, True, 1, 1, 1, 1, 1), 'degraded')
check('fixture 3', solve(True, False, 0, 1, 1, 1, 1), 'progressing')
check('fixture 4', solve(True, False, 1, 1, 0, 1, 1), 'progressing')
check('fixture 5', solve(True, False, 1, 1, 1, 0, 1), 'progressing')
check('fixture 6', solve(True, False, 1, 1, 1, 1, 1), 'ready')
check('fixture 7', solve(True, False, 1, 1, 0, 0, 0), 'ready')
check('fixture 8', solve(True, False, 2, 1, 2, 2, 1), 'ready')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Stale controller observations appear current.

Report blocked on invalid specification, degraded on failed required replicas, progressing when observed generation lags, progressing when updated or available counts are below desired, otherwise ready. Zero desired is ready when observed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(valid, failed, observed, generation, updated, available, desired):
    if not valid: return 'blocked'
    if failed: return 'degraded'
    if observed < generation: return 'progressing'
    if False: return 'progressing'
    if available < desired: return 'progressing'
    return 'ready'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, False, 1, 1, 1, 1, 1), 'blocked')
check('fixture 2', solve(True, True, 1, 1, 1, 1, 1), 'degraded')
check('fixture 3', solve(True, False, 0, 1, 1, 1, 1), 'progressing')
check('fixture 4', solve(True, False, 1, 1, 0, 1, 1), 'progressing')
check('fixture 5', solve(True, False, 1, 1, 1, 0, 1), 'progressing')
check('fixture 6', solve(True, False, 1, 1, 1, 1, 1), 'ready')
check('fixture 7', solve(True, False, 1, 1, 0, 0, 0), 'ready')
check('fixture 8', solve(True, False, 2, 1, 2, 2, 1), 'ready')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollout-summary

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(valid, failed, observed, generation, updated, available, desired):
    if not valid: return 'blocked'
    if failed: return 'degraded'
    if observed < generation: return 'progressing'
    if updated <= desired: return 'progressing'
    if available < desired: return 'progressing'
    return 'ready'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, False, 1, 1, 1, 1, 1), 'blocked')
check('fixture 2', solve(True, True, 1, 1, 1, 1, 1), 'degraded')
check('fixture 3', solve(True, False, 0, 1, 1, 1, 1), 'progressing')
check('fixture 4', solve(True, False, 1, 1, 0, 1, 1), 'progressing')
check('fixture 5', solve(True, False, 1, 1, 1, 0, 1), 'progressing')
check('fixture 6', solve(True, False, 1, 1, 1, 1, 1), 'ready')
check('fixture 7', solve(True, False, 1, 1, 0, 0, 0), 'ready')
check('fixture 8', solve(True, False, 2, 1, 2, 2, 1), 'ready')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Old-revision replicas count as updated.

Report blocked on invalid specification, degraded on failed required replicas, progressing when observed generation lags, progressing when updated or available counts are below desired, otherwise ready. Zero desired is ready when observed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(valid, failed, observed, generation, updated, available, desired):
    if not valid: return 'blocked'
    if failed: return 'degraded'
    if observed < generation: return 'progressing'
    if updated < desired: return 'progressing'
    if False: return 'progressing'
    return 'ready'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, False, 1, 1, 1, 1, 1), 'blocked')
check('fixture 2', solve(True, True, 1, 1, 1, 1, 1), 'degraded')
check('fixture 3', solve(True, False, 0, 1, 1, 1, 1), 'progressing')
check('fixture 4', solve(True, False, 1, 1, 0, 1, 1), 'progressing')
check('fixture 5', solve(True, False, 1, 1, 1, 0, 1), 'progressing')
check('fixture 6', solve(True, False, 1, 1, 1, 1, 1), 'ready')
check('fixture 7', solve(True, False, 1, 1, 0, 0, 0), 'ready')
check('fixture 8', solve(True, False, 2, 1, 2, 2, 1), 'ready')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollout-summary

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(valid, failed, observed, generation, updated, available, desired):
    if not valid: return 'blocked'
    if failed: return 'degraded'
    if observed < generation: return 'progressing'
    if updated < desired: return 'progressing'
    if available <= desired: return 'progressing'
    return 'ready'
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve(False, False, 1, 1, 1, 1, 1), 'blocked')
check('fixture 2', solve(True, True, 1, 1, 1, 1, 1), 'degraded')
check('fixture 3', solve(True, False, 0, 1, 1, 1, 1), 'progressing')
check('fixture 4', solve(True, False, 1, 1, 0, 1, 1), 'progressing')
check('fixture 5', solve(True, False, 1, 1, 1, 0, 1), 'progressing')
check('fixture 6', solve(True, False, 1, 1, 1, 1, 1), 'ready')
check('fixture 7', solve(True, False, 1, 1, 0, 0, 0), 'ready')
check('fixture 8', solve(True, False, 2, 1, 2, 2, 1), 'ready')
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unavailable replicas count toward readiness.

Report blocked on invalid specification, degraded on failed required replicas, progressing when observed generation lags, progressing when updated or available counts are below desired, otherwise ready. Zero desired is ready when observed.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(revisions, active, schema):
    candidates = [r for r in revisions if True]
    candidates = [r for r in candidates if r['success']]
    candidates = [r for r in candidates if r['retained']]
    candidates = [r for r in candidates if schema in r['schemas']]
    return max((r['number'] for r in candidates), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 6, 1), None)
check('fixture 2', solve([], 7, 1), None)
check('fixture 3', solve([], 3, 1), None)
check('fixture 4', solve([], 6, 2), None)
check('fixture 5', solve([], 1, 1), None)
check('fixture 6', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 1), 2)
check('fixture 7', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 7, 1), 6)
check('fixture 8', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 3, 1), 2)
check('fixture 9', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 2), 5)
check('fixture 10', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 1, 1), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollback-choice

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(revisions, active, schema):
    candidates = [r for r in revisions if r['number'] <= active]
    candidates = [r for r in candidates if r['success']]
    candidates = [r for r in candidates if r['retained']]
    candidates = [r for r in candidates if schema in r['schemas']]
    return max((r['number'] for r in candidates), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 6, 1), None)
check('fixture 2', solve([], 7, 1), None)
check('fixture 3', solve([], 3, 1), None)
check('fixture 4', solve([], 6, 2), None)
check('fixture 5', solve([], 1, 1), None)
check('fixture 6', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 1), 2)
check('fixture 7', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 7, 1), 6)
check('fixture 8', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 3, 1), 2)
check('fixture 9', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 2), 5)
check('fixture 10', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 1, 1), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The active or a future revision is selected for rollback.

Choose the highest earlier revision that is successful, retained, compatible with current schema, and different from the active revision. Return None when no rollback target exists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(revisions, active, schema):
    candidates = [r for r in revisions if r['number'] < active]
    candidates = [r for r in candidates if True]
    candidates = [r for r in candidates if r['retained']]
    candidates = [r for r in candidates if schema in r['schemas']]
    return max((r['number'] for r in candidates), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 6, 1), None)
check('fixture 2', solve([], 7, 1), None)
check('fixture 3', solve([], 3, 1), None)
check('fixture 4', solve([], 6, 2), None)
check('fixture 5', solve([], 1, 1), None)
check('fixture 6', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 1), 2)
check('fixture 7', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 7, 1), 6)
check('fixture 8', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 3, 1), 2)
check('fixture 9', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 2), 5)
check('fixture 10', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 1, 1), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollback-choice

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(revisions, active, schema):
    candidates = [r for r in revisions if r['number'] < active]
    candidates = [r for r in candidates if not r['success']]
    candidates = [r for r in candidates if r['retained']]
    candidates = [r for r in candidates if schema in r['schemas']]
    return max((r['number'] for r in candidates), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 6, 1), None)
check('fixture 2', solve([], 7, 1), None)
check('fixture 3', solve([], 3, 1), None)
check('fixture 4', solve([], 6, 2), None)
check('fixture 5', solve([], 1, 1), None)
check('fixture 6', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 1), 2)
check('fixture 7', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 7, 1), 6)
check('fixture 8', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 3, 1), 2)
check('fixture 9', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 2), 5)
check('fixture 10', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 1, 1), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A failed release becomes the rollback target.

Choose the highest earlier revision that is successful, retained, compatible with current schema, and different from the active revision. Return None when no rollback target exists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(revisions, active, schema):
    candidates = [r for r in revisions if r['number'] < active]
    candidates = [r for r in candidates if r['success']]
    candidates = [r for r in candidates if True]
    candidates = [r for r in candidates if schema in r['schemas']]
    return max((r['number'] for r in candidates), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 6, 1), None)
check('fixture 2', solve([], 7, 1), None)
check('fixture 3', solve([], 3, 1), None)
check('fixture 4', solve([], 6, 2), None)
check('fixture 5', solve([], 1, 1), None)
check('fixture 6', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 1), 2)
check('fixture 7', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 7, 1), 6)
check('fixture 8', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 3, 1), 2)
check('fixture 9', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 2), 5)
check('fixture 10', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 1, 1), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollback-choice

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(revisions, active, schema):
    candidates = [r for r in revisions if r['number'] < active]
    candidates = [r for r in candidates if r['success']]
    candidates = [r for r in candidates if not r['retained']]
    candidates = [r for r in candidates if schema in r['schemas']]
    return max((r['number'] for r in candidates), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 6, 1), None)
check('fixture 2', solve([], 7, 1), None)
check('fixture 3', solve([], 3, 1), None)
check('fixture 4', solve([], 6, 2), None)
check('fixture 5', solve([], 1, 1), None)
check('fixture 6', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 1), 2)
check('fixture 7', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 7, 1), 6)
check('fixture 8', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 3, 1), 2)
check('fixture 9', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 2), 5)
check('fixture 10', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 1, 1), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A garbage-collected revision is selected.

Choose the highest earlier revision that is successful, retained, compatible with current schema, and different from the active revision. Return None when no rollback target exists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(revisions, active, schema):
    candidates = [r for r in revisions if r['number'] < active]
    candidates = [r for r in candidates if r['success']]
    candidates = [r for r in candidates if r['retained']]
    candidates = [r for r in candidates if True]
    return max((r['number'] for r in candidates), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 6, 1), None)
check('fixture 2', solve([], 7, 1), None)
check('fixture 3', solve([], 3, 1), None)
check('fixture 4', solve([], 6, 2), None)
check('fixture 5', solve([], 1, 1), None)
check('fixture 6', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 1), 2)
check('fixture 7', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 7, 1), 6)
check('fixture 8', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 3, 1), 2)
check('fixture 9', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 2), 5)
check('fixture 10', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 1, 1), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollback-choice

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(revisions, active, schema):
    candidates = [r for r in revisions if r['number'] < active]
    candidates = [r for r in candidates if r['success']]
    candidates = [r for r in candidates if r['retained']]
    candidates = [r for r in candidates if schema not in r['schemas']]
    return max((r['number'] for r in candidates), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 6, 1), None)
check('fixture 2', solve([], 7, 1), None)
check('fixture 3', solve([], 3, 1), None)
check('fixture 4', solve([], 6, 2), None)
check('fixture 5', solve([], 1, 1), None)
check('fixture 6', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 1), 2)
check('fixture 7', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 7, 1), 6)
check('fixture 8', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 3, 1), 2)
check('fixture 9', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 2), 5)
check('fixture 10', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 1, 1), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Rollback chooses a revision incompatible with current data.

Choose the highest earlier revision that is successful, retained, compatible with current schema, and different from the active revision. Return None when no rollback target exists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(revisions, active, schema):
    candidates = [r for r in revisions if r['number'] < active]
    candidates = [r for r in candidates if r['success']]
    candidates = [r for r in candidates if r['retained']]
    candidates = [r for r in candidates if schema in r['schemas']]
    return min((r['number'] for r in candidates), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 6, 1), None)
check('fixture 2', solve([], 7, 1), None)
check('fixture 3', solve([], 3, 1), None)
check('fixture 4', solve([], 6, 2), None)
check('fixture 5', solve([], 1, 1), None)
check('fixture 6', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 1), 2)
check('fixture 7', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 7, 1), 6)
check('fixture 8', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 3, 1), 2)
check('fixture 9', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 2), 5)
check('fixture 10', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 1, 1), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-rollback-choice

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(revisions, active, schema):
    candidates = [r for r in revisions if r['number'] < active]
    candidates = [r for r in candidates if r['success']]
    candidates = [r for r in candidates if r['retained']]
    candidates = [r for r in candidates if schema in r['schemas']]
    return next(iter(sorted([r['number'] for r in candidates])), None) #(r['number'] for r in candidates), default=None)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 6, 1), None)
check('fixture 2', solve([], 7, 1), None)
check('fixture 3', solve([], 3, 1), None)
check('fixture 4', solve([], 6, 2), None)
check('fixture 5', solve([], 1, 1), None)
check('fixture 6', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 1), 2)
check('fixture 7', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 7, 1), 6)
check('fixture 8', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 3, 1), 2)
check('fixture 9', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 6, 2), 5)
check('fixture 10', solve([{'number': 1, 'success': True, 'retained': True, 'schemas': [1, 2]}, {'number': 2, 'success': True, 'retained': True, 'schemas': [1]}, {'number': 3, 'success': False, 'retained': True, 'schemas': [1]}, {'number': 4, 'success': True, 'retained': False, 'schemas': [1]}, {'number': 5, 'success': True, 'retained': True, 'schemas': [2]}, {'number': 6, 'success': True, 'retained': True, 'schemas': [1]}], 1, 1), None)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Rollback unnecessarily selects the oldest release.

Choose the highest earlier revision that is successful, retained, compatible with current schema, and different from the active revision. Return None when no rollback target exists.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, active, pinned, limit):
    keep = set(pinned)
    eligible = sorted([number for number, success in releases if success and number not in keep], reverse=True)
    keep.update(eligible[:limit])
    return sorted(keep)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 1, [], 0), [1])
check('fixture 2', solve([], 4, [1, 2], 0), [1, 2, 4])
check('fixture 3', solve([(1, True), (2, True), (3, True)], 3, [], 1), [2, 3])
check('fixture 4', solve([(1, True), (2, False), (3, True)], 3, [], 1), [1, 3])
check('fixture 5', solve([(1, True), (2, True), (3, True)], 3, [1], 0), [1, 3])
check('fixture 6', solve([(1, True), (2, True), (3, True), (4, True)], 4, [2], 1), [2, 3, 4])
check('fixture 7', solve([(1, True), (2, True)], 2, [], 0), [2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-release-retention

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, active, pinned, limit):
    keep = {active} & set(pinned)
    eligible = sorted([number for number, success in releases if success and number not in keep], reverse=True)
    keep.update(eligible[:limit])
    return sorted(keep)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 1, [], 0), [1])
check('fixture 2', solve([], 4, [1, 2], 0), [1, 2, 4])
check('fixture 3', solve([(1, True), (2, True), (3, True)], 3, [], 1), [2, 3])
check('fixture 4', solve([(1, True), (2, False), (3, True)], 3, [], 1), [1, 3])
check('fixture 5', solve([(1, True), (2, True), (3, True)], 3, [1], 0), [1, 3])
check('fixture 6', solve([(1, True), (2, True), (3, True), (4, True)], 4, [2], 1), [2, 3, 4])
check('fixture 7', solve([(1, True), (2, True)], 2, [], 0), [2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
The active release can be garbage-collected.

Retain active and explicitly pinned releases regardless of age, retain newest successful unpinned releases up to limit, exclude unsuccessful unpinned releases, and return sorted unique revision IDs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, active, pinned, limit):
    keep = {active}
    eligible = sorted([number for number, success in releases if success and number not in keep], reverse=True)
    keep.update(eligible[:limit])
    return sorted(keep)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 1, [], 0), [1])
check('fixture 2', solve([], 4, [1, 2], 0), [1, 2, 4])
check('fixture 3', solve([(1, True), (2, True), (3, True)], 3, [], 1), [2, 3])
check('fixture 4', solve([(1, True), (2, False), (3, True)], 3, [], 1), [1, 3])
check('fixture 5', solve([(1, True), (2, True), (3, True)], 3, [1], 0), [1, 3])
check('fixture 6', solve([(1, True), (2, True), (3, True), (4, True)], 4, [2], 1), [2, 3, 4])
check('fixture 7', solve([(1, True), (2, True)], 2, [], 0), [2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-release-retention

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, active, pinned, limit):
    keep = {active} | set(pinned[:1])
    eligible = sorted([number for number, success in releases if success and number not in keep], reverse=True)
    keep.update(eligible[:limit])
    return sorted(keep)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 1, [], 0), [1])
check('fixture 2', solve([], 4, [1, 2], 0), [1, 2, 4])
check('fixture 3', solve([(1, True), (2, True), (3, True)], 3, [], 1), [2, 3])
check('fixture 4', solve([(1, True), (2, False), (3, True)], 3, [], 1), [1, 3])
check('fixture 5', solve([(1, True), (2, True), (3, True)], 3, [1], 0), [1, 3])
check('fixture 6', solve([(1, True), (2, True), (3, True), (4, True)], 4, [2], 1), [2, 3, 4])
check('fixture 7', solve([(1, True), (2, True)], 2, [], 0), [2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Explicitly pinned releases are deleted.

Retain active and explicitly pinned releases regardless of age, retain newest successful unpinned releases up to limit, exclude unsuccessful unpinned releases, and return sorted unique revision IDs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, active, pinned, limit):
    keep = {active} | set(pinned)
    eligible = sorted([number for number, success in releases if number not in keep], reverse=True)
    keep.update(eligible[:limit])
    return sorted(keep)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 1, [], 0), [1])
check('fixture 2', solve([], 4, [1, 2], 0), [1, 2, 4])
check('fixture 3', solve([(1, True), (2, True), (3, True)], 3, [], 1), [2, 3])
check('fixture 4', solve([(1, True), (2, False), (3, True)], 3, [], 1), [1, 3])
check('fixture 5', solve([(1, True), (2, True), (3, True)], 3, [1], 0), [1, 3])
check('fixture 6', solve([(1, True), (2, True), (3, True), (4, True)], 4, [2], 1), [2, 3, 4])
check('fixture 7', solve([(1, True), (2, True)], 2, [], 0), [2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-release-retention

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, active, pinned, limit):
    keep = {active} | set(pinned)
    eligible = sorted([number for number, success in releases if not success and number not in keep], reverse=True)
    keep.update(eligible[:limit])
    return sorted(keep)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 1, [], 0), [1])
check('fixture 2', solve([], 4, [1, 2], 0), [1, 2, 4])
check('fixture 3', solve([(1, True), (2, True), (3, True)], 3, [], 1), [2, 3])
check('fixture 4', solve([(1, True), (2, False), (3, True)], 3, [], 1), [1, 3])
check('fixture 5', solve([(1, True), (2, True), (3, True)], 3, [1], 0), [1, 3])
check('fixture 6', solve([(1, True), (2, True), (3, True), (4, True)], 4, [2], 1), [2, 3, 4])
check('fixture 7', solve([(1, True), (2, True)], 2, [], 0), [2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Failed releases consume successful retention slots.

Retain active and explicitly pinned releases regardless of age, retain newest successful unpinned releases up to limit, exclude unsuccessful unpinned releases, and return sorted unique revision IDs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, active, pinned, limit):
    keep = {active} | set(pinned)
    eligible = sorted([number for number, success in releases if success and number not in keep], reverse=False)
    keep.update(eligible[:limit])
    return sorted(keep)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 1, [], 0), [1])
check('fixture 2', solve([], 4, [1, 2], 0), [1, 2, 4])
check('fixture 3', solve([(1, True), (2, True), (3, True)], 3, [], 1), [2, 3])
check('fixture 4', solve([(1, True), (2, False), (3, True)], 3, [], 1), [1, 3])
check('fixture 5', solve([(1, True), (2, True), (3, True)], 3, [1], 0), [1, 3])
check('fixture 6', solve([(1, True), (2, True), (3, True), (4, True)], 4, [2], 1), [2, 3, 4])
check('fixture 7', solve([(1, True), (2, True)], 2, [], 0), [2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-release-retention

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, active, pinned, limit):
    keep = {active} | set(pinned)
    eligible = sorted([number for number, success in releases if success and number not in keep], reverse=not bool(releases))
    keep.update(eligible[:limit])
    return sorted(keep)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 1, [], 0), [1])
check('fixture 2', solve([], 4, [1, 2], 0), [1, 2, 4])
check('fixture 3', solve([(1, True), (2, True), (3, True)], 3, [], 1), [2, 3])
check('fixture 4', solve([(1, True), (2, False), (3, True)], 3, [], 1), [1, 3])
check('fixture 5', solve([(1, True), (2, True), (3, True)], 3, [1], 0), [1, 3])
check('fixture 6', solve([(1, True), (2, True), (3, True), (4, True)], 4, [2], 1), [2, 3, 4])
check('fixture 7', solve([(1, True), (2, True)], 2, [], 0), [2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Old releases displace newer successful releases.

Retain active and explicitly pinned releases regardless of age, retain newest successful unpinned releases up to limit, exclude unsuccessful unpinned releases, and return sorted unique revision IDs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, active, pinned, limit):
    keep = {active} | set(pinned)
    eligible = sorted([number for number, success in releases if success and number not in keep], reverse=True)
    keep.update(eligible[:max(1, limit)])
    return sorted(keep)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 1, [], 0), [1])
check('fixture 2', solve([], 4, [1, 2], 0), [1, 2, 4])
check('fixture 3', solve([(1, True), (2, True), (3, True)], 3, [], 1), [2, 3])
check('fixture 4', solve([(1, True), (2, False), (3, True)], 3, [], 1), [1, 3])
check('fixture 5', solve([(1, True), (2, True), (3, True)], 3, [1], 0), [1, 3])
check('fixture 6', solve([(1, True), (2, True), (3, True), (4, True)], 4, [2], 1), [2, 3, 4])
check('fixture 7', solve([(1, True), (2, True)], 2, [], 0), [2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-release-retention

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(releases, active, pinned, limit):
    keep = {active} | set(pinned)
    eligible = sorted([number for number, success in releases if success and number not in keep], reverse=True)
    keep.update(eligible)
    return sorted(keep)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve([], 1, [], 0), [1])
check('fixture 2', solve([], 4, [1, 2], 0), [1, 2, 4])
check('fixture 3', solve([(1, True), (2, True), (3, True)], 3, [], 1), [2, 3])
check('fixture 4', solve([(1, True), (2, False), (3, True)], 3, [], 1), [1, 3])
check('fixture 5', solve([(1, True), (2, True), (3, True)], 3, [1], 0), [1, 3])
check('fixture 6', solve([(1, True), (2, True), (3, True), (4, True)], 4, [2], 1), [2, 3, 4])
check('fixture 7', solve([(1, True), (2, True)], 2, [], 0), [2])
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
A zero retention count keeps historical releases.

Retain active and explicitly pinned releases regardless of age, retain newest successful unpinned releases up to limit, exclude unsuccessful unpinned releases, and return sorted unique revision IDs.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(environments, target, version, digest, tested, allow_downgrade):
    if target not in environments: environments = dict(environments, **{target:(0, '')})
    old_version, old_digest = environments[target]
    if version < old_version and not allow_downgrade: return 'downgrade'
    if digest == old_digest: return 'unchanged'
    if not tested: return 'untested'
    result = dict(environments)
    result[target] = (version, digest)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, 'prod', 1, 'a', True, False), 'unknown')
check('fixture 2', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, False), 'downgrade')
check('fixture 3', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, True), {'prod': (1, 'a')})
check('fixture 4', solve({'prod': (1, 'a')}, 'prod', 1, 'a', False, False), 'unchanged')
check('fixture 5', solve({'prod': (1, 'a')}, 'prod', 2, 'b', False, False), 'untested')
check('fixture 6', solve({'prod': (1, 'a'), 'stage': (2, 'b')}, 'prod', 2, 'b', True, False), {'prod': (2, 'b'), 'stage': (2, 'b')})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-promotion-plan

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(environments, target, version, digest, tested, allow_downgrade):
    if target not in environments: return {}
    old_version, old_digest = environments[target]
    if version < old_version and not allow_downgrade: return 'downgrade'
    if digest == old_digest: return 'unchanged'
    if not tested: return 'untested'
    result = dict(environments)
    result[target] = (version, digest)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, 'prod', 1, 'a', True, False), 'unknown')
check('fixture 2', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, False), 'downgrade')
check('fixture 3', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, True), {'prod': (1, 'a')})
check('fixture 4', solve({'prod': (1, 'a')}, 'prod', 1, 'a', False, False), 'unchanged')
check('fixture 5', solve({'prod': (1, 'a')}, 'prod', 2, 'b', False, False), 'untested')
check('fixture 6', solve({'prod': (1, 'a'), 'stage': (2, 'b')}, 'prod', 2, 'b', True, False), {'prod': (2, 'b'), 'stage': (2, 'b')})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Unknown environments are silently created.

Reject unknown targets, reject downgrade unless allowed, skip identical digests, reject missing test evidence, otherwise update target to candidate digest and version while preserving other environments.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(environments, target, version, digest, tested, allow_downgrade):
    if target not in environments: return 'unknown'
    old_version, old_digest = environments[target]
    if False: return 'downgrade'
    if digest == old_digest: return 'unchanged'
    if not tested: return 'untested'
    result = dict(environments)
    result[target] = (version, digest)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, 'prod', 1, 'a', True, False), 'unknown')
check('fixture 2', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, False), 'downgrade')
check('fixture 3', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, True), {'prod': (1, 'a')})
check('fixture 4', solve({'prod': (1, 'a')}, 'prod', 1, 'a', False, False), 'unchanged')
check('fixture 5', solve({'prod': (1, 'a')}, 'prod', 2, 'b', False, False), 'untested')
check('fixture 6', solve({'prod': (1, 'a'), 'stage': (2, 'b')}, 'prod', 2, 'b', True, False), {'prod': (2, 'b'), 'stage': (2, 'b')})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-promotion-plan

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(environments, target, version, digest, tested, allow_downgrade):
    if target not in environments: return 'unknown'
    old_version, old_digest = environments[target]
    if version < old_version: return 'downgrade'
    if digest == old_digest: return 'unchanged'
    if not tested: return 'untested'
    result = dict(environments)
    result[target] = (version, digest)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, 'prod', 1, 'a', True, False), 'unknown')
check('fixture 2', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, False), 'downgrade')
check('fixture 3', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, True), {'prod': (1, 'a')})
check('fixture 4', solve({'prod': (1, 'a')}, 'prod', 1, 'a', False, False), 'unchanged')
check('fixture 5', solve({'prod': (1, 'a')}, 'prod', 2, 'b', False, False), 'untested')
check('fixture 6', solve({'prod': (1, 'a'), 'stage': (2, 'b')}, 'prod', 2, 'b', True, False), {'prod': (2, 'b'), 'stage': (2, 'b')})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Downgrades ignore the explicit policy.

Reject unknown targets, reject downgrade unless allowed, skip identical digests, reject missing test evidence, otherwise update target to candidate digest and version while preserving other environments.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(environments, target, version, digest, tested, allow_downgrade):
    if target not in environments: return 'unknown'
    old_version, old_digest = environments[target]
    if version < old_version and not allow_downgrade: return 'downgrade'
    if False: return 'unchanged'
    if not tested: return 'untested'
    result = dict(environments)
    result[target] = (version, digest)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, 'prod', 1, 'a', True, False), 'unknown')
check('fixture 2', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, False), 'downgrade')
check('fixture 3', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, True), {'prod': (1, 'a')})
check('fixture 4', solve({'prod': (1, 'a')}, 'prod', 1, 'a', False, False), 'unchanged')
check('fixture 5', solve({'prod': (1, 'a')}, 'prod', 2, 'b', False, False), 'untested')
check('fixture 6', solve({'prod': (1, 'a'), 'stage': (2, 'b')}, 'prod', 2, 'b', True, False), {'prod': (2, 'b'), 'stage': (2, 'b')})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-promotion-plan

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(environments, target, version, digest, tested, allow_downgrade):
    if target not in environments: return 'unknown'
    old_version, old_digest = environments[target]
    if version < old_version and not allow_downgrade: return 'downgrade'
    if version == old_version + 1: return 'unchanged'
    if not tested: return 'untested'
    result = dict(environments)
    result[target] = (version, digest)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, 'prod', 1, 'a', True, False), 'unknown')
check('fixture 2', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, False), 'downgrade')
check('fixture 3', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, True), {'prod': (1, 'a')})
check('fixture 4', solve({'prod': (1, 'a')}, 'prod', 1, 'a', False, False), 'unchanged')
check('fixture 5', solve({'prod': (1, 'a')}, 'prod', 2, 'b', False, False), 'untested')
check('fixture 6', solve({'prod': (1, 'a'), 'stage': (2, 'b')}, 'prod', 2, 'b', True, False), {'prod': (2, 'b'), 'stage': (2, 'b')})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Identical artifacts require redundant promotion.

Reject unknown targets, reject downgrade unless allowed, skip identical digests, reject missing test evidence, otherwise update target to candidate digest and version while preserving other environments.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(environments, target, version, digest, tested, allow_downgrade):
    if target not in environments: return 'unknown'
    old_version, old_digest = environments[target]
    if version < old_version and not allow_downgrade: return 'downgrade'
    if digest == old_digest: return 'unchanged'
    if False: return 'untested'
    result = dict(environments)
    result[target] = (version, digest)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, 'prod', 1, 'a', True, False), 'unknown')
check('fixture 2', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, False), 'downgrade')
check('fixture 3', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, True), {'prod': (1, 'a')})
check('fixture 4', solve({'prod': (1, 'a')}, 'prod', 1, 'a', False, False), 'unchanged')
check('fixture 5', solve({'prod': (1, 'a')}, 'prod', 2, 'b', False, False), 'untested')
check('fixture 6', solve({'prod': (1, 'a'), 'stage': (2, 'b')}, 'prod', 2, 'b', True, False), {'prod': (2, 'b'), 'stage': (2, 'b')})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-promotion-plan

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(environments, target, version, digest, tested, allow_downgrade):
    if target not in environments: return 'unknown'
    old_version, old_digest = environments[target]
    if version < old_version and not allow_downgrade: return 'downgrade'
    if digest == old_digest: return 'unchanged'
    if tested: return 'untested'
    result = dict(environments)
    result[target] = (version, digest)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, 'prod', 1, 'a', True, False), 'unknown')
check('fixture 2', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, False), 'downgrade')
check('fixture 3', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, True), {'prod': (1, 'a')})
check('fixture 4', solve({'prod': (1, 'a')}, 'prod', 1, 'a', False, False), 'unchanged')
check('fixture 5', solve({'prod': (1, 'a')}, 'prod', 2, 'b', False, False), 'untested')
check('fixture 6', solve({'prod': (1, 'a'), 'stage': (2, 'b')}, 'prod', 2, 'b', True, False), {'prod': (2, 'b'), 'stage': (2, 'b')})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Untested artifacts reach the target.

Reject unknown targets, reject downgrade unless allowed, skip identical digests, reject missing test evidence, otherwise update target to candidate digest and version while preserving other environments.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(environments, target, version, digest, tested, allow_downgrade):
    if target not in environments: return 'unknown'
    old_version, old_digest = environments[target]
    if version < old_version and not allow_downgrade: return 'downgrade'
    if digest == old_digest: return 'unchanged'
    if not tested: return 'untested'
    result = {}
    result[target] = (version, digest)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, 'prod', 1, 'a', True, False), 'unknown')
check('fixture 2', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, False), 'downgrade')
check('fixture 3', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, True), {'prod': (1, 'a')})
check('fixture 4', solve({'prod': (1, 'a')}, 'prod', 1, 'a', False, False), 'unchanged')
check('fixture 5', solve({'prod': (1, 'a')}, 'prod', 2, 'b', False, False), 'untested')
check('fixture 6', solve({'prod': (1, 'a'), 'stage': (2, 'b')}, 'prod', 2, 'b', True, False), {'prod': (2, 'b'), 'stage': (2, 'b')})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-promotion-plan

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(environments, target, version, digest, tested, allow_downgrade):
    if target not in environments: return 'unknown'
    old_version, old_digest = environments[target]
    if version < old_version and not allow_downgrade: return 'downgrade'
    if digest == old_digest: return 'unchanged'
    if not tested: return 'untested'
    result = {target: environments[target]}
    result[target] = (version, digest)
    return result
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({}, 'prod', 1, 'a', True, False), 'unknown')
check('fixture 2', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, False), 'downgrade')
check('fixture 3', solve({'prod': (2, 'b')}, 'prod', 1, 'a', True, True), {'prod': (1, 'a')})
check('fixture 4', solve({'prod': (1, 'a')}, 'prod', 1, 'a', False, False), 'unchanged')
check('fixture 5', solve({'prod': (1, 'a')}, 'prod', 2, 'b', False, False), 'untested')
check('fixture 6', solve({'prod': (1, 'a'), 'stage': (2, 'b')}, 'prod', 2, 'b', True, False), {'prod': (2, 'b'), 'stage': (2, 'b')})
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Promotion erases unrelated environments.

Reject unknown targets, reject downgrade unless allowed, skip identical digests, reject missing test evidence, otherwise update target to candidate digest and version while preserving other environments.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(resource, desired, owner, now):
    return True and resource['owner'] == owner and not resource['protected'] and resource['references'] == 0 and now >= resource['delete_after']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), True)
check('fixture 2', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 9}, [], 'app', 10), True)
check('fixture 3', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 11}, [], 'app', 10), False)
check('fixture 4', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, ['x'], 'app', 10), False)
check('fixture 5', solve({'name': 'x', 'owner': 'other', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 6', solve({'name': 'x', 'owner': 'app', 'protected': True, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 7', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 1, 'delete_after': 10}, [], 'app', 10), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-deployment-cleanup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(resource, desired, owner, now):
    return resource['name'] in desired and resource['owner'] == owner and not resource['protected'] and resource['references'] == 0 and now >= resource['delete_after']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), True)
check('fixture 2', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 9}, [], 'app', 10), True)
check('fixture 3', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 11}, [], 'app', 10), False)
check('fixture 4', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, ['x'], 'app', 10), False)
check('fixture 5', solve({'name': 'x', 'owner': 'other', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 6', solve({'name': 'x', 'owner': 'app', 'protected': True, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 7', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 1, 'delete_after': 10}, [], 'app', 10), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Desired resources are selected for deletion.

A resource can be deleted only when not desired, owned by this deployment, not protected, no active references, and grace period completed. Grace equality permits cleanup.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(resource, desired, owner, now):
    return resource['name'] not in desired and True and not resource['protected'] and resource['references'] == 0 and now >= resource['delete_after']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), True)
check('fixture 2', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 9}, [], 'app', 10), True)
check('fixture 3', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 11}, [], 'app', 10), False)
check('fixture 4', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, ['x'], 'app', 10), False)
check('fixture 5', solve({'name': 'x', 'owner': 'other', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 6', solve({'name': 'x', 'owner': 'app', 'protected': True, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 7', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 1, 'delete_after': 10}, [], 'app', 10), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-deployment-cleanup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(resource, desired, owner, now):
    return resource['name'] not in desired and resource['owner'] != owner and not resource['protected'] and resource['references'] == 0 and now >= resource['delete_after']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), True)
check('fixture 2', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 9}, [], 'app', 10), True)
check('fixture 3', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 11}, [], 'app', 10), False)
check('fixture 4', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, ['x'], 'app', 10), False)
check('fixture 5', solve({'name': 'x', 'owner': 'other', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 6', solve({'name': 'x', 'owner': 'app', 'protected': True, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 7', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 1, 'delete_after': 10}, [], 'app', 10), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Another deployment’s resources are deleted.

A resource can be deleted only when not desired, owned by this deployment, not protected, no active references, and grace period completed. Grace equality permits cleanup.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(resource, desired, owner, now):
    return resource['name'] not in desired and resource['owner'] == owner and True and resource['references'] == 0 and now >= resource['delete_after']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), True)
check('fixture 2', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 9}, [], 'app', 10), True)
check('fixture 3', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 11}, [], 'app', 10), False)
check('fixture 4', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, ['x'], 'app', 10), False)
check('fixture 5', solve({'name': 'x', 'owner': 'other', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 6', solve({'name': 'x', 'owner': 'app', 'protected': True, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 7', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 1, 'delete_after': 10}, [], 'app', 10), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-deployment-cleanup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(resource, desired, owner, now):
    return resource['name'] not in desired and resource['owner'] == owner and resource['protected'] and resource['references'] == 0 and now >= resource['delete_after']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), True)
check('fixture 2', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 9}, [], 'app', 10), True)
check('fixture 3', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 11}, [], 'app', 10), False)
check('fixture 4', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, ['x'], 'app', 10), False)
check('fixture 5', solve({'name': 'x', 'owner': 'other', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 6', solve({'name': 'x', 'owner': 'app', 'protected': True, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 7', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 1, 'delete_after': 10}, [], 'app', 10), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Protected resources are deleted.

A resource can be deleted only when not desired, owned by this deployment, not protected, no active references, and grace period completed. Grace equality permits cleanup.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(resource, desired, owner, now):
    return resource['name'] not in desired and resource['owner'] == owner and not resource['protected'] and True and now >= resource['delete_after']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), True)
check('fixture 2', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 9}, [], 'app', 10), True)
check('fixture 3', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 11}, [], 'app', 10), False)
check('fixture 4', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, ['x'], 'app', 10), False)
check('fixture 5', solve({'name': 'x', 'owner': 'other', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 6', solve({'name': 'x', 'owner': 'app', 'protected': True, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 7', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 1, 'delete_after': 10}, [], 'app', 10), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-deployment-cleanup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(resource, desired, owner, now):
    return resource['name'] not in desired and resource['owner'] == owner and not resource['protected'] and resource['references'] <= 1 and now >= resource['delete_after']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), True)
check('fixture 2', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 9}, [], 'app', 10), True)
check('fixture 3', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 11}, [], 'app', 10), False)
check('fixture 4', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, ['x'], 'app', 10), False)
check('fixture 5', solve({'name': 'x', 'owner': 'other', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 6', solve({'name': 'x', 'owner': 'app', 'protected': True, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 7', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 1, 'delete_after': 10}, [], 'app', 10), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Resources with active consumers are deleted.

A resource can be deleted only when not desired, owned by this deployment, not protected, no active references, and grace period completed. Grace equality permits cleanup.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(resource, desired, owner, now):
    return resource['name'] not in desired and resource['owner'] == owner and not resource['protected'] and resource['references'] == 0 and True
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), True)
check('fixture 2', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 9}, [], 'app', 10), True)
check('fixture 3', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 11}, [], 'app', 10), False)
check('fixture 4', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, ['x'], 'app', 10), False)
check('fixture 5', solve({'name': 'x', 'owner': 'other', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 6', solve({'name': 'x', 'owner': 'app', 'protected': True, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 7', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 1, 'delete_after': 10}, [], 'app', 10), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-deployment-cleanup

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(resource, desired, owner, now):
    return resource['name'] not in desired and resource['owner'] == owner and not resource['protected'] and resource['references'] == 0 and now > resource['delete_after']
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), True)
check('fixture 2', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 9}, [], 'app', 10), True)
check('fixture 3', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 11}, [], 'app', 10), False)
check('fixture 4', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 0, 'delete_after': 10}, ['x'], 'app', 10), False)
check('fixture 5', solve({'name': 'x', 'owner': 'other', 'protected': False, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 6', solve({'name': 'x', 'owner': 'app', 'protected': True, 'references': 0, 'delete_after': 10}, [], 'app', 10), False)
check('fixture 7', solve({'name': 'x', 'owner': 'app', 'protected': False, 'references': 1, 'delete_after': 10}, [], 'app', 10), False)
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Resources are deleted before their grace deadline.

A resource can be deleted only when not desired, owned by this deployment, not protected, no active references, and grace period completed. Grace equality permits cleanup.

split: open-access


broken_source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, labels, dropped, unit, kind):
    return (None, tuple(sorted((k,v) for k,v in labels.items() if k not in dropped)), unit, kind)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('requests', {'z': '1', 'a': '2'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 2', solve('requests', {'a': '2', 'z': '1'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 3', solve('requests', {'a': '2', 'z': '1'}, ['z'], 'count', 'counter'), ('requests', (('a', '2'),), 'count', 'counter'))
check('fixture 4', solve('requests', {'a': 'same', 'b': 'same'}, ['a'], 'count', 'counter'), ('requests', (('b', 'same'),), 'count', 'counter'))
check('fixture 5', solve('latency', {}, [], 'seconds', 'histogram'), ('latency', (), 'seconds', 'histogram'))
check('fixture 6', solve('latency', {}, [], 'milliseconds', 'gauge'), ('latency', (), 'milliseconds', 'gauge'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

evaluation_group: xt-metric-series

source:
"""Failure Map reference implementation. Python standard library only."""
import json

N = 1
observations = []
def solve(name, labels, dropped, unit, kind):
    return ('metric', tuple(sorted((k,v) for k,v in labels.items() if k not in dropped)), unit, kind)
def check(label, actual, expected):
    observations.append({"check": label, "actual": actual, "expected": expected, "passed": actual == expected})
check('fixture 1', solve('requests', {'z': '1', 'a': '2'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 2', solve('requests', {'a': '2', 'z': '1'}, [], 'count', 'counter'), ('requests', (('a', '2'), ('z', '1')), 'count', 'counter'))
check('fixture 3', solve('requests', {'a': '2', 'z': '1'}, ['z'], 'count', 'counter'), ('requests', (('a', '2'),), 'count', 'counter'))
check('fixture 4', solve('requests', {'a': 'same', 'b': 'same'}, ['a'], 'count', 'counter'), ('requests', (('b', 'same'),), 'count', 'counter'))
check('fixture 5', solve('latency', {}, [], 'seconds', 'histogram'), ('latency', (), 'seconds', 'histogram'))
check('fixture 6', solve('latency', {}, [], 'milliseconds', 'gauge'), ('latency', (), 'milliseconds', 'gauge'))
print(json.dumps({"observations": observations, "passed": all(x["passed"] for x in observations)}, ensure_ascii=False))
raise SystemExit(0 if all(x["passed"] for x in observations) else 1)

prompt:
Different instruments collide by omitting the metric name.

A series identity includes metric name, sorted label key-value pairs excluding configured dropped keys, measurement unit, and instrument kind. Dropping values never drops other equal-valued labels.

split: open-access