video-image-production-build/evals/global_data/cases.json

[
  {
    "id": "planner-critical-audit-coverage",
    "level": "single_step",
    "risk": "Planner 遗漏 Production Brief 的关键 Global Data 路径",
    "expected": "所有要求审计路径都有 critical 处置",
    "test": "tests/agents/test_planner.py::PlannerDagTest::test_all_global_audit_paths_require_critical_disposition"
  },
  {
    "id": "executor-inspect-before-remote-run",
    "level": "trajectory",
    "risk": "Executor 未 inspect Schema 就调用远程工具",
    "expected": "同一远程工具必须先 inspect 后 run",
    "test": "tests/agents/test_executor.py::GeneralExecutorTest::test_research_run_requires_prior_matching_inspect"
  },
  {
    "id": "executor-remote-artifact-localization",
    "level": "single_step",
    "risk": "远程 Candidate URI 未映射为本地正式 Artifact",
    "expected": "远程来源唯一解析为当前 Run 内本地文件",
    "test": "tests/run/test_artifacts.py::ArtifactIntegrityTest::test_remote_candidate_resolves_to_single_local_copy"
  },
  {
    "id": "remote-url-content-replacement",
    "level": "adversarial",
    "risk": "同一远程 URL 在验收后替换内容",
    "expected": "正式 Artifact 继续绑定首次本地化并封印的字节",
    "test": "tests/run/test_artifacts.py::ArtifactIntegrityTest::test_localized_artifact_survives_remote_url_content_replacement"
  },
  {
    "id": "validator-missing-artifact-preflight",
    "level": "single_step",
    "risk": "结构化 Artifact 缺失却让模型继续给出 PASS",
    "expected": "模型调用前由确定性检查拒绝",
    "test": "tests/agents/test_task_validator.py::GeneralValidatorTest::test_missing_structured_artifact_is_rejected_before_model_execution"
  },
  {
    "id": "validator-critical-failure-verdict",
    "level": "final_output",
    "risk": "关键验收项 FAIL 但报告总判定为 PASS",
    "expected": "代码确定性汇总为 FAIL",
    "test": "tests/agents/test_task_validator.py::GeneralValidatorTest::test_critical_failure_deterministically_fails_report"
  },
  {
    "id": "stage-gap-replan-preserves-pass",
    "level": "trajectory",
    "risk": "Stage 缺口 Replan 时重复执行已 PASS Task",
    "expected": "只补缺口，已 PASS Task 不重跑",
    "test": "tests/graph/test_graph.py::GlobalDataGraphTest::test_stage_missing_requirement_replans_without_rerunning_passed_task"
  },
  {
    "id": "completed-run-tamper-rejection",
    "level": "final_output",
    "risk": "COMPLETED 后正式 Artifact 同路径内容被替换",
    "expected": "同 thread no-op 终态重验拒绝内容漂移",
    "test": "tests/graph/test_recovery.py::DurableRunTest::test_completed_noop_rejects_tampered_artifact_content"
  }
]