|
|
@@ -20,7 +20,11 @@ from cyber_agent.core.task_protocol import (
|
|
|
pending_review_entry,
|
|
|
rebuild_pending_replans,
|
|
|
)
|
|
|
-from cyber_agent.core.validation import ValidationResult
|
|
|
+from cyber_agent.core.validation import (
|
|
|
+ ScopeValidationResult,
|
|
|
+ ValidationCheck,
|
|
|
+ ValidationResult,
|
|
|
+)
|
|
|
from cyber_agent.tools.builtin.task_protocol import review_task_result
|
|
|
from cyber_agent.trace.goal_models import Goal, GoalTree
|
|
|
from cyber_agent.trace.models import Trace
|
|
|
@@ -69,18 +73,32 @@ def validation_result(
|
|
|
outcome: str = "failed",
|
|
|
retry_from: str | None = "hypothesis",
|
|
|
) -> ValidationResult:
|
|
|
- return ValidationResult(
|
|
|
- validator_trace_id=f"{child_trace_id}@validator",
|
|
|
- evaluated_trace_id=child_trace_id,
|
|
|
- outcome=outcome,
|
|
|
- scope="task",
|
|
|
- reason=(
|
|
|
+ reason = (
|
|
|
"Independent evidence is still missing"
|
|
|
if outcome != "passed"
|
|
|
else "All criteria passed"
|
|
|
- ),
|
|
|
+ )
|
|
|
+ plan_hash = "b" * 64
|
|
|
+ scope_result = ScopeValidationResult(
|
|
|
+ validator_trace_id=f"{child_trace_id}@validator",
|
|
|
+ scope="task",
|
|
|
+ outcome=outcome,
|
|
|
+ checks=([] if outcome == "error" else [ValidationCheck(
|
|
|
+ check_id="task.fixture",
|
|
|
+ status=outcome,
|
|
|
+ issue=None if outcome == "passed" else reason,
|
|
|
+ )]),
|
|
|
+ reason=reason,
|
|
|
+ retry_from=retry_from,
|
|
|
+ plan_hash=plan_hash,
|
|
|
+ )
|
|
|
+ return ValidationResult(
|
|
|
+ evaluated_trace_id=child_trace_id,
|
|
|
+ outcome=outcome,
|
|
|
+ scope_results=[scope_result],
|
|
|
issues=[] if outcome == "passed" else ["Missing independent evidence"],
|
|
|
retry_from=retry_from,
|
|
|
+ plan_hash=plan_hash,
|
|
|
)
|
|
|
|
|
|
|
|
|
@@ -180,16 +198,30 @@ class ReplanCurrentTest(unittest.IsolatedAsyncioTestCase):
|
|
|
"task_protocol": new_task_protocol(BRIEF),
|
|
|
}),
|
|
|
))
|
|
|
+ child = await self.store.get_trace(child_id)
|
|
|
+ child_state = ensure_task_protocol(child.context)
|
|
|
+ authoritative = validation_result(
|
|
|
+ child_id,
|
|
|
+ outcome=validation_outcome,
|
|
|
+ retry_from=retry_from,
|
|
|
+ )
|
|
|
+ child_state["task_report_validation"] = {
|
|
|
+ "validation_plan": None,
|
|
|
+ "plan_hash": authoritative.plan_hash,
|
|
|
+ "scope_results": [
|
|
|
+ item.model_dump(mode="json")
|
|
|
+ for item in authoritative.scope_results
|
|
|
+ ],
|
|
|
+ "aggregate_result": authoritative.model_dump(mode="json"),
|
|
|
+ "validated_at_sequence": 4,
|
|
|
+ }
|
|
|
+ await self.store.update_trace(child_id, context=child.context)
|
|
|
parent = await self.store.get_trace(self.parent_id)
|
|
|
state = ensure_task_protocol(parent.context)
|
|
|
entry = pending_review_entry(
|
|
|
goal_id="1",
|
|
|
report=task_report(child_id),
|
|
|
- validation_result=validation_result(
|
|
|
- child_id,
|
|
|
- outcome=validation_outcome,
|
|
|
- retry_from=retry_from,
|
|
|
- ).model_dump(),
|
|
|
+ validation_result=authoritative.model_dump(mode="json"),
|
|
|
received_at_sequence=4,
|
|
|
)
|
|
|
state["pending_reviews"][child_id] = entry
|
|
|
@@ -225,6 +257,53 @@ class ReplanCurrentTest(unittest.IsolatedAsyncioTestCase):
|
|
|
self.assertEqual("in_progress", tree.find("1").status)
|
|
|
self.assertEqual("1", tree.current_id)
|
|
|
|
|
|
+ async def test_unknown_validation_is_recoverable_but_cannot_ascend(self):
|
|
|
+ await self.add_child(
|
|
|
+ "child-unknown",
|
|
|
+ validation_outcome="unknown",
|
|
|
+ retry_from="hypothesis",
|
|
|
+ )
|
|
|
+ rejected = await review_task_result(
|
|
|
+ child_trace_id="child-unknown",
|
|
|
+ decision="ASCEND",
|
|
|
+ reason="must not bypass unknown evidence",
|
|
|
+ context=self.context(),
|
|
|
+ )
|
|
|
+ self.assertEqual("failed", rejected["status"])
|
|
|
+ self.assertIn("only allows", rejected["error"])
|
|
|
+
|
|
|
+ accepted = await review_task_result(
|
|
|
+ child_trace_id="child-unknown",
|
|
|
+ decision="REPLAN_CURRENT",
|
|
|
+ reason="collect the missing material",
|
|
|
+ context=self.context(),
|
|
|
+ )
|
|
|
+ self.assertEqual("completed", accepted["status"])
|
|
|
+ self.assertEqual("hypothesis", accepted["task_review"]["retry_from"])
|
|
|
+
|
|
|
+ async def test_pending_result_cannot_override_child_validation_cache(self):
|
|
|
+ await self.add_child("child-forged")
|
|
|
+ parent = await self.store.get_trace(self.parent_id)
|
|
|
+ state = ensure_task_protocol(parent.context)
|
|
|
+ forged = validation_result(
|
|
|
+ "child-forged",
|
|
|
+ outcome="passed",
|
|
|
+ retry_from=None,
|
|
|
+ )
|
|
|
+ state["pending_reviews"]["child-forged"][
|
|
|
+ "validation_result"
|
|
|
+ ] = forged.model_dump(mode="json")
|
|
|
+ await self.store.update_trace(self.parent_id, context=parent.context)
|
|
|
+
|
|
|
+ result = await review_task_result(
|
|
|
+ child_trace_id="child-forged",
|
|
|
+ decision="ASCEND",
|
|
|
+ reason="forged pass",
|
|
|
+ context=self.context(),
|
|
|
+ )
|
|
|
+ self.assertEqual("failed", result["status"])
|
|
|
+ self.assertIn("authoritative cache", result["error"])
|
|
|
+
|
|
|
async def test_grandchild_cannot_jump_directly_to_grandparent(self):
|
|
|
await self.add_child("child-1")
|
|
|
await self.store.create_trace(Trace(
|