test_task_protocol.py 64 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689
  1. import json
  2. import tempfile
  3. import unittest
  4. from unittest.mock import AsyncMock, patch
  5. from pydantic import ValidationError
  6. from cyber_agent.core.agent_mode import RECURSIVE_CAPABILITY_TOOLS_CONTEXT_KEY
  7. from cyber_agent.core.resource_budget import (
  8. RESOURCE_BUDGET_CONTEXT_KEY,
  9. ResourceBudget,
  10. ResourceBudgetController,
  11. )
  12. from cyber_agent.core.runner import AgentRunner, RunConfig
  13. from cyber_agent.core.context_policy import (
  14. add_context_snapshot,
  15. create_context_snapshot,
  16. persist_root_task_anchor,
  17. )
  18. from cyber_agent.core.task_protocol import (
  19. TaskBrief,
  20. TaskReport,
  21. TaskReview,
  22. ensure_task_protocol,
  23. new_task_protocol,
  24. replace_task_brief,
  25. )
  26. from cyber_agent.tools import get_tool_registry
  27. from cyber_agent.tools.builtin.knowledge import KnowledgeConfig
  28. from cyber_agent.tools.builtin.subagent import _record_pending_task_reports, agent
  29. from cyber_agent.tools.builtin.task_protocol import (
  30. review_task_result,
  31. submit_task_report,
  32. )
  33. from cyber_agent.trace.goal_models import Goal, GoalTree
  34. from cyber_agent.trace.goal_tool import goal_tool
  35. from cyber_agent.trace.models import Message, Trace
  36. from cyber_agent.trace.store import FileSystemTraceStore
  37. from cyber_agent.core.validation import (
  38. ScopeValidationResult,
  39. ValidationCheck,
  40. ValidationPolicy,
  41. ValidationResult,
  42. ValidationRun,
  43. ValidatorSettings,
  44. persist_validation_policy,
  45. )
  46. BRIEF = {
  47. "objective": "research one concrete question",
  48. "reason": "the parent needs one independently checked result",
  49. "completion_criteria": ["provide one checked conclusion"],
  50. "expected_outputs": ["one checked conclusion"],
  51. "constraints": ["do not guess"],
  52. }
  53. ROOT_ANCHOR = {
  54. "objective": "finish the root task",
  55. "completion_criteria": ["finish the root task"],
  56. "constraints": ["do not guess"],
  57. }
  58. def with_root_anchor(context):
  59. persist_root_task_anchor(context, ROOT_ANCHOR)
  60. return context
  61. def report_payload(*, outcome="satisfied"):
  62. return {
  63. "summary": "checked result",
  64. "outcome": outcome,
  65. "validation": {
  66. "hard_passed": outcome == "satisfied",
  67. "open_issues": [] if outcome == "satisfied" else ["still failing"],
  68. },
  69. "next_step_suggestion": {
  70. "direction": "NONE",
  71. "reason": "No additional task is needed",
  72. },
  73. "outputs": [{"kind": "note", "value": "result"}],
  74. "evidence": [{"kind": "test", "value": "passed"}],
  75. "remaining_issues": [] if outcome == "satisfied" else ["still failing"],
  76. }
  77. def validation_payload(child_trace_id, *, outcome="passed"):
  78. plan_hash = "a" * 64
  79. if outcome == "passed":
  80. issues = []
  81. retry_from = None
  82. elif outcome == "failed":
  83. issues = ["task needs revision"]
  84. retry_from = "task_definition"
  85. else:
  86. issues = ["validator could not evaluate the task"]
  87. retry_from = None
  88. scope_result = ScopeValidationResult(
  89. validator_trace_id=f"{child_trace_id}@validator-fixture",
  90. scope="task",
  91. outcome=outcome,
  92. checks=[ValidationCheck(
  93. check_id="task.fixture",
  94. status=("unknown" if outcome == "error" else outcome),
  95. issue=None if outcome == "passed" else issues[0],
  96. )],
  97. reason="fixture validation",
  98. retry_from=retry_from,
  99. plan_hash=plan_hash,
  100. )
  101. return ValidationResult(
  102. evaluated_trace_id=child_trace_id,
  103. outcome=outcome,
  104. scope_results=[scope_result],
  105. issues=issues,
  106. retry_from=retry_from,
  107. plan_hash=plan_hash,
  108. ).model_dump(mode="json")
  109. def passed_validator_response(kwargs):
  110. """按当前 ValidationPlan 返回完整的通过结果。"""
  111. messages = kwargs.get("messages") or []
  112. packet = json.loads(messages[1]["content"])
  113. scope = packet["validation_scope"]
  114. checks = [
  115. {
  116. "check_id": item["check_id"],
  117. "status": "passed",
  118. "evidence_refs": [],
  119. "issue": None,
  120. }
  121. for item in packet["validation_plan"]["checks"]
  122. if item["scope"] == scope
  123. ]
  124. return {
  125. "content": json.dumps({
  126. "outcome": "passed",
  127. "scope": scope,
  128. "checks": checks,
  129. "reason": "all planned checks passed",
  130. "retry_from": None,
  131. }),
  132. "tool_calls": [],
  133. "finish_reason": "stop",
  134. }
  135. async def create_recursive_root(store, trace_id, *, state=None, uid="user-1"):
  136. budget = ResourceBudget()
  137. context = {
  138. "agent_mode": "recursive",
  139. "agent_mode_revision": 2,
  140. "agent_depth": 0,
  141. "root_trace_id": trace_id,
  142. RESOURCE_BUDGET_CONTEXT_KEY: budget.to_dict(),
  143. "task_protocol": state or new_task_protocol(),
  144. }
  145. persist_root_task_anchor(context, ROOT_ANCHOR)
  146. persist_validation_policy(
  147. context,
  148. ValidationPolicy(),
  149. ValidatorSettings(),
  150. )
  151. await store.create_trace(Trace(
  152. trace_id=trace_id,
  153. mode="agent",
  154. task="root",
  155. uid=uid,
  156. model="fake-model",
  157. context=context,
  158. ))
  159. await ResourceBudgetController(store).initialize(
  160. trace_id,
  161. budget,
  162. initial_agents=1,
  163. )
  164. return await store.get_trace(trace_id)
  165. class FakeToolRegistry:
  166. def get_tool_names(self):
  167. return [
  168. "agent",
  169. "evaluate",
  170. "bash_command",
  171. "goal",
  172. "read_file",
  173. "submit_task_report",
  174. "review_task_result",
  175. "read_context_ref",
  176. ]
  177. class ReportingRunner:
  178. def __init__(self, store):
  179. self.store = store
  180. self.tools = FakeToolRegistry()
  181. self.debug = False
  182. self.stdin_check = None
  183. self._cancel_events = {}
  184. self.configs = []
  185. self.resource_budget = ResourceBudgetController(store)
  186. self.validation_count = 0
  187. async def _resource_budget_for_trace(self, trace_id):
  188. trace = await self.store.get_trace(trace_id)
  189. root_id = trace.context["root_trace_id"]
  190. root = trace if root_id == trace_id else await self.store.get_trace(root_id)
  191. return root_id, ResourceBudget.from_dict(
  192. root.context[RESOURCE_BUDGET_CONTEXT_KEY]
  193. )
  194. async def validate_recursive_trace(
  195. self,
  196. evaluated_trace_id,
  197. *,
  198. scope,
  199. deterministic_failure=None,
  200. **_kwargs,
  201. ):
  202. evaluated = await self.store.get_trace(evaluated_trace_id)
  203. child_state = ensure_task_protocol(evaluated.context)
  204. cached = child_state.get("task_report_validation")
  205. if isinstance(cached, dict) and cached.get("aggregate_result"):
  206. result = ValidationResult.model_validate(cached["aggregate_result"])
  207. return ValidationRun(
  208. result=result,
  209. trace_ids=[
  210. item.validator_trace_id for item in result.scope_results
  211. ],
  212. cached=True,
  213. )
  214. self.validation_count += 1
  215. validator_trace_id = (
  216. f"{evaluated_trace_id}@validator-fixture-{self.validation_count}"
  217. )
  218. if deterministic_failure:
  219. outcome = deterministic_failure.get("outcome", "error")
  220. issues = deterministic_failure.get("issues") or [
  221. deterministic_failure.get("reason", "validation failed")
  222. ]
  223. retry_from = deterministic_failure.get("retry_from")
  224. else:
  225. outcome = "passed"
  226. issues = []
  227. retry_from = None
  228. result = ValidationResult.model_validate(
  229. validation_payload(evaluated_trace_id, outcome=outcome)
  230. )
  231. child_state["task_report_validation"] = {
  232. "validation_plan": None,
  233. "plan_hash": result.plan_hash,
  234. "scope_results": [
  235. item.model_dump(mode="json") for item in result.scope_results
  236. ],
  237. "aggregate_result": result.model_dump(mode="json"),
  238. "validated_at_sequence": evaluated.head_sequence,
  239. }
  240. await self.store.update_trace(
  241. evaluated_trace_id,
  242. context=evaluated.context,
  243. )
  244. await self.store.create_trace(Trace(
  245. trace_id=validator_trace_id,
  246. mode="agent",
  247. task=f"validate {evaluated_trace_id}",
  248. parent_trace_id=evaluated_trace_id,
  249. uid=evaluated.uid,
  250. status="completed",
  251. tools=[],
  252. context=with_root_anchor({
  253. "created_by_tool": "validator",
  254. "root_trace_id": evaluated.context["root_trace_id"],
  255. "agent_depth": evaluated.context["agent_depth"],
  256. }),
  257. ))
  258. return ValidationRun(result=result, trace_ids=[validator_trace_id])
  259. async def run_result(self, messages, config, on_event=None):
  260. self.configs.append(config)
  261. submitted = await submit_task_report(
  262. task_report=report_payload(),
  263. context={
  264. "store": self.store,
  265. "trace_id": config.trace_id,
  266. "sequence": 3,
  267. },
  268. )
  269. if submitted["status"] != "completed":
  270. raise AssertionError(submitted)
  271. await self.store.update_trace(config.trace_id, status="completed")
  272. return {
  273. "status": "completed",
  274. "summary": "child submitted TaskReport",
  275. "saved_knowledge_ids": [],
  276. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  277. }
  278. class FailedAfterReportingRunner(ReportingRunner):
  279. async def run_result(self, messages, config, on_event=None):
  280. result = await super().run_result(messages, config, on_event)
  281. await self.store.update_trace(
  282. config.trace_id,
  283. status="failed",
  284. error_message="runner failed after report submission",
  285. )
  286. return {
  287. **result,
  288. "status": "failed",
  289. "error": "runner failed after report submission",
  290. }
  291. class ForgedIdentityRunner(ReportingRunner):
  292. async def run_result(self, messages, config, on_event=None):
  293. trace = await self.store.get_trace(config.trace_id)
  294. state = ensure_task_protocol(trace.context)
  295. state["task_report"] = TaskReport(
  296. child_trace_id="someone-elses-trace",
  297. **report_payload(),
  298. ).model_dump()
  299. await self.store.update_trace(
  300. config.trace_id,
  301. context=trace.context,
  302. status="completed",
  303. )
  304. return {
  305. "status": "completed",
  306. "summary": "forged report identity",
  307. "saved_knowledge_ids": [],
  308. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  309. }
  310. class RecursiveTaskProtocolTest(unittest.IsolatedAsyncioTestCase):
  311. async def asyncSetUp(self):
  312. self.temp_dir = tempfile.TemporaryDirectory()
  313. self.store = FileSystemTraceStore(self.temp_dir.name)
  314. self.runner = ReportingRunner(self.store)
  315. self.root_id = "recursive-root"
  316. await create_recursive_root(self.store, self.root_id)
  317. self.tree = GoalTree(
  318. mission="root",
  319. goals=[Goal(id="1", description="delegate", status="in_progress")],
  320. current_id="1",
  321. )
  322. await self.store.update_goal_tree(self.root_id, self.tree)
  323. async def asyncTearDown(self):
  324. self.temp_dir.cleanup()
  325. def context(self, trace_id=None, goal_id="1", tree=None):
  326. return {
  327. "store": self.store,
  328. "trace_id": trace_id or self.root_id,
  329. "goal_id": goal_id,
  330. "runner": self.runner,
  331. "goal_tree": tree or self.tree,
  332. "sequence": 10,
  333. RECURSIVE_CAPABILITY_TOOLS_CONTEXT_KEY: self.runner.tools.get_tool_names(),
  334. }
  335. async def children(self):
  336. return await self.store.list_traces(
  337. parent_trace_id=self.root_id,
  338. created_by_tool="agent",
  339. limit=20,
  340. )
  341. async def test_models_validate_and_round_trip(self):
  342. brief = TaskBrief.model_validate(BRIEF)
  343. self.assertEqual(
  344. {
  345. **BRIEF,
  346. "parent_findings": [],
  347. "context": {},
  348. "context_refs": [],
  349. "validation_scopes": [],
  350. },
  351. brief.model_dump(exclude_none=True),
  352. )
  353. with self.assertRaises(ValidationError):
  354. TaskBrief(objective="", completion_criteria=[])
  355. with self.assertRaises(ValidationError):
  356. TaskReview(
  357. parent_trace_id="p",
  358. child_trace_id="c",
  359. decision="ACCEPT_REFINE",
  360. reason="continue",
  361. )
  362. invalid_suggestion = report_payload()
  363. invalid_suggestion["next_step_suggestion"] = {
  364. "direction": "NONE",
  365. "reason": "No next task should be attached",
  366. "suggested_next_task": BRIEF,
  367. }
  368. with self.assertRaises(ValidationError):
  369. TaskReport(child_trace_id="child", **invalid_suggestion)
  370. root_submission = await submit_task_report(
  371. task_report=report_payload(),
  372. context={"store": self.store, "trace_id": self.root_id},
  373. )
  374. self.assertEqual("failed", root_submission["status"])
  375. self.assertIn("Root Agent", root_submission["error"])
  376. async def test_child_report_blocks_new_child_until_parent_review(self):
  377. result = await agent(task_brief=BRIEF, context=self.context())
  378. self.assertEqual("completed", result["status"])
  379. child_id = result["sub_trace_id"]
  380. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  381. parent = await self.store.get_trace(self.root_id)
  382. state = ensure_task_protocol(parent.context)
  383. self.assertEqual([child_id], list(state["pending_reviews"]))
  384. tree = await self.store.get_goal_tree(self.root_id)
  385. self.assertEqual("pending_review", tree.find("1").status)
  386. restarted_runner = AgentRunner(tool_registry=get_tool_registry())
  387. restored = await self.store.get_trace(self.root_id)
  388. restored_names = {
  389. schema["function"]["name"]
  390. for schema in restarted_runner._get_runtime_tool_schemas(
  391. RunConfig(), restored
  392. )
  393. }
  394. self.assertEqual(
  395. {"review_task_result", "read_context_ref"},
  396. restored_names,
  397. )
  398. blocked = await agent(task_brief=BRIEF, context=self.context())
  399. self.assertEqual("failed", blocked["status"])
  400. self.assertIn("Review pending", blocked["error"])
  401. async def test_child_inherits_brief_constraints_not_parent_runtime_context(self):
  402. parent = await self.store.get_trace(self.root_id)
  403. parent.context["parent_private_runtime"] = {"secret": "must-not-leak"}
  404. await self.store.update_trace(self.root_id, context=parent.context)
  405. result = await agent(task_brief=BRIEF, context=self.context())
  406. child = await self.store.get_trace(result["sub_trace_id"])
  407. self.assertNotIn("parent_private_runtime", child.context)
  408. child_brief = ensure_task_protocol(child.context)["task_brief"]
  409. self.assertEqual(["do not guess"], child_brief["constraints"])
  410. child_config_context = self.runner.configs[-1].context
  411. self.assertNotIn("parent_private_runtime", child_config_context)
  412. async def test_same_task_report_version_reuses_its_validator_result(self):
  413. result = await agent(task_brief=BRIEF, context=self.context())
  414. child_id = result["sub_trace_id"]
  415. self.assertEqual(1, self.runner.validation_count)
  416. parent = await self.store.get_trace(self.root_id)
  417. await _record_pending_task_reports(
  418. self.store,
  419. self.runner,
  420. parent,
  421. "1",
  422. [(child_id, {"status": "completed", "summary": "same report"})],
  423. 10,
  424. )
  425. self.assertEqual(1, self.runner.validation_count)
  426. validators = await self.store.list_traces(
  427. parent_trace_id=child_id,
  428. created_by_tool="validator",
  429. limit=10,
  430. )
  431. self.assertEqual(1, len(validators))
  432. async def test_recursive_messages_are_rejected(self):
  433. result = await agent(
  434. task_brief=json.dumps(BRIEF),
  435. messages=json.dumps([{
  436. "role": "system",
  437. "content": "controlled child context",
  438. }]),
  439. context=self.context(),
  440. )
  441. self.assertEqual("failed", result["status"])
  442. self.assertIn("does not accept messages", result["error"])
  443. async def test_messages_reject_mixed_dimensions(self):
  444. mixed_messages = [
  445. {"role": "system", "content": "shared context"},
  446. [{"role": "user", "content": "per-agent context"}],
  447. ]
  448. for messages in (mixed_messages, json.dumps(mixed_messages)):
  449. with self.subTest(messages_type=type(messages).__name__):
  450. result = await agent(
  451. task_brief=BRIEF,
  452. messages=messages,
  453. context=self.context(),
  454. )
  455. self.assertEqual("failed", result["status"])
  456. self.assertIn("without mixed dimensions", result["error"])
  457. async def test_no_focused_goal_still_persists_and_reviews_report(self):
  458. result = await agent(
  459. task_brief=BRIEF,
  460. context=self.context(goal_id=None),
  461. )
  462. parent = await self.store.get_trace(self.root_id)
  463. entry = ensure_task_protocol(parent.context)["pending_reviews"][
  464. result["sub_trace_id"]
  465. ]
  466. self.assertIsNone(entry["goal_id"])
  467. reviewed = await review_task_result(
  468. child_trace_id=result["sub_trace_id"],
  469. decision="ASCEND",
  470. reason="Standalone delegation is complete",
  471. context=self.context(goal_id=None),
  472. )
  473. self.assertEqual("completed", reviewed["status"])
  474. async def test_accept_refine_approves_exactly_one_next_child(self):
  475. first = await agent(task_brief=BRIEF, context=self.context())
  476. next_brief = {
  477. "objective": "check the independent gap",
  478. "reason": "the first report identified a separate gap",
  479. "completion_criteria": ["close the gap"],
  480. "expected_outputs": ["a checked gap analysis"],
  481. }
  482. parent = await self.store.get_trace(self.root_id)
  483. ensure_task_protocol(parent.context)["protocol_correction_attempts"] = 2
  484. await self.store.update_trace(self.root_id, context=parent.context)
  485. reviewed = await review_task_result(
  486. child_trace_id=first["sub_trace_id"],
  487. decision="ACCEPT_REFINE",
  488. reason="The gap belongs to this parent",
  489. approved_next_task=json.dumps(next_brief),
  490. context=self.context(),
  491. )
  492. self.assertEqual("completed", reviewed["status"])
  493. self.assertIsNotNone(reviewed["context_ref"])
  494. self.assertIn(
  495. reviewed["context_ref"],
  496. reviewed["next_action_task_brief"]["context_refs"],
  497. )
  498. parent = await self.store.get_trace(self.root_id)
  499. self.assertEqual(
  500. 0,
  501. ensure_task_protocol(parent.context)["protocol_correction_attempts"],
  502. )
  503. schemas = AgentRunner(
  504. tool_registry=get_tool_registry()
  505. )._get_runtime_tool_schemas(RunConfig(), parent)
  506. self.assertEqual(
  507. {"agent", "read_context_ref"},
  508. {schema["function"]["name"] for schema in schemas},
  509. )
  510. wrong = await agent(task_brief=BRIEF, context=self.context())
  511. self.assertEqual("failed", wrong["status"])
  512. self.assertIn("approved next task", wrong["error"])
  513. second = await agent(
  514. task_brief=reviewed["next_action_task_brief"],
  515. context=self.context(),
  516. )
  517. self.assertEqual("completed", second["status"])
  518. self.assertEqual(2, len(await self.children()))
  519. second_child = await self.store.get_trace(second["sub_trace_id"])
  520. self.assertIn(
  521. reviewed["context_ref"]["ref_id"],
  522. second_child.context["context_access"]["snapshots"],
  523. )
  524. parent = await self.store.get_trace(self.root_id)
  525. self.assertEqual([], ensure_task_protocol(parent.context)["next_actions"])
  526. async def test_review_rejects_an_unusable_context_ref_without_consuming_report(self):
  527. first = await agent(task_brief=BRIEF, context=self.context())
  528. invalid_next = {
  529. "objective": "use an unauthorized context ref",
  530. "reason": "exercise approval validation",
  531. "completion_criteria": ["must not be approved"],
  532. "expected_outputs": ["nothing"],
  533. "context_refs": [{
  534. "ref_id": "ctx_000000000000000000000000",
  535. "version": "0" * 64,
  536. }],
  537. }
  538. reviewed = await review_task_result(
  539. child_trace_id=first["sub_trace_id"],
  540. decision="ACCEPT_REFINE",
  541. reason="try an invalid transfer",
  542. approved_next_task=invalid_next,
  543. context=self.context(),
  544. )
  545. self.assertEqual("failed", reviewed["status"])
  546. self.assertIn("not authorized", reviewed["error"])
  547. parent = await self.store.get_trace(self.root_id)
  548. state = ensure_task_protocol(parent.context)
  549. self.assertIn(first["sub_trace_id"], state["pending_reviews"])
  550. self.assertEqual([], state["reviews"])
  551. async def test_oversized_review_snapshot_warns_without_blocking_review(self):
  552. first = await agent(task_brief=BRIEF, context=self.context())
  553. parent = await self.store.get_trace(self.root_id)
  554. state = ensure_task_protocol(parent.context)
  555. state["pending_reviews"][first["sub_trace_id"]]["task_report"]["outputs"] = [
  556. {"payload": "x" * 16_000}
  557. ]
  558. await self.store.update_trace(self.root_id, context=parent.context)
  559. reviewed = await review_task_result(
  560. child_trace_id=first["sub_trace_id"],
  561. decision="ASCEND",
  562. reason="The task passed but its transferable snapshot is too large",
  563. context=self.context(),
  564. )
  565. self.assertEqual("completed", reviewed["status"])
  566. self.assertIsNone(reviewed["context_ref"])
  567. self.assertIn("16000", reviewed["context_ref_warning"])
  568. parent = await self.store.get_trace(self.root_id)
  569. self.assertEqual({}, ensure_task_protocol(parent.context)["pending_reviews"])
  570. async def test_batch_context_refs_are_validated_before_any_trace_is_created(self):
  571. invalid_ref_brief = {
  572. **BRIEF,
  573. "objective": "second child with an unauthorized reference",
  574. "context_refs": [{
  575. "ref_id": "ctx_000000000000000000000000",
  576. "version": "0" * 64,
  577. }],
  578. }
  579. result = await agent(
  580. task_brief=[BRIEF, invalid_ref_brief],
  581. context=self.context(),
  582. )
  583. self.assertEqual("failed", result["status"])
  584. self.assertIn("not authorized", result["error"])
  585. self.assertEqual([], await self.children())
  586. async def test_batch_reports_are_reviewed_one_by_one(self):
  587. briefs = [
  588. BRIEF,
  589. {
  590. "objective": "research second question",
  591. "reason": "the batch needs a second independent result",
  592. "completion_criteria": ["provide second conclusion"],
  593. "expected_outputs": ["a second conclusion"],
  594. },
  595. ]
  596. result = await agent(task_brief=briefs, context=self.context())
  597. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  598. self.assertEqual(2, len(result["task_reports"]))
  599. premature = await review_task_result(
  600. child_trace_id=child_ids[0],
  601. decision="ASCEND",
  602. reason="Cannot finish while another report is pending",
  603. context=self.context(),
  604. )
  605. self.assertEqual("failed", premature["status"])
  606. self.assertIn("final pending report", premature["error"])
  607. first_review = await review_task_result(
  608. child_trace_id=child_ids[0],
  609. decision="REJECT_REFINE",
  610. reason="No extra task needed for this report",
  611. context=self.context(),
  612. )
  613. self.assertEqual(1, first_review["pending_review_count"])
  614. tree = await self.store.get_goal_tree(self.root_id)
  615. self.assertEqual("pending_review", tree.find("1").status)
  616. final_review = await review_task_result(
  617. child_trace_id=child_ids[1],
  618. decision="ASCEND",
  619. reason="All completion criteria are satisfied",
  620. context=self.context(),
  621. )
  622. self.assertEqual(0, final_review["pending_review_count"])
  623. tree = await self.store.get_goal_tree(self.root_id)
  624. self.assertEqual("completed", tree.find("1").status)
  625. duplicate = await review_task_result(
  626. child_trace_id=child_ids[1],
  627. decision="ASCEND",
  628. reason="Duplicate review must fail",
  629. context=self.context(),
  630. )
  631. self.assertEqual("failed", duplicate["status"])
  632. self.assertIn("no pending TaskReport", duplicate["error"])
  633. async def test_terminal_review_waits_for_queued_next_action(self):
  634. result = await agent(
  635. task_brief=[
  636. BRIEF,
  637. {
  638. "objective": "second batch child",
  639. "reason": "exercise queued batch behavior",
  640. "completion_criteria": ["finish the second child"],
  641. "expected_outputs": ["the second child result"],
  642. },
  643. ],
  644. context=self.context(),
  645. )
  646. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  647. next_brief = {
  648. "objective": "approved follow-up",
  649. "reason": "the parent approved one follow-up",
  650. "completion_criteria": ["finish the approved follow-up"],
  651. "expected_outputs": ["the approved follow-up result"],
  652. }
  653. accepted = await review_task_result(
  654. child_trace_id=child_ids[0],
  655. decision="ACCEPT_REFINE",
  656. reason="A separate follow-up is required",
  657. approved_next_task=next_brief,
  658. context=self.context(),
  659. )
  660. self.assertEqual("completed", accepted["status"])
  661. blocked = await review_task_result(
  662. child_trace_id=child_ids[1],
  663. decision="ASCEND",
  664. reason="Cannot ascend before the accepted follow-up",
  665. context=self.context(),
  666. )
  667. self.assertEqual("failed", blocked["status"])
  668. self.assertIn("approved next action", blocked["error"])
  669. tree = await self.store.get_goal_tree(self.root_id)
  670. self.assertEqual("pending_review", tree.find("1").status)
  671. final_non_terminal = await review_task_result(
  672. child_trace_id=child_ids[1],
  673. decision="REJECT_REFINE",
  674. reason="This report adds no other follow-up",
  675. context=self.context(),
  676. )
  677. self.assertEqual("completed", final_non_terminal["status"])
  678. tree = await self.store.get_goal_tree(self.root_id)
  679. self.assertEqual("in_progress", tree.find("1").status)
  680. async def test_runner_failure_overrides_stale_satisfied_report(self):
  681. self.runner = FailedAfterReportingRunner(self.store)
  682. result = await agent(task_brief=BRIEF, context=self.context())
  683. self.assertEqual("failed", result["status"])
  684. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  685. child = await self.store.get_trace(result["sub_trace_id"])
  686. child_state = ensure_task_protocol(child.context)
  687. self.assertEqual(1, len(child_state["report_history"]))
  688. self.assertEqual("satisfied", child_state["report_history"][0]["outcome"])
  689. invalid_ascend = await review_task_result(
  690. child_trace_id=result["sub_trace_id"],
  691. decision="ASCEND",
  692. reason="A failed execution cannot ascend",
  693. context=self.context(),
  694. )
  695. self.assertEqual("failed", invalid_ascend["status"])
  696. self.assertIn("only allow", invalid_ascend["error"])
  697. async def test_submit_and_review_reject_child_anchor_that_differs_from_root(self):
  698. child_context = with_root_anchor({
  699. "agent_mode": "recursive",
  700. "agent_mode_revision": 2,
  701. "agent_depth": 1,
  702. "root_trace_id": self.root_id,
  703. "created_by_tool": "agent",
  704. "task_protocol": new_task_protocol(BRIEF),
  705. })
  706. persist_root_task_anchor(child_context, {
  707. **ROOT_ANCHOR,
  708. "objective": "self-consistent but unauthorized root objective",
  709. })
  710. child_id = "mismatched-anchor-child"
  711. await self.store.create_trace(Trace(
  712. trace_id=child_id,
  713. mode="agent",
  714. task="child",
  715. parent_trace_id=self.root_id,
  716. parent_goal_id="1",
  717. uid="user-1",
  718. model="fake-model",
  719. context=child_context,
  720. ))
  721. submitted = await submit_task_report(
  722. task_report=report_payload(),
  723. context={"store": self.store, "trace_id": child_id},
  724. )
  725. self.assertEqual("failed", submitted["status"])
  726. self.assertIn("does not match", submitted["error"])
  727. parent = await self.store.get_trace(self.root_id)
  728. parent_state = ensure_task_protocol(parent.context)
  729. parent_state["pending_reviews"][child_id] = {
  730. "goal_id": "1",
  731. "task_report": TaskReport(
  732. child_trace_id=child_id,
  733. **report_payload(),
  734. ).model_dump(),
  735. "validation_result": validation_payload(child_id),
  736. "received_at_sequence": 1,
  737. }
  738. await self.store.update_trace(self.root_id, context=parent.context)
  739. reviewed = await review_task_result(
  740. child_trace_id=child_id,
  741. decision="ASCEND",
  742. reason="must not accept a different root anchor",
  743. context=self.context(),
  744. )
  745. self.assertEqual("failed", reviewed["status"])
  746. self.assertIn("does not match", reviewed["error"])
  747. parent = await self.store.get_trace(self.root_id)
  748. state = ensure_task_protocol(parent.context)
  749. self.assertIn(child_id, state["pending_reviews"])
  750. self.assertEqual([], state["reviews"])
  751. async def test_review_rejects_wrong_owner_even_if_report_is_injected(self):
  752. first = await agent(task_brief=BRIEF, context=self.context())
  753. child_id = first["sub_trace_id"]
  754. await self.store.update_trace(child_id, uid="another-user")
  755. rejected = await review_task_result(
  756. child_trace_id=child_id,
  757. decision="ASCEND",
  758. reason="Owner mismatch must be rejected",
  759. context=self.context(),
  760. )
  761. self.assertEqual("failed", rejected["status"])
  762. self.assertIn("same owner", rejected["error"])
  763. async def test_report_identity_must_match_direct_child(self):
  764. self.runner = ForgedIdentityRunner(self.store)
  765. result = await agent(task_brief=BRIEF, context=self.context())
  766. child_id = result["sub_trace_id"]
  767. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  768. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  769. parent = await self.store.get_trace(self.root_id)
  770. pending_report = ensure_task_protocol(parent.context)["pending_reviews"][
  771. child_id
  772. ]["task_report"]
  773. pending_report["child_trace_id"] = "corrupted-after-recording"
  774. await self.store.update_trace(self.root_id, context=parent.context)
  775. rejected = await review_task_result(
  776. child_trace_id=child_id,
  777. decision="FAIL",
  778. reason="Corrupted identity cannot be reviewed",
  779. context=self.context(),
  780. )
  781. self.assertEqual("failed", rejected["status"])
  782. self.assertIn("does not match", rejected["error"])
  783. async def test_remote_agent_keeps_legacy_task_interface(self):
  784. remote_result = {
  785. "status": "completed",
  786. "summary": "remote result",
  787. }
  788. with patch(
  789. "cyber_agent.tools.builtin.subagent._run_remote_agent",
  790. new=AsyncMock(return_value=remote_result),
  791. ) as remote_call:
  792. result = await agent(
  793. task="remote free text",
  794. agent_type="remote_librarian",
  795. )
  796. self.assertEqual(remote_result, result)
  797. self.assertEqual("remote free text", remote_call.await_args.kwargs["task"])
  798. async def test_revise_child_reuses_trace_and_preserves_report_history(self):
  799. first = await agent(task_brief=BRIEF, context=self.context())
  800. child_id = first["sub_trace_id"]
  801. reviewed = await review_task_result(
  802. child_trace_id=child_id,
  803. decision="REVISE_CHILD",
  804. reason="Please strengthen the evidence",
  805. context=self.context(),
  806. )
  807. self.assertEqual("completed", reviewed["status"])
  808. cleared_child = await self.store.get_trace(child_id)
  809. self.assertEqual({}, cleared_child.context["context_access"]["snapshots"])
  810. wrong = await agent(
  811. task_brief={
  812. "objective": "silently replace the reviewed task",
  813. "reason": "attempt a mismatched revision",
  814. "completion_criteria": ["this must be rejected"],
  815. "expected_outputs": ["a rejected replacement"],
  816. },
  817. continue_from=child_id,
  818. context=self.context(),
  819. )
  820. self.assertEqual("failed", wrong["status"])
  821. self.assertIn("approved revision", wrong["error"])
  822. revised_brief = reviewed["next_action_task_brief"]
  823. revised = await agent(
  824. task_brief=revised_brief,
  825. continue_from=child_id,
  826. context=self.context(),
  827. )
  828. self.assertEqual(child_id, revised["sub_trace_id"])
  829. self.assertTrue(revised["continue_from"])
  830. self.assertEqual(1, len(await self.children()))
  831. child = await self.store.get_trace(child_id)
  832. self.assertEqual(1, len(ensure_task_protocol(child.context)["report_history"]))
  833. self.assertEqual(
  834. TaskBrief.model_validate(revised_brief).model_dump(),
  835. ensure_task_protocol(child.context)["task_brief"],
  836. )
  837. self.assertEqual(2, ensure_task_protocol(child.context)["task_brief_version"])
  838. self.assertEqual(
  839. 1,
  840. len(ensure_task_protocol(child.context)["task_brief_history"]),
  841. )
  842. async def test_review_cannot_reserve_more_children_than_quota(self):
  843. briefs = [
  844. {
  845. "objective": f"batch child {index}",
  846. "reason": "fill the local child quota",
  847. "completion_criteria": [f"finish child {index}"],
  848. "expected_outputs": [f"result for child {index}"],
  849. }
  850. for index in range(5)
  851. ]
  852. result = await agent(task_brief=briefs, context=self.context())
  853. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  854. next_brief = {
  855. "objective": "one allowed reserved child",
  856. "reason": "use the final available child slot",
  857. "completion_criteria": ["use the final available slot"],
  858. "expected_outputs": ["one final child result"],
  859. }
  860. first = await review_task_result(
  861. child_trace_id=child_ids[0],
  862. decision="ACCEPT_REFINE",
  863. reason="Reserve the sixth and final child slot",
  864. approved_next_task=next_brief,
  865. context=self.context(),
  866. )
  867. self.assertEqual("completed", first["status"])
  868. overflow = await review_task_result(
  869. child_trace_id=child_ids[1],
  870. decision="DESCEND_AGAIN",
  871. reason="A seventh child must not be approved",
  872. approved_next_task={
  873. "objective": "overflow child",
  874. "reason": "exercise quota rejection",
  875. "completion_criteria": ["must never be scheduled"],
  876. "expected_outputs": ["no output because creation is rejected"],
  877. },
  878. context=self.context(),
  879. )
  880. self.assertEqual("failed", overflow["status"])
  881. self.assertIn("reserved=1", overflow["error"])
  882. parent = await self.store.get_trace(self.root_id)
  883. state = ensure_task_protocol(parent.context)
  884. self.assertEqual(1, len(state["next_actions"]))
  885. self.assertIn(child_ids[1], state["pending_reviews"])
  886. async def test_recursive_goal_completion_does_not_cascade_or_select_sibling(self):
  887. tree = GoalTree(mission="recursive")
  888. parent = tree.add_goals(["parent"])[0]
  889. first, second = tree.add_goals(["first", "second"], parent_id=parent.id)
  890. tree.focus(parent.id)
  891. tree.focus(first.id)
  892. tree.complete(
  893. first.id,
  894. "done",
  895. auto_advance=False,
  896. cascade_parent=False,
  897. )
  898. self.assertEqual(parent.id, tree.current_id)
  899. self.assertEqual("pending", second.status)
  900. self.assertNotEqual("completed", parent.status)
  901. async def test_rewind_before_review_restores_report_and_review_gate(self):
  902. delegated = await agent(task_brief=BRIEF, context=self.context())
  903. child_id = delegated["sub_trace_id"]
  904. reviewed = await review_task_result(
  905. child_trace_id=child_id,
  906. decision="ASCEND",
  907. reason="Initially accepted before rewind",
  908. context={**self.context(), "sequence": 20},
  909. )
  910. self.assertEqual("completed", reviewed["status"])
  911. for sequence in (1, 15):
  912. await self.store.add_message(Message.create(
  913. trace_id=self.root_id,
  914. role="user",
  915. sequence=sequence,
  916. content=f"message-{sequence}",
  917. ))
  918. tree = await self.store.get_goal_tree(self.root_id)
  919. runner = AgentRunner(
  920. trace_store=self.store,
  921. tool_registry=get_tool_registry(),
  922. )
  923. await runner._rewind(self.root_id, 15, tree)
  924. parent = await self.store.get_trace(self.root_id)
  925. state = ensure_task_protocol(parent.context)
  926. self.assertIn(child_id, state["pending_reviews"])
  927. self.assertEqual([], state["reviews"])
  928. restored_tree = await self.store.get_goal_tree(self.root_id)
  929. self.assertEqual("pending_review", restored_tree.find("1").status)
  930. schemas = runner._get_runtime_tool_schemas(RunConfig(), parent)
  931. self.assertEqual(
  932. {"review_task_result", "read_context_ref"},
  933. {schema["function"]["name"] for schema in schemas},
  934. )
  935. async def test_grandchild_report_is_reviewed_only_by_direct_child(self):
  936. child_id = "manual-child"
  937. child_context = with_root_anchor({
  938. "agent_mode": "recursive",
  939. "agent_mode_revision": 2,
  940. "agent_depth": 1,
  941. "root_trace_id": self.root_id,
  942. "created_by_tool": "agent",
  943. "task_protocol": new_task_protocol(BRIEF),
  944. })
  945. await self.store.create_trace(Trace(
  946. trace_id=child_id,
  947. mode="agent",
  948. task="child",
  949. parent_trace_id=self.root_id,
  950. uid="user-1",
  951. model="fake-model",
  952. context=child_context,
  953. ))
  954. child_tree = GoalTree(
  955. mission="child",
  956. goals=[Goal(id="1", description="delegate grandchild", status="in_progress")],
  957. current_id="1",
  958. )
  959. await self.store.update_goal_tree(child_id, child_tree)
  960. grandchild = await agent(
  961. task_brief=BRIEF,
  962. context=self.context(trace_id=child_id, tree=child_tree),
  963. )
  964. child = await self.store.get_trace(child_id)
  965. child_state = ensure_task_protocol(child.context)
  966. self.assertIn(grandchild["sub_trace_id"], child_state["pending_reviews"])
  967. root = await self.store.get_trace(self.root_id)
  968. self.assertEqual({}, ensure_task_protocol(root.context)["pending_reviews"])
  969. blocked = await agent(
  970. task_brief=BRIEF,
  971. context=self.context(trace_id=child_id, tree=child_tree),
  972. )
  973. self.assertEqual("failed", blocked["status"])
  974. reviewed = await review_task_result(
  975. child_trace_id=grandchild["sub_trace_id"],
  976. decision="ASCEND",
  977. reason="Grandchild satisfied the child criteria",
  978. context=self.context(trace_id=child_id, tree=child_tree),
  979. )
  980. self.assertEqual("completed", reviewed["status"])
  981. submitted = await submit_task_report(
  982. task_report=json.dumps(report_payload()),
  983. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  984. )
  985. self.assertEqual("completed", submitted["status"])
  986. async def test_child_must_execute_approved_action_before_own_report(self):
  987. child_id = "child-with-approved-action"
  988. child_context = with_root_anchor({
  989. "agent_mode": "recursive",
  990. "agent_mode_revision": 2,
  991. "agent_depth": 1,
  992. "root_trace_id": self.root_id,
  993. "created_by_tool": "agent",
  994. "task_protocol": new_task_protocol(BRIEF),
  995. })
  996. await self.store.create_trace(Trace(
  997. trace_id=child_id,
  998. mode="agent",
  999. task="child",
  1000. parent_trace_id=self.root_id,
  1001. uid="user-1",
  1002. model="fake-model",
  1003. context=child_context,
  1004. ))
  1005. child_tree = GoalTree(
  1006. mission="child",
  1007. goals=[Goal(id="1", description="delegate", status="in_progress")],
  1008. current_id="1",
  1009. )
  1010. await self.store.update_goal_tree(child_id, child_tree)
  1011. child_tool_context = self.context(trace_id=child_id, tree=child_tree)
  1012. first = await agent(task_brief=BRIEF, context=child_tool_context)
  1013. next_brief = {
  1014. "objective": "check a second grandchild task",
  1015. "reason": "the child approved another direct task",
  1016. "completion_criteria": ["provide a second checked conclusion"],
  1017. "expected_outputs": ["a second checked conclusion"],
  1018. }
  1019. reviewed = await review_task_result(
  1020. child_trace_id=first["sub_trace_id"],
  1021. decision="ACCEPT_REFINE",
  1022. reason="A separate gap remains",
  1023. approved_next_task=next_brief,
  1024. context=child_tool_context,
  1025. )
  1026. self.assertEqual("completed", reviewed["status"])
  1027. blocked_report = await submit_task_report(
  1028. task_report=report_payload(),
  1029. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  1030. )
  1031. self.assertEqual("failed", blocked_report["status"])
  1032. self.assertIn("approved next action", blocked_report["error"])
  1033. second = await agent(
  1034. task_brief=reviewed["next_action_task_brief"],
  1035. context=child_tool_context,
  1036. )
  1037. self.assertEqual("completed", second["status"])
  1038. child = await self.store.get_trace(child_id)
  1039. state = ensure_task_protocol(child.context)
  1040. self.assertEqual([], state["next_actions"])
  1041. self.assertIn(second["sub_trace_id"], state["pending_reviews"])
  1042. async def test_goal_tool_cannot_bypass_queued_next_action(self):
  1043. root = await self.store.get_trace(self.root_id)
  1044. state = ensure_task_protocol(root.context)
  1045. state["next_actions"] = [{
  1046. "decision": "ACCEPT_REFINE",
  1047. "child_trace_id": "reviewed-child",
  1048. "goal_id": "1",
  1049. "task_brief": BRIEF,
  1050. "created_at_sequence": 5,
  1051. }]
  1052. await self.store.update_trace(self.root_id, context=root.context)
  1053. result = await goal_tool(
  1054. tree=self.tree,
  1055. store=self.store,
  1056. trace_id=self.root_id,
  1057. done="attempt to bypass the approved action",
  1058. )
  1059. self.assertIn("已批准的下一步待执行", result)
  1060. persisted_tree = await self.store.get_goal_tree(self.root_id)
  1061. self.assertEqual("in_progress", persisted_tree.find("1").status)
  1062. async def test_failed_report_allows_only_revise_or_fail(self):
  1063. child_id = "failed-child"
  1064. failed_report = TaskReport(
  1065. child_trace_id=child_id,
  1066. **report_payload(outcome="failed"),
  1067. )
  1068. child_context = with_root_anchor({
  1069. "agent_mode": "recursive",
  1070. "agent_mode_revision": 2,
  1071. "agent_depth": 1,
  1072. "root_trace_id": self.root_id,
  1073. "created_by_tool": "agent",
  1074. "task_protocol": new_task_protocol(BRIEF),
  1075. })
  1076. child_context["task_protocol"]["task_report"] = failed_report.model_dump()
  1077. failed_validation = validation_payload(child_id, outcome="failed")
  1078. child_context["task_protocol"]["task_report_validation"] = {
  1079. "validation_plan": None,
  1080. "plan_hash": failed_validation["plan_hash"],
  1081. "scope_results": failed_validation["scope_results"],
  1082. "aggregate_result": failed_validation,
  1083. "validated_at_sequence": 2,
  1084. }
  1085. await self.store.create_trace(Trace(
  1086. trace_id=child_id,
  1087. mode="agent",
  1088. task="failed",
  1089. parent_trace_id=self.root_id,
  1090. parent_goal_id="1",
  1091. uid="user-1",
  1092. context=child_context,
  1093. ))
  1094. root = await self.store.get_trace(self.root_id)
  1095. state = ensure_task_protocol(root.context)
  1096. state["pending_reviews"][child_id] = {
  1097. "goal_id": "1",
  1098. "task_report": failed_report.model_dump(),
  1099. "validation_result": failed_validation,
  1100. "received_at_sequence": 2,
  1101. }
  1102. await self.store.update_trace(self.root_id, context=root.context)
  1103. rejected = await review_task_result(
  1104. child_trace_id=child_id,
  1105. decision="ASCEND",
  1106. reason="incorrect",
  1107. context=self.context(),
  1108. )
  1109. self.assertEqual("failed", rejected["status"])
  1110. self.assertIn("only allow", rejected["error"])
  1111. class RunnerCompletionGateTest(unittest.IsolatedAsyncioTestCase):
  1112. async def asyncSetUp(self):
  1113. self.temp_dir = tempfile.TemporaryDirectory()
  1114. self.store = FileSystemTraceStore(self.temp_dir.name)
  1115. self.trace_id = "gate-child"
  1116. await create_recursive_root(self.store, "root")
  1117. controller = ResourceBudgetController(self.store)
  1118. budget = ResourceBudget()
  1119. await controller.reserve_agents("root", budget, 1)
  1120. await self.store.create_trace(Trace(
  1121. trace_id=self.trace_id,
  1122. mode="agent",
  1123. task="must report",
  1124. parent_trace_id="root",
  1125. uid="user-1",
  1126. context=with_root_anchor({
  1127. "agent_mode": "recursive",
  1128. "agent_mode_revision": 2,
  1129. "agent_depth": 1,
  1130. "root_trace_id": "root",
  1131. "created_by_tool": "agent",
  1132. "task_protocol": new_task_protocol(BRIEF),
  1133. }),
  1134. ))
  1135. await self.store.update_goal_tree(self.trace_id, GoalTree(mission="must report"))
  1136. async def asyncTearDown(self):
  1137. self.temp_dir.cleanup()
  1138. async def test_missing_report_becomes_protocol_error_after_two_corrections(self):
  1139. calls = 0
  1140. async def llm_call(**kwargs):
  1141. nonlocal calls
  1142. calls += 1
  1143. return {
  1144. "content": "I am done",
  1145. "tool_calls": [],
  1146. "finish_reason": "stop",
  1147. }
  1148. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1149. result = await runner.run_result(
  1150. messages=[],
  1151. config=RunConfig(
  1152. trace_id=self.trace_id,
  1153. max_iterations=8,
  1154. knowledge=KnowledgeConfig(
  1155. enable_extraction=False,
  1156. enable_completion_extraction=False,
  1157. enable_injection=False,
  1158. ),
  1159. ),
  1160. )
  1161. self.assertEqual("failed", result["status"])
  1162. trace = await self.store.get_trace(self.trace_id)
  1163. report = ensure_task_protocol(trace.context)["task_report"]
  1164. self.assertEqual("protocol_error", report["outcome"])
  1165. self.assertGreaterEqual(calls, 3)
  1166. async def test_valid_report_allows_child_to_finish(self):
  1167. calls = 0
  1168. async def llm_call(**kwargs):
  1169. nonlocal calls
  1170. calls += 1
  1171. messages = kwargs.get("messages") or []
  1172. if messages and "recursive_validation_protocol" in messages[0]["content"]:
  1173. return passed_validator_response(kwargs)
  1174. if calls == 1:
  1175. return {
  1176. "content": "",
  1177. "tool_calls": [{
  1178. "id": "submit-1",
  1179. "type": "function",
  1180. "function": {
  1181. "name": "submit_task_report",
  1182. "arguments": json.dumps({
  1183. "task_report": report_payload(),
  1184. }),
  1185. },
  1186. }],
  1187. "finish_reason": "tool_calls",
  1188. }
  1189. return {
  1190. "content": "Structured report submitted.",
  1191. "tool_calls": [],
  1192. "finish_reason": "stop",
  1193. }
  1194. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1195. result = await runner.run_result(
  1196. messages=[],
  1197. config=RunConfig(
  1198. trace_id=self.trace_id,
  1199. max_iterations=4,
  1200. knowledge=KnowledgeConfig(
  1201. enable_extraction=False,
  1202. enable_completion_extraction=False,
  1203. enable_injection=False,
  1204. ),
  1205. ),
  1206. )
  1207. self.assertEqual("completed", result["status"])
  1208. trace = await self.store.get_trace(self.trace_id)
  1209. self.assertEqual(
  1210. "satisfied",
  1211. ensure_task_protocol(trace.context)["task_report"]["outcome"],
  1212. )
  1213. async def test_lifecycle_tool_batch_is_rejected_before_execution(self):
  1214. root_id = "batch-gate-root"
  1215. await create_recursive_root(self.store, root_id)
  1216. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  1217. calls = 0
  1218. async def llm_call(**kwargs):
  1219. nonlocal calls
  1220. calls += 1
  1221. messages = kwargs.get("messages") or []
  1222. if messages and "recursive_validation_protocol" in messages[0]["content"]:
  1223. return passed_validator_response(kwargs)
  1224. if calls == 1:
  1225. return {
  1226. "content": "",
  1227. "tool_calls": [
  1228. {
  1229. "id": "agent-1",
  1230. "type": "function",
  1231. "function": {
  1232. "name": "agent",
  1233. "arguments": json.dumps({
  1234. "task_brief": BRIEF,
  1235. }),
  1236. },
  1237. },
  1238. {
  1239. "id": "context-1",
  1240. "type": "function",
  1241. "function": {
  1242. "name": "get_current_context",
  1243. "arguments": "{}",
  1244. },
  1245. },
  1246. ],
  1247. "finish_reason": "tool_calls",
  1248. }
  1249. return {
  1250. "content": "Corrected the invalid batch.",
  1251. "tool_calls": [],
  1252. "finish_reason": "stop",
  1253. }
  1254. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1255. result = await runner.run_result(
  1256. messages=[],
  1257. config=RunConfig(
  1258. trace_id=root_id,
  1259. max_iterations=3,
  1260. knowledge=KnowledgeConfig(
  1261. enable_extraction=False,
  1262. enable_completion_extraction=False,
  1263. enable_injection=False,
  1264. ),
  1265. ),
  1266. )
  1267. self.assertEqual("completed", result["status"])
  1268. children = await self.store.list_traces(
  1269. parent_trace_id=root_id,
  1270. created_by_tool="agent",
  1271. limit=10,
  1272. )
  1273. self.assertEqual([], children)
  1274. tool_messages = [
  1275. message
  1276. for message in await self.store.get_trace_messages(root_id)
  1277. if message.role == "tool"
  1278. ]
  1279. self.assertEqual(2, len(tool_messages))
  1280. self.assertTrue(all(
  1281. "only tool call" in str(message.content)
  1282. for message in tool_messages
  1283. ))
  1284. async def test_rewind_clears_protocol_state_after_cutoff(self):
  1285. trace = await self.store.get_trace(self.trace_id)
  1286. state = ensure_task_protocol(trace.context)
  1287. original_brief = TaskBrief.model_validate(state["task_brief"])
  1288. revised_brief = original_brief.model_copy(
  1289. update={"objective": "revised after the rewind cutoff"}
  1290. )
  1291. replace_task_brief(
  1292. state,
  1293. revised_brief,
  1294. effective_at_sequence=8,
  1295. )
  1296. late_snapshot = create_context_snapshot(
  1297. kind="reviewed_task_result",
  1298. summary="granted after cutoff",
  1299. source_trace_id="late-child",
  1300. root_trace_id="root",
  1301. uid="user-1",
  1302. content={"result": "must be removed by rewind"},
  1303. granted_at_sequence=8,
  1304. )
  1305. add_context_snapshot(
  1306. trace.context,
  1307. late_snapshot,
  1308. root_task_anchor=ROOT_ANCHOR,
  1309. task_brief=revised_brief,
  1310. )
  1311. report = TaskReport(
  1312. child_trace_id=self.trace_id,
  1313. **report_payload(),
  1314. ).model_dump()
  1315. state["task_report"] = report
  1316. state["task_report_submitted_at_sequence"] = 5
  1317. state["pending_reviews"] = {
  1318. "grandchild": {
  1319. "goal_id": "1",
  1320. "task_report": report,
  1321. "validation_result": validation_payload("grandchild"),
  1322. "received_at_sequence": 6,
  1323. }
  1324. }
  1325. state["reviews"] = [{"reviewed_at_sequence": 7}]
  1326. state["next_actions"] = [{"created_at_sequence": 8}]
  1327. await self.store.update_trace(self.trace_id, context=trace.context)
  1328. tree = GoalTree(
  1329. mission="rewind",
  1330. goals=[Goal(id="1", description="active", status="pending_review")],
  1331. current_id="1",
  1332. )
  1333. await self.store.update_goal_tree(self.trace_id, tree)
  1334. for sequence in (1, 2):
  1335. await self.store.add_message(Message.create(
  1336. trace_id=self.trace_id,
  1337. role="user",
  1338. sequence=sequence,
  1339. content=f"message-{sequence}",
  1340. ))
  1341. runner = AgentRunner(trace_store=self.store)
  1342. cutoff = await runner._rewind(self.trace_id, 2, tree)
  1343. self.assertEqual(3, cutoff)
  1344. rewound = await self.store.get_trace(self.trace_id)
  1345. state = ensure_task_protocol(rewound.context)
  1346. self.assertIsNone(state["task_report"])
  1347. self.assertEqual(1, len(state["report_history"]))
  1348. self.assertEqual({}, state["pending_reviews"])
  1349. self.assertEqual([], state["reviews"])
  1350. self.assertEqual([], state["next_actions"])
  1351. self.assertEqual(1, state["task_brief_version"])
  1352. self.assertEqual(original_brief.model_dump(), state["task_brief"])
  1353. self.assertEqual([], state["task_brief_history"])
  1354. self.assertEqual({}, rewound.context["context_access"]["snapshots"])
  1355. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  1356. self.assertEqual("pending", rewound_tree.find("1").status)
  1357. async def test_rewind_restores_preserved_pending_review_projection(self):
  1358. trace = await self.store.get_trace(self.trace_id)
  1359. state = ensure_task_protocol(trace.context)
  1360. report = TaskReport(
  1361. child_trace_id="preserved-grandchild",
  1362. **report_payload(),
  1363. ).model_dump()
  1364. state["pending_reviews"] = {
  1365. "preserved-grandchild": {
  1366. "goal_id": "1",
  1367. "task_report": report,
  1368. "validation_result": validation_payload(
  1369. "preserved-grandchild"
  1370. ),
  1371. "received_at_sequence": 2,
  1372. }
  1373. }
  1374. await self.store.update_trace(self.trace_id, context=trace.context)
  1375. tree = GoalTree(
  1376. mission="rewind",
  1377. goals=[Goal(id="1", description="review", status="pending_review")],
  1378. current_id="1",
  1379. )
  1380. await self.store.update_goal_tree(self.trace_id, tree)
  1381. for sequence in (1, 2):
  1382. await self.store.add_message(Message.create(
  1383. trace_id=self.trace_id,
  1384. role="user",
  1385. sequence=sequence,
  1386. content=f"message-{sequence}",
  1387. ))
  1388. runner = AgentRunner(trace_store=self.store)
  1389. await runner._rewind(self.trace_id, 2, tree)
  1390. rewound = await self.store.get_trace(self.trace_id)
  1391. self.assertIn(
  1392. "preserved-grandchild",
  1393. ensure_task_protocol(rewound.context)["pending_reviews"],
  1394. )
  1395. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  1396. self.assertEqual("pending_review", rewound_tree.find("1").status)
  1397. async def test_rewind_rebuilds_root_validation_history_and_cache(self):
  1398. root_id = "root-validation-rewind"
  1399. state = new_task_protocol()
  1400. passed = validation_payload(root_id, outcome="passed")
  1401. failed = validation_payload(root_id, outcome="failed")
  1402. state["root_validation_history"] = [
  1403. {**passed, "validated_at_sequence": 2},
  1404. {**failed, "validated_at_sequence": 5},
  1405. ]
  1406. state["root_validation_attempts"] = 2
  1407. state["root_validation_passed"] = False
  1408. state["task_report_validation"] = {
  1409. "plan_hash": failed["plan_hash"],
  1410. "aggregate_result": failed,
  1411. "scope_results": failed["scope_results"],
  1412. "validated_at_sequence": 5,
  1413. }
  1414. await create_recursive_root(self.store, root_id, state=state)
  1415. tree = GoalTree(
  1416. mission="root",
  1417. goals=[Goal(id="1", description="root", status="in_progress")],
  1418. current_id="1",
  1419. )
  1420. await self.store.update_goal_tree(root_id, tree)
  1421. for sequence in (1, 2, 3):
  1422. await self.store.add_message(Message.create(
  1423. trace_id=root_id,
  1424. role="user",
  1425. sequence=sequence,
  1426. content=f"message-{sequence}",
  1427. ))
  1428. await AgentRunner(trace_store=self.store)._rewind(root_id, 3, tree)
  1429. rewound = await self.store.get_trace(root_id)
  1430. state = ensure_task_protocol(rewound.context)
  1431. self.assertEqual(1, state["root_validation_attempts"])
  1432. self.assertTrue(state["root_validation_passed"])
  1433. self.assertEqual(
  1434. ["passed"],
  1435. [item["outcome"] for item in state["root_validation_history"]],
  1436. )
  1437. self.assertIsNone(state["task_report_validation"])
  1438. async def test_root_cannot_finish_with_unconsumed_next_action(self):
  1439. root_id = "next-action-gate-root"
  1440. state = new_task_protocol()
  1441. state["next_actions"] = [{
  1442. "decision": "ACCEPT_REFINE",
  1443. "child_trace_id": "reviewed-child",
  1444. "goal_id": None,
  1445. "task_brief": BRIEF,
  1446. "created_at_sequence": 1,
  1447. }]
  1448. await create_recursive_root(self.store, root_id, state=state)
  1449. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  1450. calls = 0
  1451. async def llm_call(**kwargs):
  1452. nonlocal calls
  1453. calls += 1
  1454. return {
  1455. "content": "Trying to finish without the approved action.",
  1456. "tool_calls": [],
  1457. "finish_reason": "stop",
  1458. }
  1459. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1460. result = await runner.run_result(
  1461. messages=[],
  1462. config=RunConfig(
  1463. trace_id=root_id,
  1464. max_iterations=5,
  1465. knowledge=KnowledgeConfig(
  1466. enable_extraction=False,
  1467. enable_completion_extraction=False,
  1468. enable_injection=False,
  1469. ),
  1470. ),
  1471. )
  1472. self.assertEqual("failed", result["status"])
  1473. self.assertEqual(3, calls)
  1474. root = await self.store.get_trace(root_id)
  1475. self.assertEqual(1, len(ensure_task_protocol(root.context)["next_actions"]))
  1476. class RuntimeProtocolSchemaTest(unittest.TestCase):
  1477. def setUp(self):
  1478. self.runner = AgentRunner(tool_registry=get_tool_registry())
  1479. def names(self, schemas):
  1480. return {schema["function"]["name"] for schema in schemas}
  1481. def test_legacy_and_recursive_runtime_schemas_are_isolated(self):
  1482. legacy = Trace(
  1483. trace_id="legacy",
  1484. mode="agent",
  1485. context={"agent_mode": "legacy", "agent_mode_revision": 1},
  1486. )
  1487. legacy_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), legacy)
  1488. self.assertNotIn("submit_task_report", self.names(legacy_schemas))
  1489. self.assertNotIn("review_task_result", self.names(legacy_schemas))
  1490. legacy_agent = next(
  1491. item for item in legacy_schemas if item["function"]["name"] == "agent"
  1492. )
  1493. properties = legacy_agent["function"]["parameters"]["properties"]
  1494. self.assertIn("task", properties)
  1495. self.assertNotIn("task_brief", properties)
  1496. child = Trace(
  1497. trace_id="child",
  1498. mode="agent",
  1499. parent_trace_id="root",
  1500. context={
  1501. "agent_mode": "recursive",
  1502. "agent_mode_revision": 2,
  1503. "task_protocol": new_task_protocol(BRIEF),
  1504. },
  1505. )
  1506. child_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1507. self.assertIn("submit_task_report", self.names(child_schemas))
  1508. self.assertNotIn("review_task_result", self.names(child_schemas))
  1509. self.assertNotIn("evaluate", self.names(child_schemas))
  1510. recursive_agent = next(
  1511. item for item in child_schemas if item["function"]["name"] == "agent"
  1512. )
  1513. recursive_properties = recursive_agent["function"]["parameters"]["properties"]
  1514. self.assertNotIn("messages", recursive_properties)
  1515. brief_object = recursive_properties["task_brief"]["oneOf"][0]
  1516. self.assertEqual(
  1517. {
  1518. "objective",
  1519. "reason",
  1520. "completion_criteria",
  1521. "expected_outputs",
  1522. },
  1523. set(brief_object["required"]),
  1524. )
  1525. state = ensure_task_protocol(child.context)
  1526. state["pending_reviews"]["grandchild"] = {"task_report": {}}
  1527. review_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1528. self.assertEqual(
  1529. {"review_task_result", "read_context_ref"},
  1530. self.names(review_schemas),
  1531. )
  1532. def test_protocol_tools_publish_nested_object_schemas(self):
  1533. schemas = {
  1534. schema["function"]["name"]: schema
  1535. for schema in get_tool_registry().get_schemas([
  1536. "submit_task_report",
  1537. "review_task_result",
  1538. ])
  1539. }
  1540. report_schema = schemas["submit_task_report"]["function"]["parameters"]
  1541. task_report = report_schema["properties"]["task_report"]
  1542. self.assertEqual("object", task_report["type"])
  1543. self.assertIn("validation", task_report["properties"])
  1544. self.assertIn("next_step_suggestion", task_report["properties"])
  1545. review_schema = schemas["review_task_result"]["function"]["parameters"]
  1546. self.assertEqual(
  1547. [
  1548. "ACCEPT_REFINE",
  1549. "REJECT_REFINE",
  1550. "REVISE_CHILD",
  1551. "DESCEND_AGAIN",
  1552. "REPLAN_CURRENT",
  1553. "ASCEND",
  1554. "FAIL",
  1555. ],
  1556. review_schema["properties"]["decision"]["enum"],
  1557. )
  1558. self.assertEqual(
  1559. "object",
  1560. review_schema["properties"]["approved_next_task"]["type"],
  1561. )
  1562. if __name__ == "__main__":
  1563. unittest.main()