test_task_protocol.py 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195
  1. import json
  2. import tempfile
  3. import unittest
  4. from unittest.mock import AsyncMock, patch
  5. from pydantic import ValidationError
  6. from cyber_agent.core.runner import AgentRunner, RunConfig
  7. from cyber_agent.core.task_protocol import (
  8. TaskBrief,
  9. TaskReport,
  10. TaskReview,
  11. ensure_task_protocol,
  12. new_task_protocol,
  13. )
  14. from cyber_agent.tools import get_tool_registry
  15. from cyber_agent.tools.builtin.knowledge import KnowledgeConfig
  16. from cyber_agent.tools.builtin.subagent import agent
  17. from cyber_agent.tools.builtin.task_protocol import (
  18. review_task_result,
  19. submit_task_report,
  20. )
  21. from cyber_agent.trace.goal_models import Goal, GoalTree
  22. from cyber_agent.trace.goal_tool import goal_tool
  23. from cyber_agent.trace.models import Message, Trace
  24. from cyber_agent.trace.store import FileSystemTraceStore
  25. BRIEF = {
  26. "objective": "research one concrete question",
  27. "completion_criteria": ["provide one checked conclusion"],
  28. "constraints": ["do not guess"],
  29. }
  30. def report_payload(*, outcome="satisfied"):
  31. return {
  32. "summary": "checked result",
  33. "outcome": outcome,
  34. "validation": {
  35. "hard_passed": outcome == "satisfied",
  36. "open_issues": [] if outcome == "satisfied" else ["still failing"],
  37. },
  38. "next_step_suggestion": {
  39. "direction": "NONE",
  40. "reason": "No additional task is needed",
  41. },
  42. "outputs": [{"kind": "note", "value": "result"}],
  43. "evidence": [{"kind": "test", "value": "passed"}],
  44. "remaining_issues": [] if outcome == "satisfied" else ["still failing"],
  45. }
  46. class FakeToolRegistry:
  47. def get_tool_names(self):
  48. return [
  49. "agent",
  50. "evaluate",
  51. "bash_command",
  52. "goal",
  53. "read_file",
  54. "submit_task_report",
  55. "review_task_result",
  56. ]
  57. class ReportingRunner:
  58. def __init__(self, store):
  59. self.store = store
  60. self.tools = FakeToolRegistry()
  61. self.debug = False
  62. self.stdin_check = None
  63. self._cancel_events = {}
  64. self.configs = []
  65. async def run_result(self, messages, config, on_event=None):
  66. self.configs.append(config)
  67. submitted = await submit_task_report(
  68. task_report=report_payload(),
  69. context={
  70. "store": self.store,
  71. "trace_id": config.trace_id,
  72. "sequence": 3,
  73. },
  74. )
  75. if submitted["status"] != "completed":
  76. raise AssertionError(submitted)
  77. await self.store.update_trace(config.trace_id, status="completed")
  78. return {
  79. "status": "completed",
  80. "summary": "child submitted TaskReport",
  81. "saved_knowledge_ids": [],
  82. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  83. }
  84. class FailedAfterReportingRunner(ReportingRunner):
  85. async def run_result(self, messages, config, on_event=None):
  86. result = await super().run_result(messages, config, on_event)
  87. await self.store.update_trace(
  88. config.trace_id,
  89. status="failed",
  90. error_message="runner failed after report submission",
  91. )
  92. return {
  93. **result,
  94. "status": "failed",
  95. "error": "runner failed after report submission",
  96. }
  97. class ForgedIdentityRunner(ReportingRunner):
  98. async def run_result(self, messages, config, on_event=None):
  99. trace = await self.store.get_trace(config.trace_id)
  100. state = ensure_task_protocol(trace.context)
  101. state["task_report"] = TaskReport(
  102. child_trace_id="someone-elses-trace",
  103. **report_payload(),
  104. ).model_dump()
  105. await self.store.update_trace(
  106. config.trace_id,
  107. context=trace.context,
  108. status="completed",
  109. )
  110. return {
  111. "status": "completed",
  112. "summary": "forged report identity",
  113. "saved_knowledge_ids": [],
  114. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  115. }
  116. class RecursiveTaskProtocolTest(unittest.IsolatedAsyncioTestCase):
  117. async def asyncSetUp(self):
  118. self.temp_dir = tempfile.TemporaryDirectory()
  119. self.store = FileSystemTraceStore(self.temp_dir.name)
  120. self.runner = ReportingRunner(self.store)
  121. self.root_id = "recursive-root"
  122. context = {
  123. "agent_mode": "recursive",
  124. "agent_mode_revision": 2,
  125. "agent_depth": 0,
  126. "root_trace_id": self.root_id,
  127. "task_protocol": new_task_protocol(),
  128. }
  129. await self.store.create_trace(Trace(
  130. trace_id=self.root_id,
  131. mode="agent",
  132. task="root",
  133. uid="user-1",
  134. model="fake-model",
  135. context=context,
  136. ))
  137. self.tree = GoalTree(
  138. mission="root",
  139. goals=[Goal(id="1", description="delegate", status="in_progress")],
  140. current_id="1",
  141. )
  142. await self.store.update_goal_tree(self.root_id, self.tree)
  143. async def asyncTearDown(self):
  144. self.temp_dir.cleanup()
  145. def context(self, trace_id=None, goal_id="1", tree=None):
  146. return {
  147. "store": self.store,
  148. "trace_id": trace_id or self.root_id,
  149. "goal_id": goal_id,
  150. "runner": self.runner,
  151. "goal_tree": tree or self.tree,
  152. "sequence": 10,
  153. }
  154. async def children(self):
  155. return await self.store.list_traces(
  156. parent_trace_id=self.root_id,
  157. created_by_tool="agent",
  158. limit=20,
  159. )
  160. async def test_models_validate_and_round_trip(self):
  161. brief = TaskBrief.model_validate(BRIEF)
  162. self.assertEqual(BRIEF, brief.model_dump(exclude_none=True))
  163. with self.assertRaises(ValidationError):
  164. TaskBrief(objective="", completion_criteria=[])
  165. with self.assertRaises(ValidationError):
  166. TaskReview(
  167. parent_trace_id="p",
  168. child_trace_id="c",
  169. decision="ACCEPT_REFINE",
  170. reason="continue",
  171. )
  172. invalid_suggestion = report_payload()
  173. invalid_suggestion["next_step_suggestion"] = {
  174. "direction": "NONE",
  175. "reason": "No next task should be attached",
  176. "suggested_next_task": BRIEF,
  177. }
  178. with self.assertRaises(ValidationError):
  179. TaskReport(child_trace_id="child", **invalid_suggestion)
  180. root_submission = await submit_task_report(
  181. task_report=report_payload(),
  182. context={"store": self.store, "trace_id": self.root_id},
  183. )
  184. self.assertEqual("failed", root_submission["status"])
  185. self.assertIn("Root Agent", root_submission["error"])
  186. async def test_child_report_blocks_new_child_until_parent_review(self):
  187. result = await agent(task_brief=BRIEF, context=self.context())
  188. self.assertEqual("completed", result["status"])
  189. child_id = result["sub_trace_id"]
  190. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  191. parent = await self.store.get_trace(self.root_id)
  192. state = ensure_task_protocol(parent.context)
  193. self.assertEqual([child_id], list(state["pending_reviews"]))
  194. tree = await self.store.get_goal_tree(self.root_id)
  195. self.assertEqual("pending_review", tree.find("1").status)
  196. restarted_runner = AgentRunner(tool_registry=get_tool_registry())
  197. restored = await self.store.get_trace(self.root_id)
  198. restored_names = {
  199. schema["function"]["name"]
  200. for schema in restarted_runner._get_runtime_tool_schemas(
  201. RunConfig(), restored
  202. )
  203. }
  204. self.assertEqual({"review_task_result"}, restored_names)
  205. blocked = await agent(task_brief=BRIEF, context=self.context())
  206. self.assertEqual("failed", blocked["status"])
  207. self.assertIn("Review pending", blocked["error"])
  208. async def test_qwen_json_string_containers_are_normalized(self):
  209. result = await agent(
  210. task_brief=json.dumps(BRIEF),
  211. messages=json.dumps([{
  212. "role": "system",
  213. "content": "controlled child context",
  214. }]),
  215. context=self.context(),
  216. )
  217. self.assertEqual("completed", result["status"])
  218. child = await self.store.get_trace(result["sub_trace_id"])
  219. self.assertEqual(
  220. TaskBrief.model_validate(BRIEF).model_dump(),
  221. ensure_task_protocol(child.context)["task_brief"],
  222. )
  223. async def test_messages_reject_mixed_dimensions(self):
  224. mixed_messages = [
  225. {"role": "system", "content": "shared context"},
  226. [{"role": "user", "content": "per-agent context"}],
  227. ]
  228. for messages in (mixed_messages, json.dumps(mixed_messages)):
  229. with self.subTest(messages_type=type(messages).__name__):
  230. result = await agent(
  231. task_brief=BRIEF,
  232. messages=messages,
  233. context=self.context(),
  234. )
  235. self.assertEqual("failed", result["status"])
  236. self.assertIn("without mixed dimensions", result["error"])
  237. async def test_no_focused_goal_still_persists_and_reviews_report(self):
  238. result = await agent(
  239. task_brief=BRIEF,
  240. context=self.context(goal_id=None),
  241. )
  242. parent = await self.store.get_trace(self.root_id)
  243. entry = ensure_task_protocol(parent.context)["pending_reviews"][
  244. result["sub_trace_id"]
  245. ]
  246. self.assertIsNone(entry["goal_id"])
  247. reviewed = await review_task_result(
  248. child_trace_id=result["sub_trace_id"],
  249. decision="ASCEND",
  250. reason="Standalone delegation is complete",
  251. context=self.context(goal_id=None),
  252. )
  253. self.assertEqual("completed", reviewed["status"])
  254. async def test_accept_refine_approves_exactly_one_next_child(self):
  255. first = await agent(task_brief=BRIEF, context=self.context())
  256. next_brief = {
  257. "objective": "check the independent gap",
  258. "completion_criteria": ["close the gap"],
  259. }
  260. parent = await self.store.get_trace(self.root_id)
  261. ensure_task_protocol(parent.context)["protocol_correction_attempts"] = 2
  262. await self.store.update_trace(self.root_id, context=parent.context)
  263. reviewed = await review_task_result(
  264. child_trace_id=first["sub_trace_id"],
  265. decision="ACCEPT_REFINE",
  266. reason="The gap belongs to this parent",
  267. approved_next_task=json.dumps(next_brief),
  268. context=self.context(),
  269. )
  270. self.assertEqual("completed", reviewed["status"])
  271. parent = await self.store.get_trace(self.root_id)
  272. self.assertEqual(
  273. 0,
  274. ensure_task_protocol(parent.context)["protocol_correction_attempts"],
  275. )
  276. schemas = AgentRunner(
  277. tool_registry=get_tool_registry()
  278. )._get_runtime_tool_schemas(RunConfig(), parent)
  279. self.assertEqual(
  280. {"agent"},
  281. {schema["function"]["name"] for schema in schemas},
  282. )
  283. wrong = await agent(task_brief=BRIEF, context=self.context())
  284. self.assertEqual("failed", wrong["status"])
  285. self.assertIn("approved next task", wrong["error"])
  286. second = await agent(task_brief=next_brief, context=self.context())
  287. self.assertEqual("completed", second["status"])
  288. self.assertEqual(2, len(await self.children()))
  289. parent = await self.store.get_trace(self.root_id)
  290. self.assertEqual([], ensure_task_protocol(parent.context)["next_actions"])
  291. async def test_batch_reports_are_reviewed_one_by_one(self):
  292. briefs = [
  293. BRIEF,
  294. {
  295. "objective": "research second question",
  296. "completion_criteria": ["provide second conclusion"],
  297. },
  298. ]
  299. result = await agent(task_brief=briefs, context=self.context())
  300. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  301. self.assertEqual(2, len(result["task_reports"]))
  302. premature = await review_task_result(
  303. child_trace_id=child_ids[0],
  304. decision="ASCEND",
  305. reason="Cannot finish while another report is pending",
  306. context=self.context(),
  307. )
  308. self.assertEqual("failed", premature["status"])
  309. self.assertIn("final pending report", premature["error"])
  310. first_review = await review_task_result(
  311. child_trace_id=child_ids[0],
  312. decision="REJECT_REFINE",
  313. reason="No extra task needed for this report",
  314. context=self.context(),
  315. )
  316. self.assertEqual(1, first_review["pending_review_count"])
  317. tree = await self.store.get_goal_tree(self.root_id)
  318. self.assertEqual("pending_review", tree.find("1").status)
  319. final_review = await review_task_result(
  320. child_trace_id=child_ids[1],
  321. decision="ASCEND",
  322. reason="All completion criteria are satisfied",
  323. context=self.context(),
  324. )
  325. self.assertEqual(0, final_review["pending_review_count"])
  326. tree = await self.store.get_goal_tree(self.root_id)
  327. self.assertEqual("completed", tree.find("1").status)
  328. duplicate = await review_task_result(
  329. child_trace_id=child_ids[1],
  330. decision="ASCEND",
  331. reason="Duplicate review must fail",
  332. context=self.context(),
  333. )
  334. self.assertEqual("failed", duplicate["status"])
  335. self.assertIn("no pending TaskReport", duplicate["error"])
  336. async def test_terminal_review_waits_for_queued_next_action(self):
  337. result = await agent(
  338. task_brief=[
  339. BRIEF,
  340. {
  341. "objective": "second batch child",
  342. "completion_criteria": ["finish the second child"],
  343. },
  344. ],
  345. context=self.context(),
  346. )
  347. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  348. next_brief = {
  349. "objective": "approved follow-up",
  350. "completion_criteria": ["finish the approved follow-up"],
  351. }
  352. accepted = await review_task_result(
  353. child_trace_id=child_ids[0],
  354. decision="ACCEPT_REFINE",
  355. reason="A separate follow-up is required",
  356. approved_next_task=next_brief,
  357. context=self.context(),
  358. )
  359. self.assertEqual("completed", accepted["status"])
  360. blocked = await review_task_result(
  361. child_trace_id=child_ids[1],
  362. decision="ASCEND",
  363. reason="Cannot ascend before the accepted follow-up",
  364. context=self.context(),
  365. )
  366. self.assertEqual("failed", blocked["status"])
  367. self.assertIn("approved next action", blocked["error"])
  368. tree = await self.store.get_goal_tree(self.root_id)
  369. self.assertEqual("pending_review", tree.find("1").status)
  370. final_non_terminal = await review_task_result(
  371. child_trace_id=child_ids[1],
  372. decision="REJECT_REFINE",
  373. reason="This report adds no other follow-up",
  374. context=self.context(),
  375. )
  376. self.assertEqual("completed", final_non_terminal["status"])
  377. tree = await self.store.get_goal_tree(self.root_id)
  378. self.assertEqual("in_progress", tree.find("1").status)
  379. async def test_runner_failure_overrides_stale_satisfied_report(self):
  380. self.runner = FailedAfterReportingRunner(self.store)
  381. result = await agent(task_brief=BRIEF, context=self.context())
  382. self.assertEqual("failed", result["status"])
  383. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  384. child = await self.store.get_trace(result["sub_trace_id"])
  385. child_state = ensure_task_protocol(child.context)
  386. self.assertEqual(1, len(child_state["report_history"]))
  387. self.assertEqual("satisfied", child_state["report_history"][0]["outcome"])
  388. invalid_ascend = await review_task_result(
  389. child_trace_id=result["sub_trace_id"],
  390. decision="ASCEND",
  391. reason="A failed execution cannot ascend",
  392. context=self.context(),
  393. )
  394. self.assertEqual("failed", invalid_ascend["status"])
  395. self.assertIn("only allow", invalid_ascend["error"])
  396. async def test_review_rejects_wrong_owner_even_if_report_is_injected(self):
  397. first = await agent(task_brief=BRIEF, context=self.context())
  398. child_id = first["sub_trace_id"]
  399. await self.store.update_trace(child_id, uid="another-user")
  400. rejected = await review_task_result(
  401. child_trace_id=child_id,
  402. decision="ASCEND",
  403. reason="Owner mismatch must be rejected",
  404. context=self.context(),
  405. )
  406. self.assertEqual("failed", rejected["status"])
  407. self.assertIn("same owner", rejected["error"])
  408. async def test_report_identity_must_match_direct_child(self):
  409. self.runner = ForgedIdentityRunner(self.store)
  410. result = await agent(task_brief=BRIEF, context=self.context())
  411. child_id = result["sub_trace_id"]
  412. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  413. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  414. parent = await self.store.get_trace(self.root_id)
  415. pending_report = ensure_task_protocol(parent.context)["pending_reviews"][
  416. child_id
  417. ]["task_report"]
  418. pending_report["child_trace_id"] = "corrupted-after-recording"
  419. await self.store.update_trace(self.root_id, context=parent.context)
  420. rejected = await review_task_result(
  421. child_trace_id=child_id,
  422. decision="FAIL",
  423. reason="Corrupted identity cannot be reviewed",
  424. context=self.context(),
  425. )
  426. self.assertEqual("failed", rejected["status"])
  427. self.assertIn("does not match", rejected["error"])
  428. async def test_remote_agent_keeps_legacy_task_interface(self):
  429. remote_result = {
  430. "status": "completed",
  431. "summary": "remote result",
  432. }
  433. with patch(
  434. "cyber_agent.tools.builtin.subagent._run_remote_agent",
  435. new=AsyncMock(return_value=remote_result),
  436. ) as remote_call:
  437. result = await agent(
  438. task="remote free text",
  439. agent_type="remote_librarian",
  440. )
  441. self.assertEqual(remote_result, result)
  442. self.assertEqual("remote free text", remote_call.await_args.kwargs["task"])
  443. async def test_revise_child_reuses_trace_and_preserves_report_history(self):
  444. first = await agent(task_brief=BRIEF, context=self.context())
  445. child_id = first["sub_trace_id"]
  446. reviewed = await review_task_result(
  447. child_trace_id=child_id,
  448. decision="REVISE_CHILD",
  449. reason="Please strengthen the evidence",
  450. context=self.context(),
  451. )
  452. self.assertEqual("completed", reviewed["status"])
  453. wrong = await agent(
  454. task_brief={
  455. "objective": "silently replace the reviewed task",
  456. "completion_criteria": ["this must be rejected"],
  457. },
  458. continue_from=child_id,
  459. context=self.context(),
  460. )
  461. self.assertEqual("failed", wrong["status"])
  462. self.assertIn("approved revision", wrong["error"])
  463. revised = await agent(
  464. task_brief=BRIEF,
  465. continue_from=child_id,
  466. context=self.context(),
  467. )
  468. self.assertEqual(child_id, revised["sub_trace_id"])
  469. self.assertTrue(revised["continue_from"])
  470. self.assertEqual(1, len(await self.children()))
  471. child = await self.store.get_trace(child_id)
  472. self.assertEqual(1, len(ensure_task_protocol(child.context)["report_history"]))
  473. self.assertEqual(
  474. TaskBrief.model_validate(BRIEF).model_dump(),
  475. ensure_task_protocol(child.context)["task_brief"],
  476. )
  477. async def test_review_cannot_reserve_more_children_than_quota(self):
  478. briefs = [
  479. {
  480. "objective": f"batch child {index}",
  481. "completion_criteria": [f"finish child {index}"],
  482. }
  483. for index in range(5)
  484. ]
  485. result = await agent(task_brief=briefs, context=self.context())
  486. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  487. next_brief = {
  488. "objective": "one allowed reserved child",
  489. "completion_criteria": ["use the final available slot"],
  490. }
  491. first = await review_task_result(
  492. child_trace_id=child_ids[0],
  493. decision="ACCEPT_REFINE",
  494. reason="Reserve the sixth and final child slot",
  495. approved_next_task=next_brief,
  496. context=self.context(),
  497. )
  498. self.assertEqual("completed", first["status"])
  499. overflow = await review_task_result(
  500. child_trace_id=child_ids[1],
  501. decision="DESCEND_AGAIN",
  502. reason="A seventh child must not be approved",
  503. approved_next_task={
  504. "objective": "overflow child",
  505. "completion_criteria": ["must never be scheduled"],
  506. },
  507. context=self.context(),
  508. )
  509. self.assertEqual("failed", overflow["status"])
  510. self.assertIn("reserved=1", overflow["error"])
  511. parent = await self.store.get_trace(self.root_id)
  512. state = ensure_task_protocol(parent.context)
  513. self.assertEqual(1, len(state["next_actions"]))
  514. self.assertIn(child_ids[1], state["pending_reviews"])
  515. async def test_recursive_goal_completion_does_not_cascade_or_select_sibling(self):
  516. tree = GoalTree(mission="recursive")
  517. parent = tree.add_goals(["parent"])[0]
  518. first, second = tree.add_goals(["first", "second"], parent_id=parent.id)
  519. tree.focus(parent.id)
  520. tree.focus(first.id)
  521. tree.complete(
  522. first.id,
  523. "done",
  524. auto_advance=False,
  525. cascade_parent=False,
  526. )
  527. self.assertEqual(parent.id, tree.current_id)
  528. self.assertEqual("pending", second.status)
  529. self.assertNotEqual("completed", parent.status)
  530. async def test_rewind_before_review_restores_report_and_review_gate(self):
  531. delegated = await agent(task_brief=BRIEF, context=self.context())
  532. child_id = delegated["sub_trace_id"]
  533. reviewed = await review_task_result(
  534. child_trace_id=child_id,
  535. decision="ASCEND",
  536. reason="Initially accepted before rewind",
  537. context={**self.context(), "sequence": 20},
  538. )
  539. self.assertEqual("completed", reviewed["status"])
  540. for sequence in (1, 15):
  541. await self.store.add_message(Message.create(
  542. trace_id=self.root_id,
  543. role="user",
  544. sequence=sequence,
  545. content=f"message-{sequence}",
  546. ))
  547. tree = await self.store.get_goal_tree(self.root_id)
  548. runner = AgentRunner(
  549. trace_store=self.store,
  550. tool_registry=get_tool_registry(),
  551. )
  552. await runner._rewind(self.root_id, 15, tree)
  553. parent = await self.store.get_trace(self.root_id)
  554. state = ensure_task_protocol(parent.context)
  555. self.assertIn(child_id, state["pending_reviews"])
  556. self.assertEqual([], state["reviews"])
  557. restored_tree = await self.store.get_goal_tree(self.root_id)
  558. self.assertEqual("pending_review", restored_tree.find("1").status)
  559. schemas = runner._get_runtime_tool_schemas(RunConfig(), parent)
  560. self.assertEqual(
  561. {"review_task_result"},
  562. {schema["function"]["name"] for schema in schemas},
  563. )
  564. async def test_grandchild_report_is_reviewed_only_by_direct_child(self):
  565. child_id = "manual-child"
  566. child_context = {
  567. "agent_mode": "recursive",
  568. "agent_mode_revision": 2,
  569. "agent_depth": 1,
  570. "root_trace_id": self.root_id,
  571. "created_by_tool": "agent",
  572. "task_protocol": new_task_protocol(BRIEF),
  573. }
  574. await self.store.create_trace(Trace(
  575. trace_id=child_id,
  576. mode="agent",
  577. task="child",
  578. parent_trace_id=self.root_id,
  579. uid="user-1",
  580. model="fake-model",
  581. context=child_context,
  582. ))
  583. child_tree = GoalTree(
  584. mission="child",
  585. goals=[Goal(id="1", description="delegate grandchild", status="in_progress")],
  586. current_id="1",
  587. )
  588. await self.store.update_goal_tree(child_id, child_tree)
  589. grandchild = await agent(
  590. task_brief=BRIEF,
  591. context=self.context(trace_id=child_id, tree=child_tree),
  592. )
  593. child = await self.store.get_trace(child_id)
  594. child_state = ensure_task_protocol(child.context)
  595. self.assertIn(grandchild["sub_trace_id"], child_state["pending_reviews"])
  596. root = await self.store.get_trace(self.root_id)
  597. self.assertEqual({}, ensure_task_protocol(root.context)["pending_reviews"])
  598. blocked = await agent(
  599. task_brief=BRIEF,
  600. context=self.context(trace_id=child_id, tree=child_tree),
  601. )
  602. self.assertEqual("failed", blocked["status"])
  603. reviewed = await review_task_result(
  604. child_trace_id=grandchild["sub_trace_id"],
  605. decision="ASCEND",
  606. reason="Grandchild satisfied the child criteria",
  607. context=self.context(trace_id=child_id, tree=child_tree),
  608. )
  609. self.assertEqual("completed", reviewed["status"])
  610. submitted = await submit_task_report(
  611. task_report=json.dumps(report_payload()),
  612. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  613. )
  614. self.assertEqual("completed", submitted["status"])
  615. async def test_child_must_execute_approved_action_before_own_report(self):
  616. child_id = "child-with-approved-action"
  617. child_context = {
  618. "agent_mode": "recursive",
  619. "agent_mode_revision": 2,
  620. "agent_depth": 1,
  621. "root_trace_id": self.root_id,
  622. "created_by_tool": "agent",
  623. "task_protocol": new_task_protocol(BRIEF),
  624. }
  625. await self.store.create_trace(Trace(
  626. trace_id=child_id,
  627. mode="agent",
  628. task="child",
  629. parent_trace_id=self.root_id,
  630. uid="user-1",
  631. model="fake-model",
  632. context=child_context,
  633. ))
  634. child_tree = GoalTree(
  635. mission="child",
  636. goals=[Goal(id="1", description="delegate", status="in_progress")],
  637. current_id="1",
  638. )
  639. await self.store.update_goal_tree(child_id, child_tree)
  640. child_tool_context = self.context(trace_id=child_id, tree=child_tree)
  641. first = await agent(task_brief=BRIEF, context=child_tool_context)
  642. next_brief = {
  643. "objective": "check a second grandchild task",
  644. "completion_criteria": ["provide a second checked conclusion"],
  645. }
  646. reviewed = await review_task_result(
  647. child_trace_id=first["sub_trace_id"],
  648. decision="ACCEPT_REFINE",
  649. reason="A separate gap remains",
  650. approved_next_task=next_brief,
  651. context=child_tool_context,
  652. )
  653. self.assertEqual("completed", reviewed["status"])
  654. blocked_report = await submit_task_report(
  655. task_report=report_payload(),
  656. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  657. )
  658. self.assertEqual("failed", blocked_report["status"])
  659. self.assertIn("approved next action", blocked_report["error"])
  660. second = await agent(task_brief=next_brief, context=child_tool_context)
  661. self.assertEqual("completed", second["status"])
  662. child = await self.store.get_trace(child_id)
  663. state = ensure_task_protocol(child.context)
  664. self.assertEqual([], state["next_actions"])
  665. self.assertIn(second["sub_trace_id"], state["pending_reviews"])
  666. async def test_goal_tool_cannot_bypass_queued_next_action(self):
  667. root = await self.store.get_trace(self.root_id)
  668. state = ensure_task_protocol(root.context)
  669. state["next_actions"] = [{
  670. "decision": "ACCEPT_REFINE",
  671. "child_trace_id": "reviewed-child",
  672. "goal_id": "1",
  673. "task_brief": BRIEF,
  674. "created_at_sequence": 5,
  675. }]
  676. await self.store.update_trace(self.root_id, context=root.context)
  677. result = await goal_tool(
  678. tree=self.tree,
  679. store=self.store,
  680. trace_id=self.root_id,
  681. done="attempt to bypass the approved action",
  682. )
  683. self.assertIn("已批准的下一步待执行", result)
  684. persisted_tree = await self.store.get_goal_tree(self.root_id)
  685. self.assertEqual("in_progress", persisted_tree.find("1").status)
  686. async def test_failed_report_allows_only_revise_or_fail(self):
  687. child_id = "failed-child"
  688. failed_report = TaskReport(
  689. child_trace_id=child_id,
  690. **report_payload(outcome="failed"),
  691. )
  692. child_context = {
  693. "agent_mode": "recursive",
  694. "agent_mode_revision": 2,
  695. "agent_depth": 1,
  696. "root_trace_id": self.root_id,
  697. "created_by_tool": "agent",
  698. "task_protocol": new_task_protocol(BRIEF),
  699. }
  700. child_context["task_protocol"]["task_report"] = failed_report.model_dump()
  701. await self.store.create_trace(Trace(
  702. trace_id=child_id,
  703. mode="agent",
  704. task="failed",
  705. parent_trace_id=self.root_id,
  706. uid="user-1",
  707. context=child_context,
  708. ))
  709. root = await self.store.get_trace(self.root_id)
  710. state = ensure_task_protocol(root.context)
  711. state["pending_reviews"][child_id] = {
  712. "goal_id": "1",
  713. "task_report": failed_report.model_dump(),
  714. "received_at_sequence": 2,
  715. }
  716. await self.store.update_trace(self.root_id, context=root.context)
  717. rejected = await review_task_result(
  718. child_trace_id=child_id,
  719. decision="ASCEND",
  720. reason="incorrect",
  721. context=self.context(),
  722. )
  723. self.assertEqual("failed", rejected["status"])
  724. self.assertIn("only allow", rejected["error"])
  725. class RunnerCompletionGateTest(unittest.IsolatedAsyncioTestCase):
  726. async def asyncSetUp(self):
  727. self.temp_dir = tempfile.TemporaryDirectory()
  728. self.store = FileSystemTraceStore(self.temp_dir.name)
  729. self.trace_id = "gate-child"
  730. await self.store.create_trace(Trace(
  731. trace_id=self.trace_id,
  732. mode="agent",
  733. task="must report",
  734. parent_trace_id="root",
  735. uid="user-1",
  736. context={
  737. "agent_mode": "recursive",
  738. "agent_mode_revision": 2,
  739. "agent_depth": 1,
  740. "root_trace_id": "root",
  741. "created_by_tool": "agent",
  742. "task_protocol": new_task_protocol(BRIEF),
  743. },
  744. ))
  745. await self.store.update_goal_tree(self.trace_id, GoalTree(mission="must report"))
  746. async def asyncTearDown(self):
  747. self.temp_dir.cleanup()
  748. async def test_missing_report_becomes_protocol_error_after_two_corrections(self):
  749. calls = 0
  750. async def llm_call(**kwargs):
  751. nonlocal calls
  752. calls += 1
  753. return {
  754. "content": "I am done",
  755. "tool_calls": [],
  756. "finish_reason": "stop",
  757. }
  758. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  759. result = await runner.run_result(
  760. messages=[],
  761. config=RunConfig(
  762. trace_id=self.trace_id,
  763. max_iterations=8,
  764. knowledge=KnowledgeConfig(
  765. enable_extraction=False,
  766. enable_completion_extraction=False,
  767. enable_injection=False,
  768. ),
  769. ),
  770. )
  771. self.assertEqual("failed", result["status"])
  772. trace = await self.store.get_trace(self.trace_id)
  773. report = ensure_task_protocol(trace.context)["task_report"]
  774. self.assertEqual("protocol_error", report["outcome"])
  775. self.assertGreaterEqual(calls, 3)
  776. async def test_valid_report_allows_child_to_finish(self):
  777. calls = 0
  778. async def llm_call(**kwargs):
  779. nonlocal calls
  780. calls += 1
  781. if calls == 1:
  782. return {
  783. "content": "",
  784. "tool_calls": [{
  785. "id": "submit-1",
  786. "type": "function",
  787. "function": {
  788. "name": "submit_task_report",
  789. "arguments": json.dumps({
  790. "task_report": report_payload(),
  791. }),
  792. },
  793. }],
  794. "finish_reason": "tool_calls",
  795. }
  796. return {
  797. "content": "Structured report submitted.",
  798. "tool_calls": [],
  799. "finish_reason": "stop",
  800. }
  801. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  802. result = await runner.run_result(
  803. messages=[],
  804. config=RunConfig(
  805. trace_id=self.trace_id,
  806. max_iterations=4,
  807. knowledge=KnowledgeConfig(
  808. enable_extraction=False,
  809. enable_completion_extraction=False,
  810. enable_injection=False,
  811. ),
  812. ),
  813. )
  814. self.assertEqual("completed", result["status"])
  815. trace = await self.store.get_trace(self.trace_id)
  816. self.assertEqual(
  817. "satisfied",
  818. ensure_task_protocol(trace.context)["task_report"]["outcome"],
  819. )
  820. async def test_lifecycle_tool_batch_is_rejected_before_execution(self):
  821. root_id = "batch-gate-root"
  822. await self.store.create_trace(Trace(
  823. trace_id=root_id,
  824. mode="agent",
  825. task="root",
  826. uid="user-1",
  827. context={
  828. "agent_mode": "recursive",
  829. "agent_mode_revision": 2,
  830. "agent_depth": 0,
  831. "root_trace_id": root_id,
  832. "task_protocol": new_task_protocol(),
  833. },
  834. ))
  835. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  836. calls = 0
  837. async def llm_call(**kwargs):
  838. nonlocal calls
  839. calls += 1
  840. if calls == 1:
  841. return {
  842. "content": "",
  843. "tool_calls": [
  844. {
  845. "id": "agent-1",
  846. "type": "function",
  847. "function": {
  848. "name": "agent",
  849. "arguments": json.dumps({
  850. "task_brief": BRIEF,
  851. }),
  852. },
  853. },
  854. {
  855. "id": "context-1",
  856. "type": "function",
  857. "function": {
  858. "name": "get_current_context",
  859. "arguments": "{}",
  860. },
  861. },
  862. ],
  863. "finish_reason": "tool_calls",
  864. }
  865. return {
  866. "content": "Corrected the invalid batch.",
  867. "tool_calls": [],
  868. "finish_reason": "stop",
  869. }
  870. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  871. result = await runner.run_result(
  872. messages=[],
  873. config=RunConfig(
  874. trace_id=root_id,
  875. max_iterations=3,
  876. knowledge=KnowledgeConfig(
  877. enable_extraction=False,
  878. enable_completion_extraction=False,
  879. enable_injection=False,
  880. ),
  881. ),
  882. )
  883. self.assertEqual("completed", result["status"])
  884. children = await self.store.list_traces(parent_trace_id=root_id, limit=10)
  885. self.assertEqual([], children)
  886. tool_messages = [
  887. message
  888. for message in await self.store.get_trace_messages(root_id)
  889. if message.role == "tool"
  890. ]
  891. self.assertEqual(2, len(tool_messages))
  892. self.assertTrue(all(
  893. "only tool call" in str(message.content)
  894. for message in tool_messages
  895. ))
  896. async def test_rewind_clears_protocol_state_after_cutoff(self):
  897. trace = await self.store.get_trace(self.trace_id)
  898. state = ensure_task_protocol(trace.context)
  899. report = TaskReport(
  900. child_trace_id=self.trace_id,
  901. **report_payload(),
  902. ).model_dump()
  903. state["task_report"] = report
  904. state["task_report_submitted_at_sequence"] = 5
  905. state["pending_reviews"] = {
  906. "grandchild": {
  907. "goal_id": "1",
  908. "task_report": report,
  909. "received_at_sequence": 6,
  910. }
  911. }
  912. state["reviews"] = [{"reviewed_at_sequence": 7}]
  913. state["next_actions"] = [{"created_at_sequence": 8}]
  914. await self.store.update_trace(self.trace_id, context=trace.context)
  915. tree = GoalTree(
  916. mission="rewind",
  917. goals=[Goal(id="1", description="active", status="pending_review")],
  918. current_id="1",
  919. )
  920. await self.store.update_goal_tree(self.trace_id, tree)
  921. for sequence in (1, 2):
  922. await self.store.add_message(Message.create(
  923. trace_id=self.trace_id,
  924. role="user",
  925. sequence=sequence,
  926. content=f"message-{sequence}",
  927. ))
  928. runner = AgentRunner(trace_store=self.store)
  929. cutoff = await runner._rewind(self.trace_id, 2, tree)
  930. self.assertEqual(3, cutoff)
  931. rewound = await self.store.get_trace(self.trace_id)
  932. state = ensure_task_protocol(rewound.context)
  933. self.assertIsNone(state["task_report"])
  934. self.assertEqual(1, len(state["report_history"]))
  935. self.assertEqual({}, state["pending_reviews"])
  936. self.assertEqual([], state["reviews"])
  937. self.assertEqual([], state["next_actions"])
  938. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  939. self.assertEqual("pending", rewound_tree.find("1").status)
  940. async def test_rewind_restores_preserved_pending_review_projection(self):
  941. trace = await self.store.get_trace(self.trace_id)
  942. state = ensure_task_protocol(trace.context)
  943. report = TaskReport(
  944. child_trace_id="preserved-grandchild",
  945. **report_payload(),
  946. ).model_dump()
  947. state["pending_reviews"] = {
  948. "preserved-grandchild": {
  949. "goal_id": "1",
  950. "task_report": report,
  951. "received_at_sequence": 2,
  952. }
  953. }
  954. await self.store.update_trace(self.trace_id, context=trace.context)
  955. tree = GoalTree(
  956. mission="rewind",
  957. goals=[Goal(id="1", description="review", status="pending_review")],
  958. current_id="1",
  959. )
  960. await self.store.update_goal_tree(self.trace_id, tree)
  961. for sequence in (1, 2):
  962. await self.store.add_message(Message.create(
  963. trace_id=self.trace_id,
  964. role="user",
  965. sequence=sequence,
  966. content=f"message-{sequence}",
  967. ))
  968. runner = AgentRunner(trace_store=self.store)
  969. await runner._rewind(self.trace_id, 2, tree)
  970. rewound = await self.store.get_trace(self.trace_id)
  971. self.assertIn(
  972. "preserved-grandchild",
  973. ensure_task_protocol(rewound.context)["pending_reviews"],
  974. )
  975. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  976. self.assertEqual("pending_review", rewound_tree.find("1").status)
  977. async def test_root_cannot_finish_with_unconsumed_next_action(self):
  978. root_id = "next-action-gate-root"
  979. state = new_task_protocol()
  980. state["next_actions"] = [{
  981. "decision": "ACCEPT_REFINE",
  982. "child_trace_id": "reviewed-child",
  983. "goal_id": None,
  984. "task_brief": BRIEF,
  985. "created_at_sequence": 1,
  986. }]
  987. await self.store.create_trace(Trace(
  988. trace_id=root_id,
  989. mode="agent",
  990. task="root",
  991. uid="user-1",
  992. context={
  993. "agent_mode": "recursive",
  994. "agent_mode_revision": 2,
  995. "agent_depth": 0,
  996. "root_trace_id": root_id,
  997. "task_protocol": state,
  998. },
  999. ))
  1000. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  1001. calls = 0
  1002. async def llm_call(**kwargs):
  1003. nonlocal calls
  1004. calls += 1
  1005. return {
  1006. "content": "Trying to finish without the approved action.",
  1007. "tool_calls": [],
  1008. "finish_reason": "stop",
  1009. }
  1010. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1011. result = await runner.run_result(
  1012. messages=[],
  1013. config=RunConfig(
  1014. trace_id=root_id,
  1015. max_iterations=5,
  1016. knowledge=KnowledgeConfig(
  1017. enable_extraction=False,
  1018. enable_completion_extraction=False,
  1019. enable_injection=False,
  1020. ),
  1021. ),
  1022. )
  1023. self.assertEqual("failed", result["status"])
  1024. self.assertEqual(3, calls)
  1025. root = await self.store.get_trace(root_id)
  1026. self.assertEqual(1, len(ensure_task_protocol(root.context)["next_actions"]))
  1027. class RuntimeProtocolSchemaTest(unittest.TestCase):
  1028. def setUp(self):
  1029. self.runner = AgentRunner(tool_registry=get_tool_registry())
  1030. def names(self, schemas):
  1031. return {schema["function"]["name"] for schema in schemas}
  1032. def test_legacy_and_recursive_runtime_schemas_are_isolated(self):
  1033. legacy = Trace(
  1034. trace_id="legacy",
  1035. mode="agent",
  1036. context={"agent_mode": "legacy", "agent_mode_revision": 1},
  1037. )
  1038. legacy_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), legacy)
  1039. self.assertNotIn("submit_task_report", self.names(legacy_schemas))
  1040. self.assertNotIn("review_task_result", self.names(legacy_schemas))
  1041. legacy_agent = next(
  1042. item for item in legacy_schemas if item["function"]["name"] == "agent"
  1043. )
  1044. properties = legacy_agent["function"]["parameters"]["properties"]
  1045. self.assertIn("task", properties)
  1046. self.assertNotIn("task_brief", properties)
  1047. child = Trace(
  1048. trace_id="child",
  1049. mode="agent",
  1050. parent_trace_id="root",
  1051. context={
  1052. "agent_mode": "recursive",
  1053. "agent_mode_revision": 2,
  1054. "task_protocol": new_task_protocol(BRIEF),
  1055. },
  1056. )
  1057. child_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1058. self.assertIn("submit_task_report", self.names(child_schemas))
  1059. self.assertNotIn("review_task_result", self.names(child_schemas))
  1060. state = ensure_task_protocol(child.context)
  1061. state["pending_reviews"]["grandchild"] = {"task_report": {}}
  1062. review_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1063. self.assertEqual({"review_task_result"}, self.names(review_schemas))
  1064. def test_protocol_tools_publish_nested_object_schemas(self):
  1065. schemas = {
  1066. schema["function"]["name"]: schema
  1067. for schema in get_tool_registry().get_schemas([
  1068. "submit_task_report",
  1069. "review_task_result",
  1070. ])
  1071. }
  1072. report_schema = schemas["submit_task_report"]["function"]["parameters"]
  1073. task_report = report_schema["properties"]["task_report"]
  1074. self.assertEqual("object", task_report["type"])
  1075. self.assertIn("validation", task_report["properties"])
  1076. self.assertIn("next_step_suggestion", task_report["properties"])
  1077. review_schema = schemas["review_task_result"]["function"]["parameters"]
  1078. self.assertEqual(
  1079. [
  1080. "ACCEPT_REFINE",
  1081. "REJECT_REFINE",
  1082. "REVISE_CHILD",
  1083. "DESCEND_AGAIN",
  1084. "ASCEND",
  1085. "FAIL",
  1086. ],
  1087. review_schema["properties"]["decision"]["enum"],
  1088. )
  1089. self.assertEqual(
  1090. "object",
  1091. review_schema["properties"]["approved_next_task"]["type"],
  1092. )
  1093. if __name__ == "__main__":
  1094. unittest.main()