test_task_protocol.py 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197
  1. import json
  2. import tempfile
  3. import unittest
  4. from unittest.mock import AsyncMock, patch
  5. from pydantic import ValidationError
  6. from cyber_agent.core.agent_mode import RECURSIVE_CAPABILITY_TOOLS_CONTEXT_KEY
  7. from cyber_agent.core.runner import AgentRunner, RunConfig
  8. from cyber_agent.core.task_protocol import (
  9. TaskBrief,
  10. TaskReport,
  11. TaskReview,
  12. ensure_task_protocol,
  13. new_task_protocol,
  14. )
  15. from cyber_agent.tools import get_tool_registry
  16. from cyber_agent.tools.builtin.knowledge import KnowledgeConfig
  17. from cyber_agent.tools.builtin.subagent import agent
  18. from cyber_agent.tools.builtin.task_protocol import (
  19. review_task_result,
  20. submit_task_report,
  21. )
  22. from cyber_agent.trace.goal_models import Goal, GoalTree
  23. from cyber_agent.trace.goal_tool import goal_tool
  24. from cyber_agent.trace.models import Message, Trace
  25. from cyber_agent.trace.store import FileSystemTraceStore
  26. BRIEF = {
  27. "objective": "research one concrete question",
  28. "completion_criteria": ["provide one checked conclusion"],
  29. "constraints": ["do not guess"],
  30. }
  31. def report_payload(*, outcome="satisfied"):
  32. return {
  33. "summary": "checked result",
  34. "outcome": outcome,
  35. "validation": {
  36. "hard_passed": outcome == "satisfied",
  37. "open_issues": [] if outcome == "satisfied" else ["still failing"],
  38. },
  39. "next_step_suggestion": {
  40. "direction": "NONE",
  41. "reason": "No additional task is needed",
  42. },
  43. "outputs": [{"kind": "note", "value": "result"}],
  44. "evidence": [{"kind": "test", "value": "passed"}],
  45. "remaining_issues": [] if outcome == "satisfied" else ["still failing"],
  46. }
  47. class FakeToolRegistry:
  48. def get_tool_names(self):
  49. return [
  50. "agent",
  51. "evaluate",
  52. "bash_command",
  53. "goal",
  54. "read_file",
  55. "submit_task_report",
  56. "review_task_result",
  57. ]
  58. class ReportingRunner:
  59. def __init__(self, store):
  60. self.store = store
  61. self.tools = FakeToolRegistry()
  62. self.debug = False
  63. self.stdin_check = None
  64. self._cancel_events = {}
  65. self.configs = []
  66. async def run_result(self, messages, config, on_event=None):
  67. self.configs.append(config)
  68. submitted = await submit_task_report(
  69. task_report=report_payload(),
  70. context={
  71. "store": self.store,
  72. "trace_id": config.trace_id,
  73. "sequence": 3,
  74. },
  75. )
  76. if submitted["status"] != "completed":
  77. raise AssertionError(submitted)
  78. await self.store.update_trace(config.trace_id, status="completed")
  79. return {
  80. "status": "completed",
  81. "summary": "child submitted TaskReport",
  82. "saved_knowledge_ids": [],
  83. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  84. }
  85. class FailedAfterReportingRunner(ReportingRunner):
  86. async def run_result(self, messages, config, on_event=None):
  87. result = await super().run_result(messages, config, on_event)
  88. await self.store.update_trace(
  89. config.trace_id,
  90. status="failed",
  91. error_message="runner failed after report submission",
  92. )
  93. return {
  94. **result,
  95. "status": "failed",
  96. "error": "runner failed after report submission",
  97. }
  98. class ForgedIdentityRunner(ReportingRunner):
  99. async def run_result(self, messages, config, on_event=None):
  100. trace = await self.store.get_trace(config.trace_id)
  101. state = ensure_task_protocol(trace.context)
  102. state["task_report"] = TaskReport(
  103. child_trace_id="someone-elses-trace",
  104. **report_payload(),
  105. ).model_dump()
  106. await self.store.update_trace(
  107. config.trace_id,
  108. context=trace.context,
  109. status="completed",
  110. )
  111. return {
  112. "status": "completed",
  113. "summary": "forged report identity",
  114. "saved_knowledge_ids": [],
  115. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  116. }
  117. class RecursiveTaskProtocolTest(unittest.IsolatedAsyncioTestCase):
  118. async def asyncSetUp(self):
  119. self.temp_dir = tempfile.TemporaryDirectory()
  120. self.store = FileSystemTraceStore(self.temp_dir.name)
  121. self.runner = ReportingRunner(self.store)
  122. self.root_id = "recursive-root"
  123. context = {
  124. "agent_mode": "recursive",
  125. "agent_mode_revision": 2,
  126. "agent_depth": 0,
  127. "root_trace_id": self.root_id,
  128. "task_protocol": new_task_protocol(),
  129. }
  130. await self.store.create_trace(Trace(
  131. trace_id=self.root_id,
  132. mode="agent",
  133. task="root",
  134. uid="user-1",
  135. model="fake-model",
  136. context=context,
  137. ))
  138. self.tree = GoalTree(
  139. mission="root",
  140. goals=[Goal(id="1", description="delegate", status="in_progress")],
  141. current_id="1",
  142. )
  143. await self.store.update_goal_tree(self.root_id, self.tree)
  144. async def asyncTearDown(self):
  145. self.temp_dir.cleanup()
  146. def context(self, trace_id=None, goal_id="1", tree=None):
  147. return {
  148. "store": self.store,
  149. "trace_id": trace_id or self.root_id,
  150. "goal_id": goal_id,
  151. "runner": self.runner,
  152. "goal_tree": tree or self.tree,
  153. "sequence": 10,
  154. RECURSIVE_CAPABILITY_TOOLS_CONTEXT_KEY: self.runner.tools.get_tool_names(),
  155. }
  156. async def children(self):
  157. return await self.store.list_traces(
  158. parent_trace_id=self.root_id,
  159. created_by_tool="agent",
  160. limit=20,
  161. )
  162. async def test_models_validate_and_round_trip(self):
  163. brief = TaskBrief.model_validate(BRIEF)
  164. self.assertEqual(BRIEF, brief.model_dump(exclude_none=True))
  165. with self.assertRaises(ValidationError):
  166. TaskBrief(objective="", completion_criteria=[])
  167. with self.assertRaises(ValidationError):
  168. TaskReview(
  169. parent_trace_id="p",
  170. child_trace_id="c",
  171. decision="ACCEPT_REFINE",
  172. reason="continue",
  173. )
  174. invalid_suggestion = report_payload()
  175. invalid_suggestion["next_step_suggestion"] = {
  176. "direction": "NONE",
  177. "reason": "No next task should be attached",
  178. "suggested_next_task": BRIEF,
  179. }
  180. with self.assertRaises(ValidationError):
  181. TaskReport(child_trace_id="child", **invalid_suggestion)
  182. root_submission = await submit_task_report(
  183. task_report=report_payload(),
  184. context={"store": self.store, "trace_id": self.root_id},
  185. )
  186. self.assertEqual("failed", root_submission["status"])
  187. self.assertIn("Root Agent", root_submission["error"])
  188. async def test_child_report_blocks_new_child_until_parent_review(self):
  189. result = await agent(task_brief=BRIEF, context=self.context())
  190. self.assertEqual("completed", result["status"])
  191. child_id = result["sub_trace_id"]
  192. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  193. parent = await self.store.get_trace(self.root_id)
  194. state = ensure_task_protocol(parent.context)
  195. self.assertEqual([child_id], list(state["pending_reviews"]))
  196. tree = await self.store.get_goal_tree(self.root_id)
  197. self.assertEqual("pending_review", tree.find("1").status)
  198. restarted_runner = AgentRunner(tool_registry=get_tool_registry())
  199. restored = await self.store.get_trace(self.root_id)
  200. restored_names = {
  201. schema["function"]["name"]
  202. for schema in restarted_runner._get_runtime_tool_schemas(
  203. RunConfig(), restored
  204. )
  205. }
  206. self.assertEqual({"review_task_result"}, restored_names)
  207. blocked = await agent(task_brief=BRIEF, context=self.context())
  208. self.assertEqual("failed", blocked["status"])
  209. self.assertIn("Review pending", blocked["error"])
  210. async def test_qwen_json_string_containers_are_normalized(self):
  211. result = await agent(
  212. task_brief=json.dumps(BRIEF),
  213. messages=json.dumps([{
  214. "role": "system",
  215. "content": "controlled child context",
  216. }]),
  217. context=self.context(),
  218. )
  219. self.assertEqual("completed", result["status"])
  220. child = await self.store.get_trace(result["sub_trace_id"])
  221. self.assertEqual(
  222. TaskBrief.model_validate(BRIEF).model_dump(),
  223. ensure_task_protocol(child.context)["task_brief"],
  224. )
  225. async def test_messages_reject_mixed_dimensions(self):
  226. mixed_messages = [
  227. {"role": "system", "content": "shared context"},
  228. [{"role": "user", "content": "per-agent context"}],
  229. ]
  230. for messages in (mixed_messages, json.dumps(mixed_messages)):
  231. with self.subTest(messages_type=type(messages).__name__):
  232. result = await agent(
  233. task_brief=BRIEF,
  234. messages=messages,
  235. context=self.context(),
  236. )
  237. self.assertEqual("failed", result["status"])
  238. self.assertIn("without mixed dimensions", result["error"])
  239. async def test_no_focused_goal_still_persists_and_reviews_report(self):
  240. result = await agent(
  241. task_brief=BRIEF,
  242. context=self.context(goal_id=None),
  243. )
  244. parent = await self.store.get_trace(self.root_id)
  245. entry = ensure_task_protocol(parent.context)["pending_reviews"][
  246. result["sub_trace_id"]
  247. ]
  248. self.assertIsNone(entry["goal_id"])
  249. reviewed = await review_task_result(
  250. child_trace_id=result["sub_trace_id"],
  251. decision="ASCEND",
  252. reason="Standalone delegation is complete",
  253. context=self.context(goal_id=None),
  254. )
  255. self.assertEqual("completed", reviewed["status"])
  256. async def test_accept_refine_approves_exactly_one_next_child(self):
  257. first = await agent(task_brief=BRIEF, context=self.context())
  258. next_brief = {
  259. "objective": "check the independent gap",
  260. "completion_criteria": ["close the gap"],
  261. }
  262. parent = await self.store.get_trace(self.root_id)
  263. ensure_task_protocol(parent.context)["protocol_correction_attempts"] = 2
  264. await self.store.update_trace(self.root_id, context=parent.context)
  265. reviewed = await review_task_result(
  266. child_trace_id=first["sub_trace_id"],
  267. decision="ACCEPT_REFINE",
  268. reason="The gap belongs to this parent",
  269. approved_next_task=json.dumps(next_brief),
  270. context=self.context(),
  271. )
  272. self.assertEqual("completed", reviewed["status"])
  273. parent = await self.store.get_trace(self.root_id)
  274. self.assertEqual(
  275. 0,
  276. ensure_task_protocol(parent.context)["protocol_correction_attempts"],
  277. )
  278. schemas = AgentRunner(
  279. tool_registry=get_tool_registry()
  280. )._get_runtime_tool_schemas(RunConfig(), parent)
  281. self.assertEqual(
  282. {"agent"},
  283. {schema["function"]["name"] for schema in schemas},
  284. )
  285. wrong = await agent(task_brief=BRIEF, context=self.context())
  286. self.assertEqual("failed", wrong["status"])
  287. self.assertIn("approved next task", wrong["error"])
  288. second = await agent(task_brief=next_brief, context=self.context())
  289. self.assertEqual("completed", second["status"])
  290. self.assertEqual(2, len(await self.children()))
  291. parent = await self.store.get_trace(self.root_id)
  292. self.assertEqual([], ensure_task_protocol(parent.context)["next_actions"])
  293. async def test_batch_reports_are_reviewed_one_by_one(self):
  294. briefs = [
  295. BRIEF,
  296. {
  297. "objective": "research second question",
  298. "completion_criteria": ["provide second conclusion"],
  299. },
  300. ]
  301. result = await agent(task_brief=briefs, context=self.context())
  302. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  303. self.assertEqual(2, len(result["task_reports"]))
  304. premature = await review_task_result(
  305. child_trace_id=child_ids[0],
  306. decision="ASCEND",
  307. reason="Cannot finish while another report is pending",
  308. context=self.context(),
  309. )
  310. self.assertEqual("failed", premature["status"])
  311. self.assertIn("final pending report", premature["error"])
  312. first_review = await review_task_result(
  313. child_trace_id=child_ids[0],
  314. decision="REJECT_REFINE",
  315. reason="No extra task needed for this report",
  316. context=self.context(),
  317. )
  318. self.assertEqual(1, first_review["pending_review_count"])
  319. tree = await self.store.get_goal_tree(self.root_id)
  320. self.assertEqual("pending_review", tree.find("1").status)
  321. final_review = await review_task_result(
  322. child_trace_id=child_ids[1],
  323. decision="ASCEND",
  324. reason="All completion criteria are satisfied",
  325. context=self.context(),
  326. )
  327. self.assertEqual(0, final_review["pending_review_count"])
  328. tree = await self.store.get_goal_tree(self.root_id)
  329. self.assertEqual("completed", tree.find("1").status)
  330. duplicate = await review_task_result(
  331. child_trace_id=child_ids[1],
  332. decision="ASCEND",
  333. reason="Duplicate review must fail",
  334. context=self.context(),
  335. )
  336. self.assertEqual("failed", duplicate["status"])
  337. self.assertIn("no pending TaskReport", duplicate["error"])
  338. async def test_terminal_review_waits_for_queued_next_action(self):
  339. result = await agent(
  340. task_brief=[
  341. BRIEF,
  342. {
  343. "objective": "second batch child",
  344. "completion_criteria": ["finish the second child"],
  345. },
  346. ],
  347. context=self.context(),
  348. )
  349. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  350. next_brief = {
  351. "objective": "approved follow-up",
  352. "completion_criteria": ["finish the approved follow-up"],
  353. }
  354. accepted = await review_task_result(
  355. child_trace_id=child_ids[0],
  356. decision="ACCEPT_REFINE",
  357. reason="A separate follow-up is required",
  358. approved_next_task=next_brief,
  359. context=self.context(),
  360. )
  361. self.assertEqual("completed", accepted["status"])
  362. blocked = await review_task_result(
  363. child_trace_id=child_ids[1],
  364. decision="ASCEND",
  365. reason="Cannot ascend before the accepted follow-up",
  366. context=self.context(),
  367. )
  368. self.assertEqual("failed", blocked["status"])
  369. self.assertIn("approved next action", blocked["error"])
  370. tree = await self.store.get_goal_tree(self.root_id)
  371. self.assertEqual("pending_review", tree.find("1").status)
  372. final_non_terminal = await review_task_result(
  373. child_trace_id=child_ids[1],
  374. decision="REJECT_REFINE",
  375. reason="This report adds no other follow-up",
  376. context=self.context(),
  377. )
  378. self.assertEqual("completed", final_non_terminal["status"])
  379. tree = await self.store.get_goal_tree(self.root_id)
  380. self.assertEqual("in_progress", tree.find("1").status)
  381. async def test_runner_failure_overrides_stale_satisfied_report(self):
  382. self.runner = FailedAfterReportingRunner(self.store)
  383. result = await agent(task_brief=BRIEF, context=self.context())
  384. self.assertEqual("failed", result["status"])
  385. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  386. child = await self.store.get_trace(result["sub_trace_id"])
  387. child_state = ensure_task_protocol(child.context)
  388. self.assertEqual(1, len(child_state["report_history"]))
  389. self.assertEqual("satisfied", child_state["report_history"][0]["outcome"])
  390. invalid_ascend = await review_task_result(
  391. child_trace_id=result["sub_trace_id"],
  392. decision="ASCEND",
  393. reason="A failed execution cannot ascend",
  394. context=self.context(),
  395. )
  396. self.assertEqual("failed", invalid_ascend["status"])
  397. self.assertIn("only allow", invalid_ascend["error"])
  398. async def test_review_rejects_wrong_owner_even_if_report_is_injected(self):
  399. first = await agent(task_brief=BRIEF, context=self.context())
  400. child_id = first["sub_trace_id"]
  401. await self.store.update_trace(child_id, uid="another-user")
  402. rejected = await review_task_result(
  403. child_trace_id=child_id,
  404. decision="ASCEND",
  405. reason="Owner mismatch must be rejected",
  406. context=self.context(),
  407. )
  408. self.assertEqual("failed", rejected["status"])
  409. self.assertIn("same owner", rejected["error"])
  410. async def test_report_identity_must_match_direct_child(self):
  411. self.runner = ForgedIdentityRunner(self.store)
  412. result = await agent(task_brief=BRIEF, context=self.context())
  413. child_id = result["sub_trace_id"]
  414. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  415. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  416. parent = await self.store.get_trace(self.root_id)
  417. pending_report = ensure_task_protocol(parent.context)["pending_reviews"][
  418. child_id
  419. ]["task_report"]
  420. pending_report["child_trace_id"] = "corrupted-after-recording"
  421. await self.store.update_trace(self.root_id, context=parent.context)
  422. rejected = await review_task_result(
  423. child_trace_id=child_id,
  424. decision="FAIL",
  425. reason="Corrupted identity cannot be reviewed",
  426. context=self.context(),
  427. )
  428. self.assertEqual("failed", rejected["status"])
  429. self.assertIn("does not match", rejected["error"])
  430. async def test_remote_agent_keeps_legacy_task_interface(self):
  431. remote_result = {
  432. "status": "completed",
  433. "summary": "remote result",
  434. }
  435. with patch(
  436. "cyber_agent.tools.builtin.subagent._run_remote_agent",
  437. new=AsyncMock(return_value=remote_result),
  438. ) as remote_call:
  439. result = await agent(
  440. task="remote free text",
  441. agent_type="remote_librarian",
  442. )
  443. self.assertEqual(remote_result, result)
  444. self.assertEqual("remote free text", remote_call.await_args.kwargs["task"])
  445. async def test_revise_child_reuses_trace_and_preserves_report_history(self):
  446. first = await agent(task_brief=BRIEF, context=self.context())
  447. child_id = first["sub_trace_id"]
  448. reviewed = await review_task_result(
  449. child_trace_id=child_id,
  450. decision="REVISE_CHILD",
  451. reason="Please strengthen the evidence",
  452. context=self.context(),
  453. )
  454. self.assertEqual("completed", reviewed["status"])
  455. wrong = await agent(
  456. task_brief={
  457. "objective": "silently replace the reviewed task",
  458. "completion_criteria": ["this must be rejected"],
  459. },
  460. continue_from=child_id,
  461. context=self.context(),
  462. )
  463. self.assertEqual("failed", wrong["status"])
  464. self.assertIn("approved revision", wrong["error"])
  465. revised = await agent(
  466. task_brief=BRIEF,
  467. continue_from=child_id,
  468. context=self.context(),
  469. )
  470. self.assertEqual(child_id, revised["sub_trace_id"])
  471. self.assertTrue(revised["continue_from"])
  472. self.assertEqual(1, len(await self.children()))
  473. child = await self.store.get_trace(child_id)
  474. self.assertEqual(1, len(ensure_task_protocol(child.context)["report_history"]))
  475. self.assertEqual(
  476. TaskBrief.model_validate(BRIEF).model_dump(),
  477. ensure_task_protocol(child.context)["task_brief"],
  478. )
  479. async def test_review_cannot_reserve_more_children_than_quota(self):
  480. briefs = [
  481. {
  482. "objective": f"batch child {index}",
  483. "completion_criteria": [f"finish child {index}"],
  484. }
  485. for index in range(5)
  486. ]
  487. result = await agent(task_brief=briefs, context=self.context())
  488. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  489. next_brief = {
  490. "objective": "one allowed reserved child",
  491. "completion_criteria": ["use the final available slot"],
  492. }
  493. first = await review_task_result(
  494. child_trace_id=child_ids[0],
  495. decision="ACCEPT_REFINE",
  496. reason="Reserve the sixth and final child slot",
  497. approved_next_task=next_brief,
  498. context=self.context(),
  499. )
  500. self.assertEqual("completed", first["status"])
  501. overflow = await review_task_result(
  502. child_trace_id=child_ids[1],
  503. decision="DESCEND_AGAIN",
  504. reason="A seventh child must not be approved",
  505. approved_next_task={
  506. "objective": "overflow child",
  507. "completion_criteria": ["must never be scheduled"],
  508. },
  509. context=self.context(),
  510. )
  511. self.assertEqual("failed", overflow["status"])
  512. self.assertIn("reserved=1", overflow["error"])
  513. parent = await self.store.get_trace(self.root_id)
  514. state = ensure_task_protocol(parent.context)
  515. self.assertEqual(1, len(state["next_actions"]))
  516. self.assertIn(child_ids[1], state["pending_reviews"])
  517. async def test_recursive_goal_completion_does_not_cascade_or_select_sibling(self):
  518. tree = GoalTree(mission="recursive")
  519. parent = tree.add_goals(["parent"])[0]
  520. first, second = tree.add_goals(["first", "second"], parent_id=parent.id)
  521. tree.focus(parent.id)
  522. tree.focus(first.id)
  523. tree.complete(
  524. first.id,
  525. "done",
  526. auto_advance=False,
  527. cascade_parent=False,
  528. )
  529. self.assertEqual(parent.id, tree.current_id)
  530. self.assertEqual("pending", second.status)
  531. self.assertNotEqual("completed", parent.status)
  532. async def test_rewind_before_review_restores_report_and_review_gate(self):
  533. delegated = await agent(task_brief=BRIEF, context=self.context())
  534. child_id = delegated["sub_trace_id"]
  535. reviewed = await review_task_result(
  536. child_trace_id=child_id,
  537. decision="ASCEND",
  538. reason="Initially accepted before rewind",
  539. context={**self.context(), "sequence": 20},
  540. )
  541. self.assertEqual("completed", reviewed["status"])
  542. for sequence in (1, 15):
  543. await self.store.add_message(Message.create(
  544. trace_id=self.root_id,
  545. role="user",
  546. sequence=sequence,
  547. content=f"message-{sequence}",
  548. ))
  549. tree = await self.store.get_goal_tree(self.root_id)
  550. runner = AgentRunner(
  551. trace_store=self.store,
  552. tool_registry=get_tool_registry(),
  553. )
  554. await runner._rewind(self.root_id, 15, tree)
  555. parent = await self.store.get_trace(self.root_id)
  556. state = ensure_task_protocol(parent.context)
  557. self.assertIn(child_id, state["pending_reviews"])
  558. self.assertEqual([], state["reviews"])
  559. restored_tree = await self.store.get_goal_tree(self.root_id)
  560. self.assertEqual("pending_review", restored_tree.find("1").status)
  561. schemas = runner._get_runtime_tool_schemas(RunConfig(), parent)
  562. self.assertEqual(
  563. {"review_task_result"},
  564. {schema["function"]["name"] for schema in schemas},
  565. )
  566. async def test_grandchild_report_is_reviewed_only_by_direct_child(self):
  567. child_id = "manual-child"
  568. child_context = {
  569. "agent_mode": "recursive",
  570. "agent_mode_revision": 2,
  571. "agent_depth": 1,
  572. "root_trace_id": self.root_id,
  573. "created_by_tool": "agent",
  574. "task_protocol": new_task_protocol(BRIEF),
  575. }
  576. await self.store.create_trace(Trace(
  577. trace_id=child_id,
  578. mode="agent",
  579. task="child",
  580. parent_trace_id=self.root_id,
  581. uid="user-1",
  582. model="fake-model",
  583. context=child_context,
  584. ))
  585. child_tree = GoalTree(
  586. mission="child",
  587. goals=[Goal(id="1", description="delegate grandchild", status="in_progress")],
  588. current_id="1",
  589. )
  590. await self.store.update_goal_tree(child_id, child_tree)
  591. grandchild = await agent(
  592. task_brief=BRIEF,
  593. context=self.context(trace_id=child_id, tree=child_tree),
  594. )
  595. child = await self.store.get_trace(child_id)
  596. child_state = ensure_task_protocol(child.context)
  597. self.assertIn(grandchild["sub_trace_id"], child_state["pending_reviews"])
  598. root = await self.store.get_trace(self.root_id)
  599. self.assertEqual({}, ensure_task_protocol(root.context)["pending_reviews"])
  600. blocked = await agent(
  601. task_brief=BRIEF,
  602. context=self.context(trace_id=child_id, tree=child_tree),
  603. )
  604. self.assertEqual("failed", blocked["status"])
  605. reviewed = await review_task_result(
  606. child_trace_id=grandchild["sub_trace_id"],
  607. decision="ASCEND",
  608. reason="Grandchild satisfied the child criteria",
  609. context=self.context(trace_id=child_id, tree=child_tree),
  610. )
  611. self.assertEqual("completed", reviewed["status"])
  612. submitted = await submit_task_report(
  613. task_report=json.dumps(report_payload()),
  614. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  615. )
  616. self.assertEqual("completed", submitted["status"])
  617. async def test_child_must_execute_approved_action_before_own_report(self):
  618. child_id = "child-with-approved-action"
  619. child_context = {
  620. "agent_mode": "recursive",
  621. "agent_mode_revision": 2,
  622. "agent_depth": 1,
  623. "root_trace_id": self.root_id,
  624. "created_by_tool": "agent",
  625. "task_protocol": new_task_protocol(BRIEF),
  626. }
  627. await self.store.create_trace(Trace(
  628. trace_id=child_id,
  629. mode="agent",
  630. task="child",
  631. parent_trace_id=self.root_id,
  632. uid="user-1",
  633. model="fake-model",
  634. context=child_context,
  635. ))
  636. child_tree = GoalTree(
  637. mission="child",
  638. goals=[Goal(id="1", description="delegate", status="in_progress")],
  639. current_id="1",
  640. )
  641. await self.store.update_goal_tree(child_id, child_tree)
  642. child_tool_context = self.context(trace_id=child_id, tree=child_tree)
  643. first = await agent(task_brief=BRIEF, context=child_tool_context)
  644. next_brief = {
  645. "objective": "check a second grandchild task",
  646. "completion_criteria": ["provide a second checked conclusion"],
  647. }
  648. reviewed = await review_task_result(
  649. child_trace_id=first["sub_trace_id"],
  650. decision="ACCEPT_REFINE",
  651. reason="A separate gap remains",
  652. approved_next_task=next_brief,
  653. context=child_tool_context,
  654. )
  655. self.assertEqual("completed", reviewed["status"])
  656. blocked_report = await submit_task_report(
  657. task_report=report_payload(),
  658. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  659. )
  660. self.assertEqual("failed", blocked_report["status"])
  661. self.assertIn("approved next action", blocked_report["error"])
  662. second = await agent(task_brief=next_brief, context=child_tool_context)
  663. self.assertEqual("completed", second["status"])
  664. child = await self.store.get_trace(child_id)
  665. state = ensure_task_protocol(child.context)
  666. self.assertEqual([], state["next_actions"])
  667. self.assertIn(second["sub_trace_id"], state["pending_reviews"])
  668. async def test_goal_tool_cannot_bypass_queued_next_action(self):
  669. root = await self.store.get_trace(self.root_id)
  670. state = ensure_task_protocol(root.context)
  671. state["next_actions"] = [{
  672. "decision": "ACCEPT_REFINE",
  673. "child_trace_id": "reviewed-child",
  674. "goal_id": "1",
  675. "task_brief": BRIEF,
  676. "created_at_sequence": 5,
  677. }]
  678. await self.store.update_trace(self.root_id, context=root.context)
  679. result = await goal_tool(
  680. tree=self.tree,
  681. store=self.store,
  682. trace_id=self.root_id,
  683. done="attempt to bypass the approved action",
  684. )
  685. self.assertIn("已批准的下一步待执行", result)
  686. persisted_tree = await self.store.get_goal_tree(self.root_id)
  687. self.assertEqual("in_progress", persisted_tree.find("1").status)
  688. async def test_failed_report_allows_only_revise_or_fail(self):
  689. child_id = "failed-child"
  690. failed_report = TaskReport(
  691. child_trace_id=child_id,
  692. **report_payload(outcome="failed"),
  693. )
  694. child_context = {
  695. "agent_mode": "recursive",
  696. "agent_mode_revision": 2,
  697. "agent_depth": 1,
  698. "root_trace_id": self.root_id,
  699. "created_by_tool": "agent",
  700. "task_protocol": new_task_protocol(BRIEF),
  701. }
  702. child_context["task_protocol"]["task_report"] = failed_report.model_dump()
  703. await self.store.create_trace(Trace(
  704. trace_id=child_id,
  705. mode="agent",
  706. task="failed",
  707. parent_trace_id=self.root_id,
  708. uid="user-1",
  709. context=child_context,
  710. ))
  711. root = await self.store.get_trace(self.root_id)
  712. state = ensure_task_protocol(root.context)
  713. state["pending_reviews"][child_id] = {
  714. "goal_id": "1",
  715. "task_report": failed_report.model_dump(),
  716. "received_at_sequence": 2,
  717. }
  718. await self.store.update_trace(self.root_id, context=root.context)
  719. rejected = await review_task_result(
  720. child_trace_id=child_id,
  721. decision="ASCEND",
  722. reason="incorrect",
  723. context=self.context(),
  724. )
  725. self.assertEqual("failed", rejected["status"])
  726. self.assertIn("only allow", rejected["error"])
  727. class RunnerCompletionGateTest(unittest.IsolatedAsyncioTestCase):
  728. async def asyncSetUp(self):
  729. self.temp_dir = tempfile.TemporaryDirectory()
  730. self.store = FileSystemTraceStore(self.temp_dir.name)
  731. self.trace_id = "gate-child"
  732. await self.store.create_trace(Trace(
  733. trace_id=self.trace_id,
  734. mode="agent",
  735. task="must report",
  736. parent_trace_id="root",
  737. uid="user-1",
  738. context={
  739. "agent_mode": "recursive",
  740. "agent_mode_revision": 2,
  741. "agent_depth": 1,
  742. "root_trace_id": "root",
  743. "created_by_tool": "agent",
  744. "task_protocol": new_task_protocol(BRIEF),
  745. },
  746. ))
  747. await self.store.update_goal_tree(self.trace_id, GoalTree(mission="must report"))
  748. async def asyncTearDown(self):
  749. self.temp_dir.cleanup()
  750. async def test_missing_report_becomes_protocol_error_after_two_corrections(self):
  751. calls = 0
  752. async def llm_call(**kwargs):
  753. nonlocal calls
  754. calls += 1
  755. return {
  756. "content": "I am done",
  757. "tool_calls": [],
  758. "finish_reason": "stop",
  759. }
  760. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  761. result = await runner.run_result(
  762. messages=[],
  763. config=RunConfig(
  764. trace_id=self.trace_id,
  765. max_iterations=8,
  766. knowledge=KnowledgeConfig(
  767. enable_extraction=False,
  768. enable_completion_extraction=False,
  769. enable_injection=False,
  770. ),
  771. ),
  772. )
  773. self.assertEqual("failed", result["status"])
  774. trace = await self.store.get_trace(self.trace_id)
  775. report = ensure_task_protocol(trace.context)["task_report"]
  776. self.assertEqual("protocol_error", report["outcome"])
  777. self.assertGreaterEqual(calls, 3)
  778. async def test_valid_report_allows_child_to_finish(self):
  779. calls = 0
  780. async def llm_call(**kwargs):
  781. nonlocal calls
  782. calls += 1
  783. if calls == 1:
  784. return {
  785. "content": "",
  786. "tool_calls": [{
  787. "id": "submit-1",
  788. "type": "function",
  789. "function": {
  790. "name": "submit_task_report",
  791. "arguments": json.dumps({
  792. "task_report": report_payload(),
  793. }),
  794. },
  795. }],
  796. "finish_reason": "tool_calls",
  797. }
  798. return {
  799. "content": "Structured report submitted.",
  800. "tool_calls": [],
  801. "finish_reason": "stop",
  802. }
  803. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  804. result = await runner.run_result(
  805. messages=[],
  806. config=RunConfig(
  807. trace_id=self.trace_id,
  808. max_iterations=4,
  809. knowledge=KnowledgeConfig(
  810. enable_extraction=False,
  811. enable_completion_extraction=False,
  812. enable_injection=False,
  813. ),
  814. ),
  815. )
  816. self.assertEqual("completed", result["status"])
  817. trace = await self.store.get_trace(self.trace_id)
  818. self.assertEqual(
  819. "satisfied",
  820. ensure_task_protocol(trace.context)["task_report"]["outcome"],
  821. )
  822. async def test_lifecycle_tool_batch_is_rejected_before_execution(self):
  823. root_id = "batch-gate-root"
  824. await self.store.create_trace(Trace(
  825. trace_id=root_id,
  826. mode="agent",
  827. task="root",
  828. uid="user-1",
  829. context={
  830. "agent_mode": "recursive",
  831. "agent_mode_revision": 2,
  832. "agent_depth": 0,
  833. "root_trace_id": root_id,
  834. "task_protocol": new_task_protocol(),
  835. },
  836. ))
  837. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  838. calls = 0
  839. async def llm_call(**kwargs):
  840. nonlocal calls
  841. calls += 1
  842. if calls == 1:
  843. return {
  844. "content": "",
  845. "tool_calls": [
  846. {
  847. "id": "agent-1",
  848. "type": "function",
  849. "function": {
  850. "name": "agent",
  851. "arguments": json.dumps({
  852. "task_brief": BRIEF,
  853. }),
  854. },
  855. },
  856. {
  857. "id": "context-1",
  858. "type": "function",
  859. "function": {
  860. "name": "get_current_context",
  861. "arguments": "{}",
  862. },
  863. },
  864. ],
  865. "finish_reason": "tool_calls",
  866. }
  867. return {
  868. "content": "Corrected the invalid batch.",
  869. "tool_calls": [],
  870. "finish_reason": "stop",
  871. }
  872. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  873. result = await runner.run_result(
  874. messages=[],
  875. config=RunConfig(
  876. trace_id=root_id,
  877. max_iterations=3,
  878. knowledge=KnowledgeConfig(
  879. enable_extraction=False,
  880. enable_completion_extraction=False,
  881. enable_injection=False,
  882. ),
  883. ),
  884. )
  885. self.assertEqual("completed", result["status"])
  886. children = await self.store.list_traces(parent_trace_id=root_id, limit=10)
  887. self.assertEqual([], children)
  888. tool_messages = [
  889. message
  890. for message in await self.store.get_trace_messages(root_id)
  891. if message.role == "tool"
  892. ]
  893. self.assertEqual(2, len(tool_messages))
  894. self.assertTrue(all(
  895. "only tool call" in str(message.content)
  896. for message in tool_messages
  897. ))
  898. async def test_rewind_clears_protocol_state_after_cutoff(self):
  899. trace = await self.store.get_trace(self.trace_id)
  900. state = ensure_task_protocol(trace.context)
  901. report = TaskReport(
  902. child_trace_id=self.trace_id,
  903. **report_payload(),
  904. ).model_dump()
  905. state["task_report"] = report
  906. state["task_report_submitted_at_sequence"] = 5
  907. state["pending_reviews"] = {
  908. "grandchild": {
  909. "goal_id": "1",
  910. "task_report": report,
  911. "received_at_sequence": 6,
  912. }
  913. }
  914. state["reviews"] = [{"reviewed_at_sequence": 7}]
  915. state["next_actions"] = [{"created_at_sequence": 8}]
  916. await self.store.update_trace(self.trace_id, context=trace.context)
  917. tree = GoalTree(
  918. mission="rewind",
  919. goals=[Goal(id="1", description="active", status="pending_review")],
  920. current_id="1",
  921. )
  922. await self.store.update_goal_tree(self.trace_id, tree)
  923. for sequence in (1, 2):
  924. await self.store.add_message(Message.create(
  925. trace_id=self.trace_id,
  926. role="user",
  927. sequence=sequence,
  928. content=f"message-{sequence}",
  929. ))
  930. runner = AgentRunner(trace_store=self.store)
  931. cutoff = await runner._rewind(self.trace_id, 2, tree)
  932. self.assertEqual(3, cutoff)
  933. rewound = await self.store.get_trace(self.trace_id)
  934. state = ensure_task_protocol(rewound.context)
  935. self.assertIsNone(state["task_report"])
  936. self.assertEqual(1, len(state["report_history"]))
  937. self.assertEqual({}, state["pending_reviews"])
  938. self.assertEqual([], state["reviews"])
  939. self.assertEqual([], state["next_actions"])
  940. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  941. self.assertEqual("pending", rewound_tree.find("1").status)
  942. async def test_rewind_restores_preserved_pending_review_projection(self):
  943. trace = await self.store.get_trace(self.trace_id)
  944. state = ensure_task_protocol(trace.context)
  945. report = TaskReport(
  946. child_trace_id="preserved-grandchild",
  947. **report_payload(),
  948. ).model_dump()
  949. state["pending_reviews"] = {
  950. "preserved-grandchild": {
  951. "goal_id": "1",
  952. "task_report": report,
  953. "received_at_sequence": 2,
  954. }
  955. }
  956. await self.store.update_trace(self.trace_id, context=trace.context)
  957. tree = GoalTree(
  958. mission="rewind",
  959. goals=[Goal(id="1", description="review", status="pending_review")],
  960. current_id="1",
  961. )
  962. await self.store.update_goal_tree(self.trace_id, tree)
  963. for sequence in (1, 2):
  964. await self.store.add_message(Message.create(
  965. trace_id=self.trace_id,
  966. role="user",
  967. sequence=sequence,
  968. content=f"message-{sequence}",
  969. ))
  970. runner = AgentRunner(trace_store=self.store)
  971. await runner._rewind(self.trace_id, 2, tree)
  972. rewound = await self.store.get_trace(self.trace_id)
  973. self.assertIn(
  974. "preserved-grandchild",
  975. ensure_task_protocol(rewound.context)["pending_reviews"],
  976. )
  977. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  978. self.assertEqual("pending_review", rewound_tree.find("1").status)
  979. async def test_root_cannot_finish_with_unconsumed_next_action(self):
  980. root_id = "next-action-gate-root"
  981. state = new_task_protocol()
  982. state["next_actions"] = [{
  983. "decision": "ACCEPT_REFINE",
  984. "child_trace_id": "reviewed-child",
  985. "goal_id": None,
  986. "task_brief": BRIEF,
  987. "created_at_sequence": 1,
  988. }]
  989. await self.store.create_trace(Trace(
  990. trace_id=root_id,
  991. mode="agent",
  992. task="root",
  993. uid="user-1",
  994. context={
  995. "agent_mode": "recursive",
  996. "agent_mode_revision": 2,
  997. "agent_depth": 0,
  998. "root_trace_id": root_id,
  999. "task_protocol": state,
  1000. },
  1001. ))
  1002. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  1003. calls = 0
  1004. async def llm_call(**kwargs):
  1005. nonlocal calls
  1006. calls += 1
  1007. return {
  1008. "content": "Trying to finish without the approved action.",
  1009. "tool_calls": [],
  1010. "finish_reason": "stop",
  1011. }
  1012. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1013. result = await runner.run_result(
  1014. messages=[],
  1015. config=RunConfig(
  1016. trace_id=root_id,
  1017. max_iterations=5,
  1018. knowledge=KnowledgeConfig(
  1019. enable_extraction=False,
  1020. enable_completion_extraction=False,
  1021. enable_injection=False,
  1022. ),
  1023. ),
  1024. )
  1025. self.assertEqual("failed", result["status"])
  1026. self.assertEqual(3, calls)
  1027. root = await self.store.get_trace(root_id)
  1028. self.assertEqual(1, len(ensure_task_protocol(root.context)["next_actions"]))
  1029. class RuntimeProtocolSchemaTest(unittest.TestCase):
  1030. def setUp(self):
  1031. self.runner = AgentRunner(tool_registry=get_tool_registry())
  1032. def names(self, schemas):
  1033. return {schema["function"]["name"] for schema in schemas}
  1034. def test_legacy_and_recursive_runtime_schemas_are_isolated(self):
  1035. legacy = Trace(
  1036. trace_id="legacy",
  1037. mode="agent",
  1038. context={"agent_mode": "legacy", "agent_mode_revision": 1},
  1039. )
  1040. legacy_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), legacy)
  1041. self.assertNotIn("submit_task_report", self.names(legacy_schemas))
  1042. self.assertNotIn("review_task_result", self.names(legacy_schemas))
  1043. legacy_agent = next(
  1044. item for item in legacy_schemas if item["function"]["name"] == "agent"
  1045. )
  1046. properties = legacy_agent["function"]["parameters"]["properties"]
  1047. self.assertIn("task", properties)
  1048. self.assertNotIn("task_brief", properties)
  1049. child = Trace(
  1050. trace_id="child",
  1051. mode="agent",
  1052. parent_trace_id="root",
  1053. context={
  1054. "agent_mode": "recursive",
  1055. "agent_mode_revision": 2,
  1056. "task_protocol": new_task_protocol(BRIEF),
  1057. },
  1058. )
  1059. child_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1060. self.assertIn("submit_task_report", self.names(child_schemas))
  1061. self.assertNotIn("review_task_result", self.names(child_schemas))
  1062. state = ensure_task_protocol(child.context)
  1063. state["pending_reviews"]["grandchild"] = {"task_report": {}}
  1064. review_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1065. self.assertEqual({"review_task_result"}, self.names(review_schemas))
  1066. def test_protocol_tools_publish_nested_object_schemas(self):
  1067. schemas = {
  1068. schema["function"]["name"]: schema
  1069. for schema in get_tool_registry().get_schemas([
  1070. "submit_task_report",
  1071. "review_task_result",
  1072. ])
  1073. }
  1074. report_schema = schemas["submit_task_report"]["function"]["parameters"]
  1075. task_report = report_schema["properties"]["task_report"]
  1076. self.assertEqual("object", task_report["type"])
  1077. self.assertIn("validation", task_report["properties"])
  1078. self.assertIn("next_step_suggestion", task_report["properties"])
  1079. review_schema = schemas["review_task_result"]["function"]["parameters"]
  1080. self.assertEqual(
  1081. [
  1082. "ACCEPT_REFINE",
  1083. "REJECT_REFINE",
  1084. "REVISE_CHILD",
  1085. "DESCEND_AGAIN",
  1086. "ASCEND",
  1087. "FAIL",
  1088. ],
  1089. review_schema["properties"]["decision"]["enum"],
  1090. )
  1091. self.assertEqual(
  1092. "object",
  1093. review_schema["properties"]["approved_next_task"]["type"],
  1094. )
  1095. if __name__ == "__main__":
  1096. unittest.main()