test_task_protocol.py 60 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582
  1. import json
  2. import tempfile
  3. import unittest
  4. from unittest.mock import AsyncMock, patch
  5. from pydantic import ValidationError
  6. from cyber_agent.core.agent_mode import RECURSIVE_CAPABILITY_TOOLS_CONTEXT_KEY
  7. from cyber_agent.core.resource_budget import (
  8. RESOURCE_BUDGET_CONTEXT_KEY,
  9. ResourceBudget,
  10. ResourceBudgetController,
  11. )
  12. from cyber_agent.core.runner import AgentRunner, RunConfig
  13. from cyber_agent.core.context_policy import (
  14. add_context_snapshot,
  15. create_context_snapshot,
  16. persist_root_task_anchor,
  17. )
  18. from cyber_agent.core.task_protocol import (
  19. TaskBrief,
  20. TaskReport,
  21. TaskReview,
  22. ensure_task_protocol,
  23. new_task_protocol,
  24. replace_task_brief,
  25. )
  26. from cyber_agent.tools import get_tool_registry
  27. from cyber_agent.tools.builtin.knowledge import KnowledgeConfig
  28. from cyber_agent.tools.builtin.subagent import _record_pending_task_reports, agent
  29. from cyber_agent.tools.builtin.task_protocol import (
  30. review_task_result,
  31. submit_task_report,
  32. )
  33. from cyber_agent.trace.goal_models import Goal, GoalTree
  34. from cyber_agent.trace.goal_tool import goal_tool
  35. from cyber_agent.trace.models import Message, Trace
  36. from cyber_agent.trace.store import FileSystemTraceStore
  37. from cyber_agent.core.validation import ValidationResult, ValidationRun
  38. BRIEF = {
  39. "objective": "research one concrete question",
  40. "reason": "the parent needs one independently checked result",
  41. "completion_criteria": ["provide one checked conclusion"],
  42. "expected_outputs": ["one checked conclusion"],
  43. "constraints": ["do not guess"],
  44. }
  45. ROOT_ANCHOR = {
  46. "objective": "finish the root task",
  47. "completion_criteria": ["finish the root task"],
  48. "constraints": ["do not guess"],
  49. }
  50. def with_root_anchor(context):
  51. persist_root_task_anchor(context, ROOT_ANCHOR)
  52. return context
  53. def report_payload(*, outcome="satisfied"):
  54. return {
  55. "summary": "checked result",
  56. "outcome": outcome,
  57. "validation": {
  58. "hard_passed": outcome == "satisfied",
  59. "open_issues": [] if outcome == "satisfied" else ["still failing"],
  60. },
  61. "next_step_suggestion": {
  62. "direction": "NONE",
  63. "reason": "No additional task is needed",
  64. },
  65. "outputs": [{"kind": "note", "value": "result"}],
  66. "evidence": [{"kind": "test", "value": "passed"}],
  67. "remaining_issues": [] if outcome == "satisfied" else ["still failing"],
  68. }
  69. def validation_payload(child_trace_id, *, outcome="passed"):
  70. if outcome == "passed":
  71. issues = []
  72. retry_from = None
  73. elif outcome == "failed":
  74. issues = ["task needs revision"]
  75. retry_from = "task_definition"
  76. else:
  77. issues = ["validator could not evaluate the task"]
  78. retry_from = None
  79. return ValidationResult(
  80. validator_trace_id=f"{child_trace_id}@validator-fixture",
  81. evaluated_trace_id=child_trace_id,
  82. outcome=outcome,
  83. scope="task",
  84. reason="fixture validation",
  85. issues=issues,
  86. retry_from=retry_from,
  87. ).model_dump()
  88. async def create_recursive_root(store, trace_id, *, state=None, uid="user-1"):
  89. budget = ResourceBudget()
  90. context = {
  91. "agent_mode": "recursive",
  92. "agent_mode_revision": 2,
  93. "agent_depth": 0,
  94. "root_trace_id": trace_id,
  95. RESOURCE_BUDGET_CONTEXT_KEY: budget.to_dict(),
  96. "task_protocol": state or new_task_protocol(),
  97. }
  98. persist_root_task_anchor(context, ROOT_ANCHOR)
  99. await store.create_trace(Trace(
  100. trace_id=trace_id,
  101. mode="agent",
  102. task="root",
  103. uid=uid,
  104. model="fake-model",
  105. context=context,
  106. ))
  107. await ResourceBudgetController(store).initialize(
  108. trace_id,
  109. budget,
  110. initial_agents=1,
  111. )
  112. return await store.get_trace(trace_id)
  113. class FakeToolRegistry:
  114. def get_tool_names(self):
  115. return [
  116. "agent",
  117. "evaluate",
  118. "bash_command",
  119. "goal",
  120. "read_file",
  121. "submit_task_report",
  122. "review_task_result",
  123. "read_context_ref",
  124. ]
  125. class ReportingRunner:
  126. def __init__(self, store):
  127. self.store = store
  128. self.tools = FakeToolRegistry()
  129. self.debug = False
  130. self.stdin_check = None
  131. self._cancel_events = {}
  132. self.configs = []
  133. self.resource_budget = ResourceBudgetController(store)
  134. self.validation_count = 0
  135. async def _resource_budget_for_trace(self, trace_id):
  136. trace = await self.store.get_trace(trace_id)
  137. root_id = trace.context["root_trace_id"]
  138. root = trace if root_id == trace_id else await self.store.get_trace(root_id)
  139. return root_id, ResourceBudget.from_dict(
  140. root.context[RESOURCE_BUDGET_CONTEXT_KEY]
  141. )
  142. async def validate_recursive_trace(
  143. self,
  144. evaluated_trace_id,
  145. *,
  146. scope,
  147. deterministic_failure=None,
  148. **_kwargs,
  149. ):
  150. self.validation_count += 1
  151. evaluated = await self.store.get_trace(evaluated_trace_id)
  152. validator_trace_id = (
  153. f"{evaluated_trace_id}@validator-fixture-{self.validation_count}"
  154. )
  155. if deterministic_failure:
  156. outcome = deterministic_failure.get("outcome", "error")
  157. issues = deterministic_failure.get("issues") or [
  158. deterministic_failure.get("reason", "validation failed")
  159. ]
  160. retry_from = deterministic_failure.get("retry_from")
  161. else:
  162. outcome = "passed"
  163. issues = []
  164. retry_from = None
  165. result = ValidationResult(
  166. validator_trace_id=validator_trace_id,
  167. evaluated_trace_id=evaluated_trace_id,
  168. outcome=outcome,
  169. scope=scope,
  170. reason="fixture validation",
  171. issues=issues,
  172. retry_from=retry_from,
  173. )
  174. await self.store.create_trace(Trace(
  175. trace_id=validator_trace_id,
  176. mode="agent",
  177. task=f"validate {evaluated_trace_id}",
  178. parent_trace_id=evaluated_trace_id,
  179. uid=evaluated.uid,
  180. status="completed",
  181. tools=[],
  182. context=with_root_anchor({
  183. "created_by_tool": "validator",
  184. "root_trace_id": evaluated.context["root_trace_id"],
  185. "agent_depth": evaluated.context["agent_depth"],
  186. }),
  187. ))
  188. return ValidationRun(result=result, trace_id=validator_trace_id)
  189. async def run_result(self, messages, config, on_event=None):
  190. self.configs.append(config)
  191. submitted = await submit_task_report(
  192. task_report=report_payload(),
  193. context={
  194. "store": self.store,
  195. "trace_id": config.trace_id,
  196. "sequence": 3,
  197. },
  198. )
  199. if submitted["status"] != "completed":
  200. raise AssertionError(submitted)
  201. await self.store.update_trace(config.trace_id, status="completed")
  202. return {
  203. "status": "completed",
  204. "summary": "child submitted TaskReport",
  205. "saved_knowledge_ids": [],
  206. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  207. }
  208. class FailedAfterReportingRunner(ReportingRunner):
  209. async def run_result(self, messages, config, on_event=None):
  210. result = await super().run_result(messages, config, on_event)
  211. await self.store.update_trace(
  212. config.trace_id,
  213. status="failed",
  214. error_message="runner failed after report submission",
  215. )
  216. return {
  217. **result,
  218. "status": "failed",
  219. "error": "runner failed after report submission",
  220. }
  221. class ForgedIdentityRunner(ReportingRunner):
  222. async def run_result(self, messages, config, on_event=None):
  223. trace = await self.store.get_trace(config.trace_id)
  224. state = ensure_task_protocol(trace.context)
  225. state["task_report"] = TaskReport(
  226. child_trace_id="someone-elses-trace",
  227. **report_payload(),
  228. ).model_dump()
  229. await self.store.update_trace(
  230. config.trace_id,
  231. context=trace.context,
  232. status="completed",
  233. )
  234. return {
  235. "status": "completed",
  236. "summary": "forged report identity",
  237. "saved_knowledge_ids": [],
  238. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  239. }
  240. class RecursiveTaskProtocolTest(unittest.IsolatedAsyncioTestCase):
  241. async def asyncSetUp(self):
  242. self.temp_dir = tempfile.TemporaryDirectory()
  243. self.store = FileSystemTraceStore(self.temp_dir.name)
  244. self.runner = ReportingRunner(self.store)
  245. self.root_id = "recursive-root"
  246. await create_recursive_root(self.store, self.root_id)
  247. self.tree = GoalTree(
  248. mission="root",
  249. goals=[Goal(id="1", description="delegate", status="in_progress")],
  250. current_id="1",
  251. )
  252. await self.store.update_goal_tree(self.root_id, self.tree)
  253. async def asyncTearDown(self):
  254. self.temp_dir.cleanup()
  255. def context(self, trace_id=None, goal_id="1", tree=None):
  256. return {
  257. "store": self.store,
  258. "trace_id": trace_id or self.root_id,
  259. "goal_id": goal_id,
  260. "runner": self.runner,
  261. "goal_tree": tree or self.tree,
  262. "sequence": 10,
  263. RECURSIVE_CAPABILITY_TOOLS_CONTEXT_KEY: self.runner.tools.get_tool_names(),
  264. }
  265. async def children(self):
  266. return await self.store.list_traces(
  267. parent_trace_id=self.root_id,
  268. created_by_tool="agent",
  269. limit=20,
  270. )
  271. async def test_models_validate_and_round_trip(self):
  272. brief = TaskBrief.model_validate(BRIEF)
  273. self.assertEqual(
  274. {
  275. **BRIEF,
  276. "parent_findings": [],
  277. "context": {},
  278. "context_refs": [],
  279. },
  280. brief.model_dump(exclude_none=True),
  281. )
  282. with self.assertRaises(ValidationError):
  283. TaskBrief(objective="", completion_criteria=[])
  284. with self.assertRaises(ValidationError):
  285. TaskReview(
  286. parent_trace_id="p",
  287. child_trace_id="c",
  288. decision="ACCEPT_REFINE",
  289. reason="continue",
  290. )
  291. invalid_suggestion = report_payload()
  292. invalid_suggestion["next_step_suggestion"] = {
  293. "direction": "NONE",
  294. "reason": "No next task should be attached",
  295. "suggested_next_task": BRIEF,
  296. }
  297. with self.assertRaises(ValidationError):
  298. TaskReport(child_trace_id="child", **invalid_suggestion)
  299. root_submission = await submit_task_report(
  300. task_report=report_payload(),
  301. context={"store": self.store, "trace_id": self.root_id},
  302. )
  303. self.assertEqual("failed", root_submission["status"])
  304. self.assertIn("Root Agent", root_submission["error"])
  305. async def test_child_report_blocks_new_child_until_parent_review(self):
  306. result = await agent(task_brief=BRIEF, context=self.context())
  307. self.assertEqual("completed", result["status"])
  308. child_id = result["sub_trace_id"]
  309. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  310. parent = await self.store.get_trace(self.root_id)
  311. state = ensure_task_protocol(parent.context)
  312. self.assertEqual([child_id], list(state["pending_reviews"]))
  313. tree = await self.store.get_goal_tree(self.root_id)
  314. self.assertEqual("pending_review", tree.find("1").status)
  315. restarted_runner = AgentRunner(tool_registry=get_tool_registry())
  316. restored = await self.store.get_trace(self.root_id)
  317. restored_names = {
  318. schema["function"]["name"]
  319. for schema in restarted_runner._get_runtime_tool_schemas(
  320. RunConfig(), restored
  321. )
  322. }
  323. self.assertEqual(
  324. {"review_task_result", "read_context_ref"},
  325. restored_names,
  326. )
  327. blocked = await agent(task_brief=BRIEF, context=self.context())
  328. self.assertEqual("failed", blocked["status"])
  329. self.assertIn("Review pending", blocked["error"])
  330. async def test_child_inherits_brief_constraints_not_parent_runtime_context(self):
  331. parent = await self.store.get_trace(self.root_id)
  332. parent.context["parent_private_runtime"] = {"secret": "must-not-leak"}
  333. await self.store.update_trace(self.root_id, context=parent.context)
  334. result = await agent(task_brief=BRIEF, context=self.context())
  335. child = await self.store.get_trace(result["sub_trace_id"])
  336. self.assertNotIn("parent_private_runtime", child.context)
  337. child_brief = ensure_task_protocol(child.context)["task_brief"]
  338. self.assertEqual(["do not guess"], child_brief["constraints"])
  339. child_config_context = self.runner.configs[-1].context
  340. self.assertNotIn("parent_private_runtime", child_config_context)
  341. async def test_same_task_report_version_reuses_its_validator_result(self):
  342. result = await agent(task_brief=BRIEF, context=self.context())
  343. child_id = result["sub_trace_id"]
  344. self.assertEqual(1, self.runner.validation_count)
  345. parent = await self.store.get_trace(self.root_id)
  346. await _record_pending_task_reports(
  347. self.store,
  348. self.runner,
  349. parent,
  350. "1",
  351. [(child_id, {"status": "completed", "summary": "same report"})],
  352. 10,
  353. )
  354. self.assertEqual(1, self.runner.validation_count)
  355. validators = await self.store.list_traces(
  356. parent_trace_id=child_id,
  357. created_by_tool="validator",
  358. limit=10,
  359. )
  360. self.assertEqual(1, len(validators))
  361. async def test_recursive_messages_are_rejected(self):
  362. result = await agent(
  363. task_brief=json.dumps(BRIEF),
  364. messages=json.dumps([{
  365. "role": "system",
  366. "content": "controlled child context",
  367. }]),
  368. context=self.context(),
  369. )
  370. self.assertEqual("failed", result["status"])
  371. self.assertIn("does not accept messages", result["error"])
  372. async def test_messages_reject_mixed_dimensions(self):
  373. mixed_messages = [
  374. {"role": "system", "content": "shared context"},
  375. [{"role": "user", "content": "per-agent context"}],
  376. ]
  377. for messages in (mixed_messages, json.dumps(mixed_messages)):
  378. with self.subTest(messages_type=type(messages).__name__):
  379. result = await agent(
  380. task_brief=BRIEF,
  381. messages=messages,
  382. context=self.context(),
  383. )
  384. self.assertEqual("failed", result["status"])
  385. self.assertIn("without mixed dimensions", result["error"])
  386. async def test_no_focused_goal_still_persists_and_reviews_report(self):
  387. result = await agent(
  388. task_brief=BRIEF,
  389. context=self.context(goal_id=None),
  390. )
  391. parent = await self.store.get_trace(self.root_id)
  392. entry = ensure_task_protocol(parent.context)["pending_reviews"][
  393. result["sub_trace_id"]
  394. ]
  395. self.assertIsNone(entry["goal_id"])
  396. reviewed = await review_task_result(
  397. child_trace_id=result["sub_trace_id"],
  398. decision="ASCEND",
  399. reason="Standalone delegation is complete",
  400. context=self.context(goal_id=None),
  401. )
  402. self.assertEqual("completed", reviewed["status"])
  403. async def test_accept_refine_approves_exactly_one_next_child(self):
  404. first = await agent(task_brief=BRIEF, context=self.context())
  405. next_brief = {
  406. "objective": "check the independent gap",
  407. "reason": "the first report identified a separate gap",
  408. "completion_criteria": ["close the gap"],
  409. "expected_outputs": ["a checked gap analysis"],
  410. }
  411. parent = await self.store.get_trace(self.root_id)
  412. ensure_task_protocol(parent.context)["protocol_correction_attempts"] = 2
  413. await self.store.update_trace(self.root_id, context=parent.context)
  414. reviewed = await review_task_result(
  415. child_trace_id=first["sub_trace_id"],
  416. decision="ACCEPT_REFINE",
  417. reason="The gap belongs to this parent",
  418. approved_next_task=json.dumps(next_brief),
  419. context=self.context(),
  420. )
  421. self.assertEqual("completed", reviewed["status"])
  422. self.assertIsNotNone(reviewed["context_ref"])
  423. self.assertIn(
  424. reviewed["context_ref"],
  425. reviewed["next_action_task_brief"]["context_refs"],
  426. )
  427. parent = await self.store.get_trace(self.root_id)
  428. self.assertEqual(
  429. 0,
  430. ensure_task_protocol(parent.context)["protocol_correction_attempts"],
  431. )
  432. schemas = AgentRunner(
  433. tool_registry=get_tool_registry()
  434. )._get_runtime_tool_schemas(RunConfig(), parent)
  435. self.assertEqual(
  436. {"agent", "read_context_ref"},
  437. {schema["function"]["name"] for schema in schemas},
  438. )
  439. wrong = await agent(task_brief=BRIEF, context=self.context())
  440. self.assertEqual("failed", wrong["status"])
  441. self.assertIn("approved next task", wrong["error"])
  442. second = await agent(
  443. task_brief=reviewed["next_action_task_brief"],
  444. context=self.context(),
  445. )
  446. self.assertEqual("completed", second["status"])
  447. self.assertEqual(2, len(await self.children()))
  448. second_child = await self.store.get_trace(second["sub_trace_id"])
  449. self.assertIn(
  450. reviewed["context_ref"]["ref_id"],
  451. second_child.context["context_access"]["snapshots"],
  452. )
  453. parent = await self.store.get_trace(self.root_id)
  454. self.assertEqual([], ensure_task_protocol(parent.context)["next_actions"])
  455. async def test_review_rejects_an_unusable_context_ref_without_consuming_report(self):
  456. first = await agent(task_brief=BRIEF, context=self.context())
  457. invalid_next = {
  458. "objective": "use an unauthorized context ref",
  459. "reason": "exercise approval validation",
  460. "completion_criteria": ["must not be approved"],
  461. "expected_outputs": ["nothing"],
  462. "context_refs": [{
  463. "ref_id": "ctx_000000000000000000000000",
  464. "version": "0" * 64,
  465. }],
  466. }
  467. reviewed = await review_task_result(
  468. child_trace_id=first["sub_trace_id"],
  469. decision="ACCEPT_REFINE",
  470. reason="try an invalid transfer",
  471. approved_next_task=invalid_next,
  472. context=self.context(),
  473. )
  474. self.assertEqual("failed", reviewed["status"])
  475. self.assertIn("not authorized", reviewed["error"])
  476. parent = await self.store.get_trace(self.root_id)
  477. state = ensure_task_protocol(parent.context)
  478. self.assertIn(first["sub_trace_id"], state["pending_reviews"])
  479. self.assertEqual([], state["reviews"])
  480. async def test_oversized_review_snapshot_warns_without_blocking_review(self):
  481. first = await agent(task_brief=BRIEF, context=self.context())
  482. parent = await self.store.get_trace(self.root_id)
  483. state = ensure_task_protocol(parent.context)
  484. state["pending_reviews"][first["sub_trace_id"]]["task_report"]["outputs"] = [
  485. {"payload": "x" * 16_000}
  486. ]
  487. await self.store.update_trace(self.root_id, context=parent.context)
  488. reviewed = await review_task_result(
  489. child_trace_id=first["sub_trace_id"],
  490. decision="ASCEND",
  491. reason="The task passed but its transferable snapshot is too large",
  492. context=self.context(),
  493. )
  494. self.assertEqual("completed", reviewed["status"])
  495. self.assertIsNone(reviewed["context_ref"])
  496. self.assertIn("16000", reviewed["context_ref_warning"])
  497. parent = await self.store.get_trace(self.root_id)
  498. self.assertEqual({}, ensure_task_protocol(parent.context)["pending_reviews"])
  499. async def test_batch_context_refs_are_validated_before_any_trace_is_created(self):
  500. invalid_ref_brief = {
  501. **BRIEF,
  502. "objective": "second child with an unauthorized reference",
  503. "context_refs": [{
  504. "ref_id": "ctx_000000000000000000000000",
  505. "version": "0" * 64,
  506. }],
  507. }
  508. result = await agent(
  509. task_brief=[BRIEF, invalid_ref_brief],
  510. context=self.context(),
  511. )
  512. self.assertEqual("failed", result["status"])
  513. self.assertIn("not authorized", result["error"])
  514. self.assertEqual([], await self.children())
  515. async def test_batch_reports_are_reviewed_one_by_one(self):
  516. briefs = [
  517. BRIEF,
  518. {
  519. "objective": "research second question",
  520. "reason": "the batch needs a second independent result",
  521. "completion_criteria": ["provide second conclusion"],
  522. "expected_outputs": ["a second conclusion"],
  523. },
  524. ]
  525. result = await agent(task_brief=briefs, context=self.context())
  526. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  527. self.assertEqual(2, len(result["task_reports"]))
  528. premature = await review_task_result(
  529. child_trace_id=child_ids[0],
  530. decision="ASCEND",
  531. reason="Cannot finish while another report is pending",
  532. context=self.context(),
  533. )
  534. self.assertEqual("failed", premature["status"])
  535. self.assertIn("final pending report", premature["error"])
  536. first_review = await review_task_result(
  537. child_trace_id=child_ids[0],
  538. decision="REJECT_REFINE",
  539. reason="No extra task needed for this report",
  540. context=self.context(),
  541. )
  542. self.assertEqual(1, first_review["pending_review_count"])
  543. tree = await self.store.get_goal_tree(self.root_id)
  544. self.assertEqual("pending_review", tree.find("1").status)
  545. final_review = await review_task_result(
  546. child_trace_id=child_ids[1],
  547. decision="ASCEND",
  548. reason="All completion criteria are satisfied",
  549. context=self.context(),
  550. )
  551. self.assertEqual(0, final_review["pending_review_count"])
  552. tree = await self.store.get_goal_tree(self.root_id)
  553. self.assertEqual("completed", tree.find("1").status)
  554. duplicate = await review_task_result(
  555. child_trace_id=child_ids[1],
  556. decision="ASCEND",
  557. reason="Duplicate review must fail",
  558. context=self.context(),
  559. )
  560. self.assertEqual("failed", duplicate["status"])
  561. self.assertIn("no pending TaskReport", duplicate["error"])
  562. async def test_terminal_review_waits_for_queued_next_action(self):
  563. result = await agent(
  564. task_brief=[
  565. BRIEF,
  566. {
  567. "objective": "second batch child",
  568. "reason": "exercise queued batch behavior",
  569. "completion_criteria": ["finish the second child"],
  570. "expected_outputs": ["the second child result"],
  571. },
  572. ],
  573. context=self.context(),
  574. )
  575. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  576. next_brief = {
  577. "objective": "approved follow-up",
  578. "reason": "the parent approved one follow-up",
  579. "completion_criteria": ["finish the approved follow-up"],
  580. "expected_outputs": ["the approved follow-up result"],
  581. }
  582. accepted = await review_task_result(
  583. child_trace_id=child_ids[0],
  584. decision="ACCEPT_REFINE",
  585. reason="A separate follow-up is required",
  586. approved_next_task=next_brief,
  587. context=self.context(),
  588. )
  589. self.assertEqual("completed", accepted["status"])
  590. blocked = await review_task_result(
  591. child_trace_id=child_ids[1],
  592. decision="ASCEND",
  593. reason="Cannot ascend before the accepted follow-up",
  594. context=self.context(),
  595. )
  596. self.assertEqual("failed", blocked["status"])
  597. self.assertIn("approved next action", blocked["error"])
  598. tree = await self.store.get_goal_tree(self.root_id)
  599. self.assertEqual("pending_review", tree.find("1").status)
  600. final_non_terminal = await review_task_result(
  601. child_trace_id=child_ids[1],
  602. decision="REJECT_REFINE",
  603. reason="This report adds no other follow-up",
  604. context=self.context(),
  605. )
  606. self.assertEqual("completed", final_non_terminal["status"])
  607. tree = await self.store.get_goal_tree(self.root_id)
  608. self.assertEqual("in_progress", tree.find("1").status)
  609. async def test_runner_failure_overrides_stale_satisfied_report(self):
  610. self.runner = FailedAfterReportingRunner(self.store)
  611. result = await agent(task_brief=BRIEF, context=self.context())
  612. self.assertEqual("failed", result["status"])
  613. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  614. child = await self.store.get_trace(result["sub_trace_id"])
  615. child_state = ensure_task_protocol(child.context)
  616. self.assertEqual(1, len(child_state["report_history"]))
  617. self.assertEqual("satisfied", child_state["report_history"][0]["outcome"])
  618. invalid_ascend = await review_task_result(
  619. child_trace_id=result["sub_trace_id"],
  620. decision="ASCEND",
  621. reason="A failed execution cannot ascend",
  622. context=self.context(),
  623. )
  624. self.assertEqual("failed", invalid_ascend["status"])
  625. self.assertIn("only allow", invalid_ascend["error"])
  626. async def test_submit_and_review_reject_child_anchor_that_differs_from_root(self):
  627. child_context = with_root_anchor({
  628. "agent_mode": "recursive",
  629. "agent_mode_revision": 2,
  630. "agent_depth": 1,
  631. "root_trace_id": self.root_id,
  632. "created_by_tool": "agent",
  633. "task_protocol": new_task_protocol(BRIEF),
  634. })
  635. persist_root_task_anchor(child_context, {
  636. **ROOT_ANCHOR,
  637. "objective": "self-consistent but unauthorized root objective",
  638. })
  639. child_id = "mismatched-anchor-child"
  640. await self.store.create_trace(Trace(
  641. trace_id=child_id,
  642. mode="agent",
  643. task="child",
  644. parent_trace_id=self.root_id,
  645. parent_goal_id="1",
  646. uid="user-1",
  647. model="fake-model",
  648. context=child_context,
  649. ))
  650. submitted = await submit_task_report(
  651. task_report=report_payload(),
  652. context={"store": self.store, "trace_id": child_id},
  653. )
  654. self.assertEqual("failed", submitted["status"])
  655. self.assertIn("does not match", submitted["error"])
  656. parent = await self.store.get_trace(self.root_id)
  657. parent_state = ensure_task_protocol(parent.context)
  658. parent_state["pending_reviews"][child_id] = {
  659. "goal_id": "1",
  660. "task_report": TaskReport(
  661. child_trace_id=child_id,
  662. **report_payload(),
  663. ).model_dump(),
  664. "validation_result": validation_payload(child_id),
  665. "received_at_sequence": 1,
  666. }
  667. await self.store.update_trace(self.root_id, context=parent.context)
  668. reviewed = await review_task_result(
  669. child_trace_id=child_id,
  670. decision="ASCEND",
  671. reason="must not accept a different root anchor",
  672. context=self.context(),
  673. )
  674. self.assertEqual("failed", reviewed["status"])
  675. self.assertIn("does not match", reviewed["error"])
  676. parent = await self.store.get_trace(self.root_id)
  677. state = ensure_task_protocol(parent.context)
  678. self.assertIn(child_id, state["pending_reviews"])
  679. self.assertEqual([], state["reviews"])
  680. async def test_review_rejects_wrong_owner_even_if_report_is_injected(self):
  681. first = await agent(task_brief=BRIEF, context=self.context())
  682. child_id = first["sub_trace_id"]
  683. await self.store.update_trace(child_id, uid="another-user")
  684. rejected = await review_task_result(
  685. child_trace_id=child_id,
  686. decision="ASCEND",
  687. reason="Owner mismatch must be rejected",
  688. context=self.context(),
  689. )
  690. self.assertEqual("failed", rejected["status"])
  691. self.assertIn("same owner", rejected["error"])
  692. async def test_report_identity_must_match_direct_child(self):
  693. self.runner = ForgedIdentityRunner(self.store)
  694. result = await agent(task_brief=BRIEF, context=self.context())
  695. child_id = result["sub_trace_id"]
  696. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  697. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  698. parent = await self.store.get_trace(self.root_id)
  699. pending_report = ensure_task_protocol(parent.context)["pending_reviews"][
  700. child_id
  701. ]["task_report"]
  702. pending_report["child_trace_id"] = "corrupted-after-recording"
  703. await self.store.update_trace(self.root_id, context=parent.context)
  704. rejected = await review_task_result(
  705. child_trace_id=child_id,
  706. decision="FAIL",
  707. reason="Corrupted identity cannot be reviewed",
  708. context=self.context(),
  709. )
  710. self.assertEqual("failed", rejected["status"])
  711. self.assertIn("does not match", rejected["error"])
  712. async def test_remote_agent_keeps_legacy_task_interface(self):
  713. remote_result = {
  714. "status": "completed",
  715. "summary": "remote result",
  716. }
  717. with patch(
  718. "cyber_agent.tools.builtin.subagent._run_remote_agent",
  719. new=AsyncMock(return_value=remote_result),
  720. ) as remote_call:
  721. result = await agent(
  722. task="remote free text",
  723. agent_type="remote_librarian",
  724. )
  725. self.assertEqual(remote_result, result)
  726. self.assertEqual("remote free text", remote_call.await_args.kwargs["task"])
  727. async def test_revise_child_reuses_trace_and_preserves_report_history(self):
  728. first = await agent(task_brief=BRIEF, context=self.context())
  729. child_id = first["sub_trace_id"]
  730. reviewed = await review_task_result(
  731. child_trace_id=child_id,
  732. decision="REVISE_CHILD",
  733. reason="Please strengthen the evidence",
  734. context=self.context(),
  735. )
  736. self.assertEqual("completed", reviewed["status"])
  737. cleared_child = await self.store.get_trace(child_id)
  738. self.assertEqual({}, cleared_child.context["context_access"]["snapshots"])
  739. wrong = await agent(
  740. task_brief={
  741. "objective": "silently replace the reviewed task",
  742. "reason": "attempt a mismatched revision",
  743. "completion_criteria": ["this must be rejected"],
  744. "expected_outputs": ["a rejected replacement"],
  745. },
  746. continue_from=child_id,
  747. context=self.context(),
  748. )
  749. self.assertEqual("failed", wrong["status"])
  750. self.assertIn("approved revision", wrong["error"])
  751. revised_brief = reviewed["next_action_task_brief"]
  752. revised = await agent(
  753. task_brief=revised_brief,
  754. continue_from=child_id,
  755. context=self.context(),
  756. )
  757. self.assertEqual(child_id, revised["sub_trace_id"])
  758. self.assertTrue(revised["continue_from"])
  759. self.assertEqual(1, len(await self.children()))
  760. child = await self.store.get_trace(child_id)
  761. self.assertEqual(1, len(ensure_task_protocol(child.context)["report_history"]))
  762. self.assertEqual(
  763. TaskBrief.model_validate(revised_brief).model_dump(),
  764. ensure_task_protocol(child.context)["task_brief"],
  765. )
  766. self.assertEqual(2, ensure_task_protocol(child.context)["task_brief_version"])
  767. self.assertEqual(
  768. 1,
  769. len(ensure_task_protocol(child.context)["task_brief_history"]),
  770. )
  771. async def test_review_cannot_reserve_more_children_than_quota(self):
  772. briefs = [
  773. {
  774. "objective": f"batch child {index}",
  775. "reason": "fill the local child quota",
  776. "completion_criteria": [f"finish child {index}"],
  777. "expected_outputs": [f"result for child {index}"],
  778. }
  779. for index in range(5)
  780. ]
  781. result = await agent(task_brief=briefs, context=self.context())
  782. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  783. next_brief = {
  784. "objective": "one allowed reserved child",
  785. "reason": "use the final available child slot",
  786. "completion_criteria": ["use the final available slot"],
  787. "expected_outputs": ["one final child result"],
  788. }
  789. first = await review_task_result(
  790. child_trace_id=child_ids[0],
  791. decision="ACCEPT_REFINE",
  792. reason="Reserve the sixth and final child slot",
  793. approved_next_task=next_brief,
  794. context=self.context(),
  795. )
  796. self.assertEqual("completed", first["status"])
  797. overflow = await review_task_result(
  798. child_trace_id=child_ids[1],
  799. decision="DESCEND_AGAIN",
  800. reason="A seventh child must not be approved",
  801. approved_next_task={
  802. "objective": "overflow child",
  803. "reason": "exercise quota rejection",
  804. "completion_criteria": ["must never be scheduled"],
  805. "expected_outputs": ["no output because creation is rejected"],
  806. },
  807. context=self.context(),
  808. )
  809. self.assertEqual("failed", overflow["status"])
  810. self.assertIn("reserved=1", overflow["error"])
  811. parent = await self.store.get_trace(self.root_id)
  812. state = ensure_task_protocol(parent.context)
  813. self.assertEqual(1, len(state["next_actions"]))
  814. self.assertIn(child_ids[1], state["pending_reviews"])
  815. async def test_recursive_goal_completion_does_not_cascade_or_select_sibling(self):
  816. tree = GoalTree(mission="recursive")
  817. parent = tree.add_goals(["parent"])[0]
  818. first, second = tree.add_goals(["first", "second"], parent_id=parent.id)
  819. tree.focus(parent.id)
  820. tree.focus(first.id)
  821. tree.complete(
  822. first.id,
  823. "done",
  824. auto_advance=False,
  825. cascade_parent=False,
  826. )
  827. self.assertEqual(parent.id, tree.current_id)
  828. self.assertEqual("pending", second.status)
  829. self.assertNotEqual("completed", parent.status)
  830. async def test_rewind_before_review_restores_report_and_review_gate(self):
  831. delegated = await agent(task_brief=BRIEF, context=self.context())
  832. child_id = delegated["sub_trace_id"]
  833. reviewed = await review_task_result(
  834. child_trace_id=child_id,
  835. decision="ASCEND",
  836. reason="Initially accepted before rewind",
  837. context={**self.context(), "sequence": 20},
  838. )
  839. self.assertEqual("completed", reviewed["status"])
  840. for sequence in (1, 15):
  841. await self.store.add_message(Message.create(
  842. trace_id=self.root_id,
  843. role="user",
  844. sequence=sequence,
  845. content=f"message-{sequence}",
  846. ))
  847. tree = await self.store.get_goal_tree(self.root_id)
  848. runner = AgentRunner(
  849. trace_store=self.store,
  850. tool_registry=get_tool_registry(),
  851. )
  852. await runner._rewind(self.root_id, 15, tree)
  853. parent = await self.store.get_trace(self.root_id)
  854. state = ensure_task_protocol(parent.context)
  855. self.assertIn(child_id, state["pending_reviews"])
  856. self.assertEqual([], state["reviews"])
  857. restored_tree = await self.store.get_goal_tree(self.root_id)
  858. self.assertEqual("pending_review", restored_tree.find("1").status)
  859. schemas = runner._get_runtime_tool_schemas(RunConfig(), parent)
  860. self.assertEqual(
  861. {"review_task_result", "read_context_ref"},
  862. {schema["function"]["name"] for schema in schemas},
  863. )
  864. async def test_grandchild_report_is_reviewed_only_by_direct_child(self):
  865. child_id = "manual-child"
  866. child_context = with_root_anchor({
  867. "agent_mode": "recursive",
  868. "agent_mode_revision": 2,
  869. "agent_depth": 1,
  870. "root_trace_id": self.root_id,
  871. "created_by_tool": "agent",
  872. "task_protocol": new_task_protocol(BRIEF),
  873. })
  874. await self.store.create_trace(Trace(
  875. trace_id=child_id,
  876. mode="agent",
  877. task="child",
  878. parent_trace_id=self.root_id,
  879. uid="user-1",
  880. model="fake-model",
  881. context=child_context,
  882. ))
  883. child_tree = GoalTree(
  884. mission="child",
  885. goals=[Goal(id="1", description="delegate grandchild", status="in_progress")],
  886. current_id="1",
  887. )
  888. await self.store.update_goal_tree(child_id, child_tree)
  889. grandchild = await agent(
  890. task_brief=BRIEF,
  891. context=self.context(trace_id=child_id, tree=child_tree),
  892. )
  893. child = await self.store.get_trace(child_id)
  894. child_state = ensure_task_protocol(child.context)
  895. self.assertIn(grandchild["sub_trace_id"], child_state["pending_reviews"])
  896. root = await self.store.get_trace(self.root_id)
  897. self.assertEqual({}, ensure_task_protocol(root.context)["pending_reviews"])
  898. blocked = await agent(
  899. task_brief=BRIEF,
  900. context=self.context(trace_id=child_id, tree=child_tree),
  901. )
  902. self.assertEqual("failed", blocked["status"])
  903. reviewed = await review_task_result(
  904. child_trace_id=grandchild["sub_trace_id"],
  905. decision="ASCEND",
  906. reason="Grandchild satisfied the child criteria",
  907. context=self.context(trace_id=child_id, tree=child_tree),
  908. )
  909. self.assertEqual("completed", reviewed["status"])
  910. submitted = await submit_task_report(
  911. task_report=json.dumps(report_payload()),
  912. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  913. )
  914. self.assertEqual("completed", submitted["status"])
  915. async def test_child_must_execute_approved_action_before_own_report(self):
  916. child_id = "child-with-approved-action"
  917. child_context = with_root_anchor({
  918. "agent_mode": "recursive",
  919. "agent_mode_revision": 2,
  920. "agent_depth": 1,
  921. "root_trace_id": self.root_id,
  922. "created_by_tool": "agent",
  923. "task_protocol": new_task_protocol(BRIEF),
  924. })
  925. await self.store.create_trace(Trace(
  926. trace_id=child_id,
  927. mode="agent",
  928. task="child",
  929. parent_trace_id=self.root_id,
  930. uid="user-1",
  931. model="fake-model",
  932. context=child_context,
  933. ))
  934. child_tree = GoalTree(
  935. mission="child",
  936. goals=[Goal(id="1", description="delegate", status="in_progress")],
  937. current_id="1",
  938. )
  939. await self.store.update_goal_tree(child_id, child_tree)
  940. child_tool_context = self.context(trace_id=child_id, tree=child_tree)
  941. first = await agent(task_brief=BRIEF, context=child_tool_context)
  942. next_brief = {
  943. "objective": "check a second grandchild task",
  944. "reason": "the child approved another direct task",
  945. "completion_criteria": ["provide a second checked conclusion"],
  946. "expected_outputs": ["a second checked conclusion"],
  947. }
  948. reviewed = await review_task_result(
  949. child_trace_id=first["sub_trace_id"],
  950. decision="ACCEPT_REFINE",
  951. reason="A separate gap remains",
  952. approved_next_task=next_brief,
  953. context=child_tool_context,
  954. )
  955. self.assertEqual("completed", reviewed["status"])
  956. blocked_report = await submit_task_report(
  957. task_report=report_payload(),
  958. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  959. )
  960. self.assertEqual("failed", blocked_report["status"])
  961. self.assertIn("approved next action", blocked_report["error"])
  962. second = await agent(
  963. task_brief=reviewed["next_action_task_brief"],
  964. context=child_tool_context,
  965. )
  966. self.assertEqual("completed", second["status"])
  967. child = await self.store.get_trace(child_id)
  968. state = ensure_task_protocol(child.context)
  969. self.assertEqual([], state["next_actions"])
  970. self.assertIn(second["sub_trace_id"], state["pending_reviews"])
  971. async def test_goal_tool_cannot_bypass_queued_next_action(self):
  972. root = await self.store.get_trace(self.root_id)
  973. state = ensure_task_protocol(root.context)
  974. state["next_actions"] = [{
  975. "decision": "ACCEPT_REFINE",
  976. "child_trace_id": "reviewed-child",
  977. "goal_id": "1",
  978. "task_brief": BRIEF,
  979. "created_at_sequence": 5,
  980. }]
  981. await self.store.update_trace(self.root_id, context=root.context)
  982. result = await goal_tool(
  983. tree=self.tree,
  984. store=self.store,
  985. trace_id=self.root_id,
  986. done="attempt to bypass the approved action",
  987. )
  988. self.assertIn("已批准的下一步待执行", result)
  989. persisted_tree = await self.store.get_goal_tree(self.root_id)
  990. self.assertEqual("in_progress", persisted_tree.find("1").status)
  991. async def test_failed_report_allows_only_revise_or_fail(self):
  992. child_id = "failed-child"
  993. failed_report = TaskReport(
  994. child_trace_id=child_id,
  995. **report_payload(outcome="failed"),
  996. )
  997. child_context = with_root_anchor({
  998. "agent_mode": "recursive",
  999. "agent_mode_revision": 2,
  1000. "agent_depth": 1,
  1001. "root_trace_id": self.root_id,
  1002. "created_by_tool": "agent",
  1003. "task_protocol": new_task_protocol(BRIEF),
  1004. })
  1005. child_context["task_protocol"]["task_report"] = failed_report.model_dump()
  1006. await self.store.create_trace(Trace(
  1007. trace_id=child_id,
  1008. mode="agent",
  1009. task="failed",
  1010. parent_trace_id=self.root_id,
  1011. parent_goal_id="1",
  1012. uid="user-1",
  1013. context=child_context,
  1014. ))
  1015. root = await self.store.get_trace(self.root_id)
  1016. state = ensure_task_protocol(root.context)
  1017. state["pending_reviews"][child_id] = {
  1018. "goal_id": "1",
  1019. "task_report": failed_report.model_dump(),
  1020. "validation_result": validation_payload(child_id, outcome="failed"),
  1021. "received_at_sequence": 2,
  1022. }
  1023. await self.store.update_trace(self.root_id, context=root.context)
  1024. rejected = await review_task_result(
  1025. child_trace_id=child_id,
  1026. decision="ASCEND",
  1027. reason="incorrect",
  1028. context=self.context(),
  1029. )
  1030. self.assertEqual("failed", rejected["status"])
  1031. self.assertIn("only allow", rejected["error"])
  1032. class RunnerCompletionGateTest(unittest.IsolatedAsyncioTestCase):
  1033. async def asyncSetUp(self):
  1034. self.temp_dir = tempfile.TemporaryDirectory()
  1035. self.store = FileSystemTraceStore(self.temp_dir.name)
  1036. self.trace_id = "gate-child"
  1037. await create_recursive_root(self.store, "root")
  1038. controller = ResourceBudgetController(self.store)
  1039. budget = ResourceBudget()
  1040. await controller.reserve_agents("root", budget, 1)
  1041. await self.store.create_trace(Trace(
  1042. trace_id=self.trace_id,
  1043. mode="agent",
  1044. task="must report",
  1045. parent_trace_id="root",
  1046. uid="user-1",
  1047. context=with_root_anchor({
  1048. "agent_mode": "recursive",
  1049. "agent_mode_revision": 2,
  1050. "agent_depth": 1,
  1051. "root_trace_id": "root",
  1052. "created_by_tool": "agent",
  1053. "task_protocol": new_task_protocol(BRIEF),
  1054. }),
  1055. ))
  1056. await self.store.update_goal_tree(self.trace_id, GoalTree(mission="must report"))
  1057. async def asyncTearDown(self):
  1058. self.temp_dir.cleanup()
  1059. async def test_missing_report_becomes_protocol_error_after_two_corrections(self):
  1060. calls = 0
  1061. async def llm_call(**kwargs):
  1062. nonlocal calls
  1063. calls += 1
  1064. return {
  1065. "content": "I am done",
  1066. "tool_calls": [],
  1067. "finish_reason": "stop",
  1068. }
  1069. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1070. result = await runner.run_result(
  1071. messages=[],
  1072. config=RunConfig(
  1073. trace_id=self.trace_id,
  1074. max_iterations=8,
  1075. knowledge=KnowledgeConfig(
  1076. enable_extraction=False,
  1077. enable_completion_extraction=False,
  1078. enable_injection=False,
  1079. ),
  1080. ),
  1081. )
  1082. self.assertEqual("failed", result["status"])
  1083. trace = await self.store.get_trace(self.trace_id)
  1084. report = ensure_task_protocol(trace.context)["task_report"]
  1085. self.assertEqual("protocol_error", report["outcome"])
  1086. self.assertGreaterEqual(calls, 3)
  1087. async def test_valid_report_allows_child_to_finish(self):
  1088. calls = 0
  1089. async def llm_call(**kwargs):
  1090. nonlocal calls
  1091. calls += 1
  1092. if kwargs.get("tools") == []:
  1093. return {
  1094. "content": json.dumps({
  1095. "outcome": "passed",
  1096. "scope": "root",
  1097. "reason": "the corrected behavior meets the criterion",
  1098. "issues": [],
  1099. "retry_from": None,
  1100. }),
  1101. "tool_calls": [],
  1102. "finish_reason": "stop",
  1103. }
  1104. if calls == 1:
  1105. return {
  1106. "content": "",
  1107. "tool_calls": [{
  1108. "id": "submit-1",
  1109. "type": "function",
  1110. "function": {
  1111. "name": "submit_task_report",
  1112. "arguments": json.dumps({
  1113. "task_report": report_payload(),
  1114. }),
  1115. },
  1116. }],
  1117. "finish_reason": "tool_calls",
  1118. }
  1119. return {
  1120. "content": "Structured report submitted.",
  1121. "tool_calls": [],
  1122. "finish_reason": "stop",
  1123. }
  1124. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1125. result = await runner.run_result(
  1126. messages=[],
  1127. config=RunConfig(
  1128. trace_id=self.trace_id,
  1129. max_iterations=4,
  1130. knowledge=KnowledgeConfig(
  1131. enable_extraction=False,
  1132. enable_completion_extraction=False,
  1133. enable_injection=False,
  1134. ),
  1135. ),
  1136. )
  1137. self.assertEqual("completed", result["status"])
  1138. trace = await self.store.get_trace(self.trace_id)
  1139. self.assertEqual(
  1140. "satisfied",
  1141. ensure_task_protocol(trace.context)["task_report"]["outcome"],
  1142. )
  1143. async def test_lifecycle_tool_batch_is_rejected_before_execution(self):
  1144. root_id = "batch-gate-root"
  1145. await create_recursive_root(self.store, root_id)
  1146. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  1147. calls = 0
  1148. async def llm_call(**kwargs):
  1149. nonlocal calls
  1150. calls += 1
  1151. if kwargs.get("tools") == []:
  1152. return {
  1153. "content": json.dumps({
  1154. "outcome": "passed",
  1155. "scope": "root",
  1156. "reason": "the corrected lifecycle behavior is valid",
  1157. "issues": [],
  1158. "retry_from": None,
  1159. }),
  1160. "tool_calls": [],
  1161. "finish_reason": "stop",
  1162. }
  1163. if calls == 1:
  1164. return {
  1165. "content": "",
  1166. "tool_calls": [
  1167. {
  1168. "id": "agent-1",
  1169. "type": "function",
  1170. "function": {
  1171. "name": "agent",
  1172. "arguments": json.dumps({
  1173. "task_brief": BRIEF,
  1174. }),
  1175. },
  1176. },
  1177. {
  1178. "id": "context-1",
  1179. "type": "function",
  1180. "function": {
  1181. "name": "get_current_context",
  1182. "arguments": "{}",
  1183. },
  1184. },
  1185. ],
  1186. "finish_reason": "tool_calls",
  1187. }
  1188. return {
  1189. "content": "Corrected the invalid batch.",
  1190. "tool_calls": [],
  1191. "finish_reason": "stop",
  1192. }
  1193. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1194. result = await runner.run_result(
  1195. messages=[],
  1196. config=RunConfig(
  1197. trace_id=root_id,
  1198. max_iterations=3,
  1199. knowledge=KnowledgeConfig(
  1200. enable_extraction=False,
  1201. enable_completion_extraction=False,
  1202. enable_injection=False,
  1203. ),
  1204. ),
  1205. )
  1206. self.assertEqual("completed", result["status"])
  1207. children = await self.store.list_traces(
  1208. parent_trace_id=root_id,
  1209. created_by_tool="agent",
  1210. limit=10,
  1211. )
  1212. self.assertEqual([], children)
  1213. tool_messages = [
  1214. message
  1215. for message in await self.store.get_trace_messages(root_id)
  1216. if message.role == "tool"
  1217. ]
  1218. self.assertEqual(2, len(tool_messages))
  1219. self.assertTrue(all(
  1220. "only tool call" in str(message.content)
  1221. for message in tool_messages
  1222. ))
  1223. async def test_rewind_clears_protocol_state_after_cutoff(self):
  1224. trace = await self.store.get_trace(self.trace_id)
  1225. state = ensure_task_protocol(trace.context)
  1226. original_brief = TaskBrief.model_validate(state["task_brief"])
  1227. revised_brief = original_brief.model_copy(
  1228. update={"objective": "revised after the rewind cutoff"}
  1229. )
  1230. replace_task_brief(
  1231. state,
  1232. revised_brief,
  1233. effective_at_sequence=8,
  1234. )
  1235. late_snapshot = create_context_snapshot(
  1236. kind="reviewed_task_result",
  1237. summary="granted after cutoff",
  1238. source_trace_id="late-child",
  1239. root_trace_id="root",
  1240. uid="user-1",
  1241. content={"result": "must be removed by rewind"},
  1242. granted_at_sequence=8,
  1243. )
  1244. add_context_snapshot(
  1245. trace.context,
  1246. late_snapshot,
  1247. root_task_anchor=ROOT_ANCHOR,
  1248. task_brief=revised_brief,
  1249. )
  1250. report = TaskReport(
  1251. child_trace_id=self.trace_id,
  1252. **report_payload(),
  1253. ).model_dump()
  1254. state["task_report"] = report
  1255. state["task_report_submitted_at_sequence"] = 5
  1256. state["pending_reviews"] = {
  1257. "grandchild": {
  1258. "goal_id": "1",
  1259. "task_report": report,
  1260. "validation_result": validation_payload("grandchild"),
  1261. "received_at_sequence": 6,
  1262. }
  1263. }
  1264. state["reviews"] = [{"reviewed_at_sequence": 7}]
  1265. state["next_actions"] = [{"created_at_sequence": 8}]
  1266. await self.store.update_trace(self.trace_id, context=trace.context)
  1267. tree = GoalTree(
  1268. mission="rewind",
  1269. goals=[Goal(id="1", description="active", status="pending_review")],
  1270. current_id="1",
  1271. )
  1272. await self.store.update_goal_tree(self.trace_id, tree)
  1273. for sequence in (1, 2):
  1274. await self.store.add_message(Message.create(
  1275. trace_id=self.trace_id,
  1276. role="user",
  1277. sequence=sequence,
  1278. content=f"message-{sequence}",
  1279. ))
  1280. runner = AgentRunner(trace_store=self.store)
  1281. cutoff = await runner._rewind(self.trace_id, 2, tree)
  1282. self.assertEqual(3, cutoff)
  1283. rewound = await self.store.get_trace(self.trace_id)
  1284. state = ensure_task_protocol(rewound.context)
  1285. self.assertIsNone(state["task_report"])
  1286. self.assertEqual(1, len(state["report_history"]))
  1287. self.assertEqual({}, state["pending_reviews"])
  1288. self.assertEqual([], state["reviews"])
  1289. self.assertEqual([], state["next_actions"])
  1290. self.assertEqual(1, state["task_brief_version"])
  1291. self.assertEqual(original_brief.model_dump(), state["task_brief"])
  1292. self.assertEqual([], state["task_brief_history"])
  1293. self.assertEqual({}, rewound.context["context_access"]["snapshots"])
  1294. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  1295. self.assertEqual("pending", rewound_tree.find("1").status)
  1296. async def test_rewind_restores_preserved_pending_review_projection(self):
  1297. trace = await self.store.get_trace(self.trace_id)
  1298. state = ensure_task_protocol(trace.context)
  1299. report = TaskReport(
  1300. child_trace_id="preserved-grandchild",
  1301. **report_payload(),
  1302. ).model_dump()
  1303. state["pending_reviews"] = {
  1304. "preserved-grandchild": {
  1305. "goal_id": "1",
  1306. "task_report": report,
  1307. "validation_result": validation_payload(
  1308. "preserved-grandchild"
  1309. ),
  1310. "received_at_sequence": 2,
  1311. }
  1312. }
  1313. await self.store.update_trace(self.trace_id, context=trace.context)
  1314. tree = GoalTree(
  1315. mission="rewind",
  1316. goals=[Goal(id="1", description="review", status="pending_review")],
  1317. current_id="1",
  1318. )
  1319. await self.store.update_goal_tree(self.trace_id, tree)
  1320. for sequence in (1, 2):
  1321. await self.store.add_message(Message.create(
  1322. trace_id=self.trace_id,
  1323. role="user",
  1324. sequence=sequence,
  1325. content=f"message-{sequence}",
  1326. ))
  1327. runner = AgentRunner(trace_store=self.store)
  1328. await runner._rewind(self.trace_id, 2, tree)
  1329. rewound = await self.store.get_trace(self.trace_id)
  1330. self.assertIn(
  1331. "preserved-grandchild",
  1332. ensure_task_protocol(rewound.context)["pending_reviews"],
  1333. )
  1334. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  1335. self.assertEqual("pending_review", rewound_tree.find("1").status)
  1336. async def test_root_cannot_finish_with_unconsumed_next_action(self):
  1337. root_id = "next-action-gate-root"
  1338. state = new_task_protocol()
  1339. state["next_actions"] = [{
  1340. "decision": "ACCEPT_REFINE",
  1341. "child_trace_id": "reviewed-child",
  1342. "goal_id": None,
  1343. "task_brief": BRIEF,
  1344. "created_at_sequence": 1,
  1345. }]
  1346. await create_recursive_root(self.store, root_id, state=state)
  1347. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  1348. calls = 0
  1349. async def llm_call(**kwargs):
  1350. nonlocal calls
  1351. calls += 1
  1352. return {
  1353. "content": "Trying to finish without the approved action.",
  1354. "tool_calls": [],
  1355. "finish_reason": "stop",
  1356. }
  1357. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1358. result = await runner.run_result(
  1359. messages=[],
  1360. config=RunConfig(
  1361. trace_id=root_id,
  1362. max_iterations=5,
  1363. knowledge=KnowledgeConfig(
  1364. enable_extraction=False,
  1365. enable_completion_extraction=False,
  1366. enable_injection=False,
  1367. ),
  1368. ),
  1369. )
  1370. self.assertEqual("failed", result["status"])
  1371. self.assertEqual(3, calls)
  1372. root = await self.store.get_trace(root_id)
  1373. self.assertEqual(1, len(ensure_task_protocol(root.context)["next_actions"]))
  1374. class RuntimeProtocolSchemaTest(unittest.TestCase):
  1375. def setUp(self):
  1376. self.runner = AgentRunner(tool_registry=get_tool_registry())
  1377. def names(self, schemas):
  1378. return {schema["function"]["name"] for schema in schemas}
  1379. def test_legacy_and_recursive_runtime_schemas_are_isolated(self):
  1380. legacy = Trace(
  1381. trace_id="legacy",
  1382. mode="agent",
  1383. context={"agent_mode": "legacy", "agent_mode_revision": 1},
  1384. )
  1385. legacy_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), legacy)
  1386. self.assertNotIn("submit_task_report", self.names(legacy_schemas))
  1387. self.assertNotIn("review_task_result", self.names(legacy_schemas))
  1388. legacy_agent = next(
  1389. item for item in legacy_schemas if item["function"]["name"] == "agent"
  1390. )
  1391. properties = legacy_agent["function"]["parameters"]["properties"]
  1392. self.assertIn("task", properties)
  1393. self.assertNotIn("task_brief", properties)
  1394. child = Trace(
  1395. trace_id="child",
  1396. mode="agent",
  1397. parent_trace_id="root",
  1398. context={
  1399. "agent_mode": "recursive",
  1400. "agent_mode_revision": 2,
  1401. "task_protocol": new_task_protocol(BRIEF),
  1402. },
  1403. )
  1404. child_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1405. self.assertIn("submit_task_report", self.names(child_schemas))
  1406. self.assertNotIn("review_task_result", self.names(child_schemas))
  1407. self.assertNotIn("evaluate", self.names(child_schemas))
  1408. recursive_agent = next(
  1409. item for item in child_schemas if item["function"]["name"] == "agent"
  1410. )
  1411. recursive_properties = recursive_agent["function"]["parameters"]["properties"]
  1412. self.assertNotIn("messages", recursive_properties)
  1413. brief_object = recursive_properties["task_brief"]["oneOf"][0]
  1414. self.assertEqual(
  1415. {
  1416. "objective",
  1417. "reason",
  1418. "completion_criteria",
  1419. "expected_outputs",
  1420. },
  1421. set(brief_object["required"]),
  1422. )
  1423. state = ensure_task_protocol(child.context)
  1424. state["pending_reviews"]["grandchild"] = {"task_report": {}}
  1425. review_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1426. self.assertEqual(
  1427. {"review_task_result", "read_context_ref"},
  1428. self.names(review_schemas),
  1429. )
  1430. def test_protocol_tools_publish_nested_object_schemas(self):
  1431. schemas = {
  1432. schema["function"]["name"]: schema
  1433. for schema in get_tool_registry().get_schemas([
  1434. "submit_task_report",
  1435. "review_task_result",
  1436. ])
  1437. }
  1438. report_schema = schemas["submit_task_report"]["function"]["parameters"]
  1439. task_report = report_schema["properties"]["task_report"]
  1440. self.assertEqual("object", task_report["type"])
  1441. self.assertIn("validation", task_report["properties"])
  1442. self.assertIn("next_step_suggestion", task_report["properties"])
  1443. review_schema = schemas["review_task_result"]["function"]["parameters"]
  1444. self.assertEqual(
  1445. [
  1446. "ACCEPT_REFINE",
  1447. "REJECT_REFINE",
  1448. "REVISE_CHILD",
  1449. "DESCEND_AGAIN",
  1450. "REPLAN_CURRENT",
  1451. "ASCEND",
  1452. "FAIL",
  1453. ],
  1454. review_schema["properties"]["decision"]["enum"],
  1455. )
  1456. self.assertEqual(
  1457. "object",
  1458. review_schema["properties"]["approved_next_task"]["type"],
  1459. )
  1460. if __name__ == "__main__":
  1461. unittest.main()