test_task_protocol.py 65 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715
  1. import json
  2. import tempfile
  3. import unittest
  4. from unittest.mock import AsyncMock, patch
  5. from pydantic import ValidationError
  6. from cyber_agent.core.agent_mode import RECURSIVE_CAPABILITY_TOOLS_CONTEXT_KEY
  7. from cyber_agent.core.resource_budget import (
  8. RESOURCE_BUDGET_CONTEXT_KEY,
  9. ResourceBudget,
  10. ResourceBudgetController,
  11. )
  12. from cyber_agent.core.runner import AgentRunner, RunConfig
  13. from cyber_agent.core.run_snapshot import (
  14. RunConfigSnapshotV1,
  15. persist_run_config_snapshot,
  16. )
  17. from cyber_agent.core.context_policy import (
  18. add_context_snapshot,
  19. create_context_snapshot,
  20. persist_root_task_anchor,
  21. )
  22. from cyber_agent.core.task_protocol import (
  23. TaskBrief,
  24. TaskReport,
  25. TaskReview,
  26. ensure_task_protocol,
  27. new_task_protocol,
  28. replace_task_brief,
  29. )
  30. from cyber_agent.tools import get_tool_registry
  31. from cyber_agent.tools.builtin.knowledge import KnowledgeConfig
  32. from cyber_agent.tools.builtin.subagent import _record_pending_task_reports, agent
  33. from cyber_agent.tools.builtin.task_protocol import (
  34. review_task_result,
  35. submit_task_report,
  36. )
  37. from cyber_agent.trace.goal_models import Goal, GoalTree
  38. from cyber_agent.trace.goal_tool import goal_tool
  39. from cyber_agent.trace.models import Message, Trace
  40. from cyber_agent.trace.store import FileSystemTraceStore
  41. from cyber_agent.core.validation import (
  42. ScopeValidationResult,
  43. ValidationCheck,
  44. ValidationPolicy,
  45. ValidationResult,
  46. ValidationRun,
  47. ValidatorSettings,
  48. persist_validation_policy,
  49. )
  50. BRIEF = {
  51. "objective": "research one concrete question",
  52. "reason": "the parent needs one independently checked result",
  53. "completion_criteria": ["provide one checked conclusion"],
  54. "expected_outputs": ["one checked conclusion"],
  55. "constraints": ["do not guess"],
  56. }
  57. ROOT_ANCHOR = {
  58. "objective": "finish the root task",
  59. "completion_criteria": ["finish the root task"],
  60. "constraints": ["do not guess"],
  61. }
  62. def with_root_anchor(context):
  63. persist_root_task_anchor(context, ROOT_ANCHOR)
  64. return context
  65. def report_payload(*, outcome="satisfied"):
  66. return {
  67. "summary": "checked result",
  68. "outcome": outcome,
  69. "validation": {
  70. "hard_passed": outcome == "satisfied",
  71. "open_issues": [] if outcome == "satisfied" else ["still failing"],
  72. },
  73. "next_step_suggestion": {
  74. "direction": "NONE",
  75. "reason": "No additional task is needed",
  76. },
  77. "outputs": [{"kind": "note", "value": "result"}],
  78. "evidence": [{"kind": "test", "value": "passed"}],
  79. "remaining_issues": [] if outcome == "satisfied" else ["still failing"],
  80. }
  81. def validation_payload(child_trace_id, *, outcome="passed"):
  82. plan_hash = "a" * 64
  83. if outcome == "passed":
  84. issues = []
  85. retry_from = None
  86. elif outcome == "failed":
  87. issues = ["task needs revision"]
  88. retry_from = "task_definition"
  89. else:
  90. issues = ["validator could not evaluate the task"]
  91. retry_from = None
  92. scope_result = ScopeValidationResult(
  93. validator_trace_id=f"{child_trace_id}@validator-fixture",
  94. scope="task",
  95. outcome=outcome,
  96. checks=[ValidationCheck(
  97. check_id="task.fixture",
  98. status=("unknown" if outcome == "error" else outcome),
  99. issue=None if outcome == "passed" else issues[0],
  100. )],
  101. reason="fixture validation",
  102. retry_from=retry_from,
  103. plan_hash=plan_hash,
  104. )
  105. return ValidationResult(
  106. evaluated_trace_id=child_trace_id,
  107. outcome=outcome,
  108. scope_results=[scope_result],
  109. issues=issues,
  110. retry_from=retry_from,
  111. plan_hash=plan_hash,
  112. ).model_dump(mode="json")
  113. def passed_validator_response(kwargs):
  114. """按当前 ValidationPlan 返回完整的通过结果。"""
  115. messages = kwargs.get("messages") or []
  116. packet = json.loads(messages[1]["content"])
  117. scope = packet["validation_scope"]
  118. checks = [
  119. {
  120. "check_id": item["check_id"],
  121. "status": "passed",
  122. "evidence_refs": [],
  123. "issue": None,
  124. }
  125. for item in packet["validation_plan"]["checks"]
  126. if item["scope"] == scope
  127. ]
  128. return {
  129. "content": json.dumps({
  130. "outcome": "passed",
  131. "scope": scope,
  132. "checks": checks,
  133. "reason": "all planned checks passed",
  134. "retry_from": None,
  135. }),
  136. "tool_calls": [],
  137. "finish_reason": "stop",
  138. }
  139. async def create_recursive_root(store, trace_id, *, state=None, uid="user-1"):
  140. budget = ResourceBudget()
  141. context = {
  142. "agent_mode": "recursive",
  143. "agent_mode_revision": 2,
  144. "agent_depth": 0,
  145. "root_trace_id": trace_id,
  146. RESOURCE_BUDGET_CONTEXT_KEY: budget.to_dict(),
  147. "task_protocol": state or new_task_protocol(),
  148. }
  149. persist_root_task_anchor(context, ROOT_ANCHOR)
  150. persist_validation_policy(
  151. context,
  152. ValidationPolicy(),
  153. ValidatorSettings(),
  154. )
  155. await store.create_trace(Trace(
  156. trace_id=trace_id,
  157. mode="agent",
  158. task="root",
  159. uid=uid,
  160. model="fake-model",
  161. context=context,
  162. ))
  163. await ResourceBudgetController(store).initialize(
  164. trace_id,
  165. budget,
  166. initial_agents=1,
  167. )
  168. return await store.get_trace(trace_id)
  169. class FakeToolRegistry:
  170. def get_tool_names(self):
  171. return [
  172. "agent",
  173. "evaluate",
  174. "bash_command",
  175. "goal",
  176. "read_file",
  177. "submit_task_report",
  178. "review_task_result",
  179. "read_context_ref",
  180. ]
  181. class ReportingRunner:
  182. def __init__(self, store):
  183. self.store = store
  184. self.tools = FakeToolRegistry()
  185. self.debug = False
  186. self.stdin_check = None
  187. self._cancel_events = {}
  188. self.configs = []
  189. self.resource_budget = ResourceBudgetController(store)
  190. self.validation_count = 0
  191. async def _resource_budget_for_trace(self, trace_id):
  192. trace = await self.store.get_trace(trace_id)
  193. root_id = trace.context["root_trace_id"]
  194. root = trace if root_id == trace_id else await self.store.get_trace(root_id)
  195. return root_id, ResourceBudget.from_dict(
  196. root.context[RESOURCE_BUDGET_CONTEXT_KEY]
  197. )
  198. async def validate_recursive_trace(
  199. self,
  200. evaluated_trace_id,
  201. *,
  202. scope,
  203. deterministic_failure=None,
  204. **_kwargs,
  205. ):
  206. evaluated = await self.store.get_trace(evaluated_trace_id)
  207. child_state = ensure_task_protocol(evaluated.context)
  208. cached = child_state.get("task_report_validation")
  209. if isinstance(cached, dict) and cached.get("aggregate_result"):
  210. result = ValidationResult.model_validate(cached["aggregate_result"])
  211. return ValidationRun(
  212. result=result,
  213. trace_ids=[
  214. item.validator_trace_id for item in result.scope_results
  215. ],
  216. cached=True,
  217. )
  218. self.validation_count += 1
  219. validator_trace_id = (
  220. f"{evaluated_trace_id}@validator-fixture-{self.validation_count}"
  221. )
  222. if deterministic_failure:
  223. outcome = deterministic_failure.get("outcome", "error")
  224. issues = deterministic_failure.get("issues") or [
  225. deterministic_failure.get("reason", "validation failed")
  226. ]
  227. retry_from = deterministic_failure.get("retry_from")
  228. else:
  229. outcome = "passed"
  230. issues = []
  231. retry_from = None
  232. result = ValidationResult.model_validate(
  233. validation_payload(evaluated_trace_id, outcome=outcome)
  234. )
  235. child_state["task_report_validation"] = {
  236. "validation_plan": None,
  237. "plan_hash": result.plan_hash,
  238. "scope_results": [
  239. item.model_dump(mode="json") for item in result.scope_results
  240. ],
  241. "aggregate_result": result.model_dump(mode="json"),
  242. "validated_at_sequence": evaluated.head_sequence,
  243. }
  244. await self.store.update_trace(
  245. evaluated_trace_id,
  246. context=evaluated.context,
  247. )
  248. await self.store.create_trace(Trace(
  249. trace_id=validator_trace_id,
  250. mode="agent",
  251. task=f"validate {evaluated_trace_id}",
  252. parent_trace_id=evaluated_trace_id,
  253. uid=evaluated.uid,
  254. status="completed",
  255. tools=[],
  256. context=with_root_anchor({
  257. "created_by_tool": "validator",
  258. "root_trace_id": evaluated.context["root_trace_id"],
  259. "agent_depth": evaluated.context["agent_depth"],
  260. }),
  261. ))
  262. return ValidationRun(result=result, trace_ids=[validator_trace_id])
  263. async def run_result(self, messages, config, on_event=None):
  264. self.configs.append(config)
  265. submitted = await submit_task_report(
  266. task_report=report_payload(),
  267. context={
  268. "store": self.store,
  269. "trace_id": config.trace_id,
  270. "sequence": 3,
  271. },
  272. )
  273. if submitted["status"] != "completed":
  274. raise AssertionError(submitted)
  275. await self.store.update_trace(config.trace_id, status="completed")
  276. return {
  277. "status": "completed",
  278. "summary": "child submitted TaskReport",
  279. "saved_knowledge_ids": [],
  280. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  281. }
  282. class FailedAfterReportingRunner(ReportingRunner):
  283. async def run_result(self, messages, config, on_event=None):
  284. result = await super().run_result(messages, config, on_event)
  285. await self.store.update_trace(
  286. config.trace_id,
  287. status="failed",
  288. error_message="runner failed after report submission",
  289. )
  290. return {
  291. **result,
  292. "status": "failed",
  293. "error": "runner failed after report submission",
  294. }
  295. class ForgedIdentityRunner(ReportingRunner):
  296. async def run_result(self, messages, config, on_event=None):
  297. trace = await self.store.get_trace(config.trace_id)
  298. state = ensure_task_protocol(trace.context)
  299. state["task_report"] = TaskReport(
  300. child_trace_id="someone-elses-trace",
  301. **report_payload(),
  302. ).model_dump()
  303. await self.store.update_trace(
  304. config.trace_id,
  305. context=trace.context,
  306. status="completed",
  307. )
  308. return {
  309. "status": "completed",
  310. "summary": "forged report identity",
  311. "saved_knowledge_ids": [],
  312. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  313. }
  314. class RecursiveTaskProtocolTest(unittest.IsolatedAsyncioTestCase):
  315. async def asyncSetUp(self):
  316. self.temp_dir = tempfile.TemporaryDirectory()
  317. self.store = FileSystemTraceStore(self.temp_dir.name)
  318. self.runner = ReportingRunner(self.store)
  319. self.root_id = "recursive-root"
  320. await create_recursive_root(self.store, self.root_id)
  321. self.tree = GoalTree(
  322. mission="root",
  323. goals=[Goal(id="1", description="delegate", status="in_progress")],
  324. current_id="1",
  325. )
  326. await self.store.update_goal_tree(self.root_id, self.tree)
  327. async def asyncTearDown(self):
  328. self.temp_dir.cleanup()
  329. def context(self, trace_id=None, goal_id="1", tree=None):
  330. return {
  331. "store": self.store,
  332. "trace_id": trace_id or self.root_id,
  333. "goal_id": goal_id,
  334. "runner": self.runner,
  335. "goal_tree": tree or self.tree,
  336. "sequence": 10,
  337. RECURSIVE_CAPABILITY_TOOLS_CONTEXT_KEY: self.runner.tools.get_tool_names(),
  338. }
  339. async def children(self):
  340. return await self.store.list_traces(
  341. parent_trace_id=self.root_id,
  342. created_by_tool="agent",
  343. limit=20,
  344. )
  345. async def test_models_validate_and_round_trip(self):
  346. brief = TaskBrief.model_validate(BRIEF)
  347. self.assertEqual(
  348. {
  349. **BRIEF,
  350. "parent_findings": [],
  351. "context": {},
  352. "context_refs": [],
  353. "validation_scopes": [],
  354. },
  355. brief.model_dump(exclude_none=True),
  356. )
  357. with self.assertRaises(ValidationError):
  358. TaskBrief(objective="", completion_criteria=[])
  359. with self.assertRaises(ValidationError):
  360. TaskReview(
  361. parent_trace_id="p",
  362. child_trace_id="c",
  363. decision="ACCEPT_REFINE",
  364. reason="continue",
  365. )
  366. invalid_suggestion = report_payload()
  367. invalid_suggestion["next_step_suggestion"] = {
  368. "direction": "NONE",
  369. "reason": "No next task should be attached",
  370. "suggested_next_task": BRIEF,
  371. }
  372. with self.assertRaises(ValidationError):
  373. TaskReport(child_trace_id="child", **invalid_suggestion)
  374. root_submission = await submit_task_report(
  375. task_report=report_payload(),
  376. context={"store": self.store, "trace_id": self.root_id},
  377. )
  378. self.assertEqual("failed", root_submission["status"])
  379. self.assertIn("Root Agent", root_submission["error"])
  380. async def test_child_report_blocks_new_child_until_parent_review(self):
  381. result = await agent(task_brief=BRIEF, context=self.context())
  382. self.assertEqual("completed", result["status"])
  383. child_id = result["sub_trace_id"]
  384. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  385. parent = await self.store.get_trace(self.root_id)
  386. state = ensure_task_protocol(parent.context)
  387. self.assertEqual([child_id], list(state["pending_reviews"]))
  388. tree = await self.store.get_goal_tree(self.root_id)
  389. self.assertEqual("pending_review", tree.find("1").status)
  390. restarted_runner = AgentRunner(tool_registry=get_tool_registry())
  391. restored = await self.store.get_trace(self.root_id)
  392. restored_names = {
  393. schema["function"]["name"]
  394. for schema in restarted_runner._get_runtime_tool_schemas(
  395. RunConfig(), restored
  396. )
  397. }
  398. self.assertEqual(
  399. {"review_task_result", "read_context_ref"},
  400. restored_names,
  401. )
  402. blocked = await agent(task_brief=BRIEF, context=self.context())
  403. self.assertEqual("failed", blocked["status"])
  404. self.assertIn("Review pending", blocked["error"])
  405. async def test_child_inherits_brief_constraints_not_parent_runtime_context(self):
  406. parent = await self.store.get_trace(self.root_id)
  407. parent.context["parent_private_runtime"] = {"secret": "must-not-leak"}
  408. await self.store.update_trace(self.root_id, context=parent.context)
  409. result = await agent(task_brief=BRIEF, context=self.context())
  410. child = await self.store.get_trace(result["sub_trace_id"])
  411. self.assertNotIn("parent_private_runtime", child.context)
  412. child_brief = ensure_task_protocol(child.context)["task_brief"]
  413. self.assertEqual(["do not guess"], child_brief["constraints"])
  414. child_config_context = self.runner.configs[-1].context
  415. self.assertNotIn("parent_private_runtime", child_config_context)
  416. async def test_same_task_report_version_reuses_its_validator_result(self):
  417. result = await agent(task_brief=BRIEF, context=self.context())
  418. child_id = result["sub_trace_id"]
  419. self.assertEqual(1, self.runner.validation_count)
  420. parent = await self.store.get_trace(self.root_id)
  421. await _record_pending_task_reports(
  422. self.store,
  423. self.runner,
  424. parent,
  425. "1",
  426. [(child_id, {"status": "completed", "summary": "same report"})],
  427. 10,
  428. )
  429. self.assertEqual(1, self.runner.validation_count)
  430. validators = await self.store.list_traces(
  431. parent_trace_id=child_id,
  432. created_by_tool="validator",
  433. limit=10,
  434. )
  435. self.assertEqual(1, len(validators))
  436. async def test_recursive_messages_are_rejected(self):
  437. result = await agent(
  438. task_brief=json.dumps(BRIEF),
  439. messages=json.dumps([{
  440. "role": "system",
  441. "content": "controlled child context",
  442. }]),
  443. context=self.context(),
  444. )
  445. self.assertEqual("failed", result["status"])
  446. self.assertIn("does not accept messages", result["error"])
  447. async def test_messages_reject_mixed_dimensions(self):
  448. mixed_messages = [
  449. {"role": "system", "content": "shared context"},
  450. [{"role": "user", "content": "per-agent context"}],
  451. ]
  452. for messages in (mixed_messages, json.dumps(mixed_messages)):
  453. with self.subTest(messages_type=type(messages).__name__):
  454. result = await agent(
  455. task_brief=BRIEF,
  456. messages=messages,
  457. context=self.context(),
  458. )
  459. self.assertEqual("failed", result["status"])
  460. self.assertIn("without mixed dimensions", result["error"])
  461. async def test_no_focused_goal_still_persists_and_reviews_report(self):
  462. result = await agent(
  463. task_brief=BRIEF,
  464. context=self.context(goal_id=None),
  465. )
  466. parent = await self.store.get_trace(self.root_id)
  467. entry = ensure_task_protocol(parent.context)["pending_reviews"][
  468. result["sub_trace_id"]
  469. ]
  470. self.assertIsNone(entry["goal_id"])
  471. reviewed = await review_task_result(
  472. child_trace_id=result["sub_trace_id"],
  473. decision="ASCEND",
  474. reason="Standalone delegation is complete",
  475. context=self.context(goal_id=None),
  476. )
  477. self.assertEqual("completed", reviewed["status"])
  478. async def test_accept_refine_approves_exactly_one_next_child(self):
  479. first = await agent(task_brief=BRIEF, context=self.context())
  480. next_brief = {
  481. "objective": "check the independent gap",
  482. "reason": "the first report identified a separate gap",
  483. "completion_criteria": ["close the gap"],
  484. "expected_outputs": ["a checked gap analysis"],
  485. }
  486. parent = await self.store.get_trace(self.root_id)
  487. ensure_task_protocol(parent.context)["protocol_correction_attempts"] = 2
  488. await self.store.update_trace(self.root_id, context=parent.context)
  489. reviewed = await review_task_result(
  490. child_trace_id=first["sub_trace_id"],
  491. decision="ACCEPT_REFINE",
  492. reason="The gap belongs to this parent",
  493. approved_next_task=json.dumps(next_brief),
  494. context=self.context(),
  495. )
  496. self.assertEqual("completed", reviewed["status"])
  497. self.assertIsNotNone(reviewed["context_ref"])
  498. self.assertIn(
  499. reviewed["context_ref"],
  500. reviewed["next_action_task_brief"]["context_refs"],
  501. )
  502. parent = await self.store.get_trace(self.root_id)
  503. self.assertEqual(
  504. 0,
  505. ensure_task_protocol(parent.context)["protocol_correction_attempts"],
  506. )
  507. schemas = AgentRunner(
  508. tool_registry=get_tool_registry()
  509. )._get_runtime_tool_schemas(RunConfig(), parent)
  510. self.assertEqual(
  511. {"agent", "read_context_ref"},
  512. {schema["function"]["name"] for schema in schemas},
  513. )
  514. wrong = await agent(task_brief=BRIEF, context=self.context())
  515. self.assertEqual("failed", wrong["status"])
  516. self.assertIn("approved next task", wrong["error"])
  517. second = await agent(
  518. task_brief=reviewed["next_action_task_brief"],
  519. context=self.context(),
  520. )
  521. self.assertEqual("completed", second["status"])
  522. self.assertEqual(2, len(await self.children()))
  523. second_child = await self.store.get_trace(second["sub_trace_id"])
  524. self.assertIn(
  525. reviewed["context_ref"]["ref_id"],
  526. second_child.context["context_access"]["snapshots"],
  527. )
  528. parent = await self.store.get_trace(self.root_id)
  529. self.assertEqual([], ensure_task_protocol(parent.context)["next_actions"])
  530. async def test_review_rejects_an_unusable_context_ref_without_consuming_report(self):
  531. first = await agent(task_brief=BRIEF, context=self.context())
  532. invalid_next = {
  533. "objective": "use an unauthorized context ref",
  534. "reason": "exercise approval validation",
  535. "completion_criteria": ["must not be approved"],
  536. "expected_outputs": ["nothing"],
  537. "context_refs": [{
  538. "ref_id": "ctx_000000000000000000000000",
  539. "version": "0" * 64,
  540. }],
  541. }
  542. reviewed = await review_task_result(
  543. child_trace_id=first["sub_trace_id"],
  544. decision="ACCEPT_REFINE",
  545. reason="try an invalid transfer",
  546. approved_next_task=invalid_next,
  547. context=self.context(),
  548. )
  549. self.assertEqual("failed", reviewed["status"])
  550. self.assertIn("not authorized", reviewed["error"])
  551. parent = await self.store.get_trace(self.root_id)
  552. state = ensure_task_protocol(parent.context)
  553. self.assertIn(first["sub_trace_id"], state["pending_reviews"])
  554. self.assertEqual([], state["reviews"])
  555. async def test_oversized_review_snapshot_warns_without_blocking_review(self):
  556. first = await agent(task_brief=BRIEF, context=self.context())
  557. parent = await self.store.get_trace(self.root_id)
  558. state = ensure_task_protocol(parent.context)
  559. state["pending_reviews"][first["sub_trace_id"]]["task_report"]["outputs"] = [
  560. {"payload": "x" * 16_000}
  561. ]
  562. await self.store.update_trace(self.root_id, context=parent.context)
  563. reviewed = await review_task_result(
  564. child_trace_id=first["sub_trace_id"],
  565. decision="ASCEND",
  566. reason="The task passed but its transferable snapshot is too large",
  567. context=self.context(),
  568. )
  569. self.assertEqual("completed", reviewed["status"])
  570. self.assertIsNone(reviewed["context_ref"])
  571. self.assertIn("16000", reviewed["context_ref_warning"])
  572. parent = await self.store.get_trace(self.root_id)
  573. self.assertEqual({}, ensure_task_protocol(parent.context)["pending_reviews"])
  574. async def test_batch_context_refs_are_validated_before_any_trace_is_created(self):
  575. invalid_ref_brief = {
  576. **BRIEF,
  577. "objective": "second child with an unauthorized reference",
  578. "context_refs": [{
  579. "ref_id": "ctx_000000000000000000000000",
  580. "version": "0" * 64,
  581. }],
  582. }
  583. result = await agent(
  584. task_brief=[BRIEF, invalid_ref_brief],
  585. context=self.context(),
  586. )
  587. self.assertEqual("failed", result["status"])
  588. self.assertIn("not authorized", result["error"])
  589. self.assertEqual([], await self.children())
  590. async def test_batch_reports_are_reviewed_one_by_one(self):
  591. briefs = [
  592. BRIEF,
  593. {
  594. "objective": "research second question",
  595. "reason": "the batch needs a second independent result",
  596. "completion_criteria": ["provide second conclusion"],
  597. "expected_outputs": ["a second conclusion"],
  598. },
  599. ]
  600. result = await agent(task_brief=briefs, context=self.context())
  601. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  602. self.assertEqual(2, len(result["task_reports"]))
  603. premature = await review_task_result(
  604. child_trace_id=child_ids[0],
  605. decision="ASCEND",
  606. reason="Cannot finish while another report is pending",
  607. context=self.context(),
  608. )
  609. self.assertEqual("failed", premature["status"])
  610. self.assertIn("final pending report", premature["error"])
  611. first_review = await review_task_result(
  612. child_trace_id=child_ids[0],
  613. decision="REJECT_REFINE",
  614. reason="No extra task needed for this report",
  615. context=self.context(),
  616. )
  617. self.assertEqual(1, first_review["pending_review_count"])
  618. tree = await self.store.get_goal_tree(self.root_id)
  619. self.assertEqual("pending_review", tree.find("1").status)
  620. final_review = await review_task_result(
  621. child_trace_id=child_ids[1],
  622. decision="ASCEND",
  623. reason="All completion criteria are satisfied",
  624. context=self.context(),
  625. )
  626. self.assertEqual(0, final_review["pending_review_count"])
  627. tree = await self.store.get_goal_tree(self.root_id)
  628. self.assertEqual("completed", tree.find("1").status)
  629. duplicate = await review_task_result(
  630. child_trace_id=child_ids[1],
  631. decision="ASCEND",
  632. reason="Duplicate review must fail",
  633. context=self.context(),
  634. )
  635. self.assertEqual("failed", duplicate["status"])
  636. self.assertIn("no pending TaskReport", duplicate["error"])
  637. async def test_terminal_review_waits_for_queued_next_action(self):
  638. result = await agent(
  639. task_brief=[
  640. BRIEF,
  641. {
  642. "objective": "second batch child",
  643. "reason": "exercise queued batch behavior",
  644. "completion_criteria": ["finish the second child"],
  645. "expected_outputs": ["the second child result"],
  646. },
  647. ],
  648. context=self.context(),
  649. )
  650. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  651. next_brief = {
  652. "objective": "approved follow-up",
  653. "reason": "the parent approved one follow-up",
  654. "completion_criteria": ["finish the approved follow-up"],
  655. "expected_outputs": ["the approved follow-up result"],
  656. }
  657. accepted = await review_task_result(
  658. child_trace_id=child_ids[0],
  659. decision="ACCEPT_REFINE",
  660. reason="A separate follow-up is required",
  661. approved_next_task=next_brief,
  662. context=self.context(),
  663. )
  664. self.assertEqual("completed", accepted["status"])
  665. blocked = await review_task_result(
  666. child_trace_id=child_ids[1],
  667. decision="ASCEND",
  668. reason="Cannot ascend before the accepted follow-up",
  669. context=self.context(),
  670. )
  671. self.assertEqual("failed", blocked["status"])
  672. self.assertIn("approved next action", blocked["error"])
  673. tree = await self.store.get_goal_tree(self.root_id)
  674. self.assertEqual("pending_review", tree.find("1").status)
  675. final_non_terminal = await review_task_result(
  676. child_trace_id=child_ids[1],
  677. decision="REJECT_REFINE",
  678. reason="This report adds no other follow-up",
  679. context=self.context(),
  680. )
  681. self.assertEqual("completed", final_non_terminal["status"])
  682. tree = await self.store.get_goal_tree(self.root_id)
  683. self.assertEqual("in_progress", tree.find("1").status)
  684. async def test_runner_failure_overrides_stale_satisfied_report(self):
  685. self.runner = FailedAfterReportingRunner(self.store)
  686. result = await agent(task_brief=BRIEF, context=self.context())
  687. self.assertEqual("failed", result["status"])
  688. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  689. child = await self.store.get_trace(result["sub_trace_id"])
  690. child_state = ensure_task_protocol(child.context)
  691. self.assertEqual(1, len(child_state["report_history"]))
  692. self.assertEqual("satisfied", child_state["report_history"][0]["outcome"])
  693. invalid_ascend = await review_task_result(
  694. child_trace_id=result["sub_trace_id"],
  695. decision="ASCEND",
  696. reason="A failed execution cannot ascend",
  697. context=self.context(),
  698. )
  699. self.assertEqual("failed", invalid_ascend["status"])
  700. self.assertIn("only allow", invalid_ascend["error"])
  701. async def test_submit_and_review_reject_child_anchor_that_differs_from_root(self):
  702. child_context = with_root_anchor({
  703. "agent_mode": "recursive",
  704. "agent_mode_revision": 2,
  705. "agent_depth": 1,
  706. "root_trace_id": self.root_id,
  707. "created_by_tool": "agent",
  708. "task_protocol": new_task_protocol(BRIEF),
  709. })
  710. persist_root_task_anchor(child_context, {
  711. **ROOT_ANCHOR,
  712. "objective": "self-consistent but unauthorized root objective",
  713. })
  714. child_id = "mismatched-anchor-child"
  715. await self.store.create_trace(Trace(
  716. trace_id=child_id,
  717. mode="agent",
  718. task="child",
  719. parent_trace_id=self.root_id,
  720. parent_goal_id="1",
  721. uid="user-1",
  722. model="fake-model",
  723. context=child_context,
  724. ))
  725. submitted = await submit_task_report(
  726. task_report=report_payload(),
  727. context={"store": self.store, "trace_id": child_id},
  728. )
  729. self.assertEqual("failed", submitted["status"])
  730. self.assertIn("does not match", submitted["error"])
  731. parent = await self.store.get_trace(self.root_id)
  732. parent_state = ensure_task_protocol(parent.context)
  733. parent_state["pending_reviews"][child_id] = {
  734. "goal_id": "1",
  735. "task_report": TaskReport(
  736. child_trace_id=child_id,
  737. **report_payload(),
  738. ).model_dump(),
  739. "validation_result": validation_payload(child_id),
  740. "received_at_sequence": 1,
  741. }
  742. await self.store.update_trace(self.root_id, context=parent.context)
  743. reviewed = await review_task_result(
  744. child_trace_id=child_id,
  745. decision="ASCEND",
  746. reason="must not accept a different root anchor",
  747. context=self.context(),
  748. )
  749. self.assertEqual("failed", reviewed["status"])
  750. self.assertIn("does not match", reviewed["error"])
  751. parent = await self.store.get_trace(self.root_id)
  752. state = ensure_task_protocol(parent.context)
  753. self.assertIn(child_id, state["pending_reviews"])
  754. self.assertEqual([], state["reviews"])
  755. async def test_review_rejects_wrong_owner_even_if_report_is_injected(self):
  756. first = await agent(task_brief=BRIEF, context=self.context())
  757. child_id = first["sub_trace_id"]
  758. await self.store.update_trace(child_id, uid="another-user")
  759. rejected = await review_task_result(
  760. child_trace_id=child_id,
  761. decision="ASCEND",
  762. reason="Owner mismatch must be rejected",
  763. context=self.context(),
  764. )
  765. self.assertEqual("failed", rejected["status"])
  766. self.assertIn("same owner", rejected["error"])
  767. async def test_report_identity_must_match_direct_child(self):
  768. self.runner = ForgedIdentityRunner(self.store)
  769. result = await agent(task_brief=BRIEF, context=self.context())
  770. child_id = result["sub_trace_id"]
  771. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  772. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  773. parent = await self.store.get_trace(self.root_id)
  774. pending_report = ensure_task_protocol(parent.context)["pending_reviews"][
  775. child_id
  776. ]["task_report"]
  777. pending_report["child_trace_id"] = "corrupted-after-recording"
  778. await self.store.update_trace(self.root_id, context=parent.context)
  779. rejected = await review_task_result(
  780. child_trace_id=child_id,
  781. decision="FAIL",
  782. reason="Corrupted identity cannot be reviewed",
  783. context=self.context(),
  784. )
  785. self.assertEqual("failed", rejected["status"])
  786. self.assertIn("does not match", rejected["error"])
  787. async def test_remote_agent_is_rejected_by_recursive_revision_2(self):
  788. remote_call = AsyncMock(return_value={
  789. "status": "completed",
  790. "summary": "remote result",
  791. })
  792. with patch(
  793. "cyber_agent.tools.builtin.subagent._run_remote_agent",
  794. new=remote_call,
  795. ):
  796. result = await agent(
  797. task="remote free text",
  798. agent_type="remote_librarian",
  799. context=self.context(),
  800. )
  801. self.assertEqual("failed", result["status"])
  802. self.assertIn("does not support remote agents", result["error"])
  803. remote_call.assert_not_awaited()
  804. async def test_revise_child_reuses_trace_and_preserves_report_history(self):
  805. first = await agent(task_brief=BRIEF, context=self.context())
  806. child_id = first["sub_trace_id"]
  807. reviewed = await review_task_result(
  808. child_trace_id=child_id,
  809. decision="REVISE_CHILD",
  810. reason="Please strengthen the evidence",
  811. context=self.context(),
  812. )
  813. self.assertEqual("completed", reviewed["status"])
  814. cleared_child = await self.store.get_trace(child_id)
  815. self.assertEqual({}, cleared_child.context["context_access"]["snapshots"])
  816. wrong = await agent(
  817. task_brief={
  818. "objective": "silently replace the reviewed task",
  819. "reason": "attempt a mismatched revision",
  820. "completion_criteria": ["this must be rejected"],
  821. "expected_outputs": ["a rejected replacement"],
  822. },
  823. continue_from=child_id,
  824. context=self.context(),
  825. )
  826. self.assertEqual("failed", wrong["status"])
  827. self.assertIn("approved revision", wrong["error"])
  828. revised_brief = reviewed["next_action_task_brief"]
  829. revised = await agent(
  830. task_brief=revised_brief,
  831. continue_from=child_id,
  832. context=self.context(),
  833. )
  834. self.assertEqual(child_id, revised["sub_trace_id"])
  835. self.assertTrue(revised["continue_from"])
  836. self.assertEqual(1, len(await self.children()))
  837. child = await self.store.get_trace(child_id)
  838. self.assertEqual(1, len(ensure_task_protocol(child.context)["report_history"]))
  839. self.assertEqual(
  840. TaskBrief.model_validate(revised_brief).model_dump(),
  841. ensure_task_protocol(child.context)["task_brief"],
  842. )
  843. self.assertEqual(2, ensure_task_protocol(child.context)["task_brief_version"])
  844. self.assertEqual(
  845. 1,
  846. len(ensure_task_protocol(child.context)["task_brief_history"]),
  847. )
  848. async def test_review_cannot_reserve_more_children_than_quota(self):
  849. briefs = [
  850. {
  851. "objective": f"batch child {index}",
  852. "reason": "fill the local child quota",
  853. "completion_criteria": [f"finish child {index}"],
  854. "expected_outputs": [f"result for child {index}"],
  855. }
  856. for index in range(5)
  857. ]
  858. result = await agent(task_brief=briefs, context=self.context())
  859. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  860. next_brief = {
  861. "objective": "one allowed reserved child",
  862. "reason": "use the final available child slot",
  863. "completion_criteria": ["use the final available slot"],
  864. "expected_outputs": ["one final child result"],
  865. }
  866. first = await review_task_result(
  867. child_trace_id=child_ids[0],
  868. decision="ACCEPT_REFINE",
  869. reason="Reserve the sixth and final child slot",
  870. approved_next_task=next_brief,
  871. context=self.context(),
  872. )
  873. self.assertEqual("completed", first["status"])
  874. overflow = await review_task_result(
  875. child_trace_id=child_ids[1],
  876. decision="DESCEND_AGAIN",
  877. reason="A seventh child must not be approved",
  878. approved_next_task={
  879. "objective": "overflow child",
  880. "reason": "exercise quota rejection",
  881. "completion_criteria": ["must never be scheduled"],
  882. "expected_outputs": ["no output because creation is rejected"],
  883. },
  884. context=self.context(),
  885. )
  886. self.assertEqual("failed", overflow["status"])
  887. self.assertIn("reserved=1", overflow["error"])
  888. parent = await self.store.get_trace(self.root_id)
  889. state = ensure_task_protocol(parent.context)
  890. self.assertEqual(1, len(state["next_actions"]))
  891. self.assertIn(child_ids[1], state["pending_reviews"])
  892. async def test_recursive_goal_completion_does_not_cascade_or_select_sibling(self):
  893. tree = GoalTree(mission="recursive")
  894. parent = tree.add_goals(["parent"])[0]
  895. first, second = tree.add_goals(["first", "second"], parent_id=parent.id)
  896. tree.focus(parent.id)
  897. tree.focus(first.id)
  898. tree.complete(
  899. first.id,
  900. "done",
  901. auto_advance=False,
  902. cascade_parent=False,
  903. )
  904. self.assertEqual(parent.id, tree.current_id)
  905. self.assertEqual("pending", second.status)
  906. self.assertNotEqual("completed", parent.status)
  907. async def test_rewind_before_review_restores_report_and_review_gate(self):
  908. delegated = await agent(task_brief=BRIEF, context=self.context())
  909. child_id = delegated["sub_trace_id"]
  910. reviewed = await review_task_result(
  911. child_trace_id=child_id,
  912. decision="ASCEND",
  913. reason="Initially accepted before rewind",
  914. context={**self.context(), "sequence": 20},
  915. )
  916. self.assertEqual("completed", reviewed["status"])
  917. for sequence in (1, 15):
  918. await self.store.add_message(Message.create(
  919. trace_id=self.root_id,
  920. role="user",
  921. sequence=sequence,
  922. content=f"message-{sequence}",
  923. ))
  924. tree = await self.store.get_goal_tree(self.root_id)
  925. runner = AgentRunner(
  926. trace_store=self.store,
  927. tool_registry=get_tool_registry(),
  928. )
  929. await runner._rewind(self.root_id, 15, tree)
  930. parent = await self.store.get_trace(self.root_id)
  931. state = ensure_task_protocol(parent.context)
  932. self.assertIn(child_id, state["pending_reviews"])
  933. self.assertEqual([], state["reviews"])
  934. restored_tree = await self.store.get_goal_tree(self.root_id)
  935. self.assertEqual("pending_review", restored_tree.find("1").status)
  936. schemas = runner._get_runtime_tool_schemas(RunConfig(), parent)
  937. self.assertEqual(
  938. {"review_task_result", "read_context_ref"},
  939. {schema["function"]["name"] for schema in schemas},
  940. )
  941. async def test_grandchild_report_is_reviewed_only_by_direct_child(self):
  942. child_id = "manual-child"
  943. child_context = with_root_anchor({
  944. "agent_mode": "recursive",
  945. "agent_mode_revision": 2,
  946. "agent_depth": 1,
  947. "root_trace_id": self.root_id,
  948. "created_by_tool": "agent",
  949. "task_protocol": new_task_protocol(BRIEF),
  950. })
  951. await self.store.create_trace(Trace(
  952. trace_id=child_id,
  953. mode="agent",
  954. task="child",
  955. parent_trace_id=self.root_id,
  956. uid="user-1",
  957. model="fake-model",
  958. context=child_context,
  959. ))
  960. child_tree = GoalTree(
  961. mission="child",
  962. goals=[Goal(id="1", description="delegate grandchild", status="in_progress")],
  963. current_id="1",
  964. )
  965. await self.store.update_goal_tree(child_id, child_tree)
  966. grandchild = await agent(
  967. task_brief=BRIEF,
  968. context=self.context(trace_id=child_id, tree=child_tree),
  969. )
  970. child = await self.store.get_trace(child_id)
  971. child_state = ensure_task_protocol(child.context)
  972. self.assertIn(grandchild["sub_trace_id"], child_state["pending_reviews"])
  973. root = await self.store.get_trace(self.root_id)
  974. self.assertEqual({}, ensure_task_protocol(root.context)["pending_reviews"])
  975. blocked = await agent(
  976. task_brief=BRIEF,
  977. context=self.context(trace_id=child_id, tree=child_tree),
  978. )
  979. self.assertEqual("failed", blocked["status"])
  980. reviewed = await review_task_result(
  981. child_trace_id=grandchild["sub_trace_id"],
  982. decision="ASCEND",
  983. reason="Grandchild satisfied the child criteria",
  984. context=self.context(trace_id=child_id, tree=child_tree),
  985. )
  986. self.assertEqual("completed", reviewed["status"])
  987. submitted = await submit_task_report(
  988. task_report=json.dumps(report_payload()),
  989. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  990. )
  991. self.assertEqual("completed", submitted["status"])
  992. async def test_child_must_execute_approved_action_before_own_report(self):
  993. child_id = "child-with-approved-action"
  994. child_context = with_root_anchor({
  995. "agent_mode": "recursive",
  996. "agent_mode_revision": 2,
  997. "agent_depth": 1,
  998. "root_trace_id": self.root_id,
  999. "created_by_tool": "agent",
  1000. "task_protocol": new_task_protocol(BRIEF),
  1001. })
  1002. await self.store.create_trace(Trace(
  1003. trace_id=child_id,
  1004. mode="agent",
  1005. task="child",
  1006. parent_trace_id=self.root_id,
  1007. uid="user-1",
  1008. model="fake-model",
  1009. context=child_context,
  1010. ))
  1011. child_tree = GoalTree(
  1012. mission="child",
  1013. goals=[Goal(id="1", description="delegate", status="in_progress")],
  1014. current_id="1",
  1015. )
  1016. await self.store.update_goal_tree(child_id, child_tree)
  1017. child_tool_context = self.context(trace_id=child_id, tree=child_tree)
  1018. first = await agent(task_brief=BRIEF, context=child_tool_context)
  1019. next_brief = {
  1020. "objective": "check a second grandchild task",
  1021. "reason": "the child approved another direct task",
  1022. "completion_criteria": ["provide a second checked conclusion"],
  1023. "expected_outputs": ["a second checked conclusion"],
  1024. }
  1025. reviewed = await review_task_result(
  1026. child_trace_id=first["sub_trace_id"],
  1027. decision="ACCEPT_REFINE",
  1028. reason="A separate gap remains",
  1029. approved_next_task=next_brief,
  1030. context=child_tool_context,
  1031. )
  1032. self.assertEqual("completed", reviewed["status"])
  1033. blocked_report = await submit_task_report(
  1034. task_report=report_payload(),
  1035. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  1036. )
  1037. self.assertEqual("failed", blocked_report["status"])
  1038. self.assertIn("approved next action", blocked_report["error"])
  1039. second = await agent(
  1040. task_brief=reviewed["next_action_task_brief"],
  1041. context=child_tool_context,
  1042. )
  1043. self.assertEqual("completed", second["status"])
  1044. child = await self.store.get_trace(child_id)
  1045. state = ensure_task_protocol(child.context)
  1046. self.assertEqual([], state["next_actions"])
  1047. self.assertIn(second["sub_trace_id"], state["pending_reviews"])
  1048. async def test_goal_tool_cannot_bypass_queued_next_action(self):
  1049. root = await self.store.get_trace(self.root_id)
  1050. state = ensure_task_protocol(root.context)
  1051. state["next_actions"] = [{
  1052. "decision": "ACCEPT_REFINE",
  1053. "child_trace_id": "reviewed-child",
  1054. "goal_id": "1",
  1055. "task_brief": BRIEF,
  1056. "created_at_sequence": 5,
  1057. }]
  1058. await self.store.update_trace(self.root_id, context=root.context)
  1059. result = await goal_tool(
  1060. tree=self.tree,
  1061. store=self.store,
  1062. trace_id=self.root_id,
  1063. done="attempt to bypass the approved action",
  1064. )
  1065. self.assertIn("已批准的下一步待执行", result)
  1066. persisted_tree = await self.store.get_goal_tree(self.root_id)
  1067. self.assertEqual("in_progress", persisted_tree.find("1").status)
  1068. async def test_failed_report_allows_only_revise_or_fail(self):
  1069. child_id = "failed-child"
  1070. failed_report = TaskReport(
  1071. child_trace_id=child_id,
  1072. **report_payload(outcome="failed"),
  1073. )
  1074. child_context = with_root_anchor({
  1075. "agent_mode": "recursive",
  1076. "agent_mode_revision": 2,
  1077. "agent_depth": 1,
  1078. "root_trace_id": self.root_id,
  1079. "created_by_tool": "agent",
  1080. "task_protocol": new_task_protocol(BRIEF),
  1081. })
  1082. child_context["task_protocol"]["task_report"] = failed_report.model_dump()
  1083. failed_validation = validation_payload(child_id, outcome="failed")
  1084. child_context["task_protocol"]["task_report_validation"] = {
  1085. "validation_plan": None,
  1086. "plan_hash": failed_validation["plan_hash"],
  1087. "scope_results": failed_validation["scope_results"],
  1088. "aggregate_result": failed_validation,
  1089. "validated_at_sequence": 2,
  1090. }
  1091. await self.store.create_trace(Trace(
  1092. trace_id=child_id,
  1093. mode="agent",
  1094. task="failed",
  1095. parent_trace_id=self.root_id,
  1096. parent_goal_id="1",
  1097. uid="user-1",
  1098. context=child_context,
  1099. ))
  1100. root = await self.store.get_trace(self.root_id)
  1101. state = ensure_task_protocol(root.context)
  1102. state["pending_reviews"][child_id] = {
  1103. "goal_id": "1",
  1104. "task_report": failed_report.model_dump(),
  1105. "validation_result": failed_validation,
  1106. "received_at_sequence": 2,
  1107. }
  1108. await self.store.update_trace(self.root_id, context=root.context)
  1109. rejected = await review_task_result(
  1110. child_trace_id=child_id,
  1111. decision="ASCEND",
  1112. reason="incorrect",
  1113. context=self.context(),
  1114. )
  1115. self.assertEqual("failed", rejected["status"])
  1116. self.assertIn("only allow", rejected["error"])
  1117. class RunnerCompletionGateTest(unittest.IsolatedAsyncioTestCase):
  1118. async def asyncSetUp(self):
  1119. self.temp_dir = tempfile.TemporaryDirectory()
  1120. self.store = FileSystemTraceStore(self.temp_dir.name)
  1121. self.trace_id = "gate-child"
  1122. await create_recursive_root(self.store, "root")
  1123. controller = ResourceBudgetController(self.store)
  1124. budget = ResourceBudget()
  1125. await controller.reserve_agents("root", budget, 1)
  1126. await self.store.create_trace(Trace(
  1127. trace_id=self.trace_id,
  1128. mode="agent",
  1129. task="must report",
  1130. parent_trace_id="root",
  1131. uid="user-1",
  1132. context=with_root_anchor({
  1133. "agent_mode": "recursive",
  1134. "agent_mode_revision": 2,
  1135. "agent_depth": 1,
  1136. "root_trace_id": "root",
  1137. "created_by_tool": "agent",
  1138. "task_protocol": new_task_protocol(BRIEF),
  1139. }),
  1140. ))
  1141. await self.store.update_goal_tree(self.trace_id, GoalTree(mission="must report"))
  1142. async def asyncTearDown(self):
  1143. self.temp_dir.cleanup()
  1144. async def bind_snapshot(self, trace_id, config):
  1145. """Make manually-created Recursive fixtures equivalent to new traces."""
  1146. trace = await self.store.get_trace(trace_id)
  1147. config.model = trace.model or config.model
  1148. config.uid = trace.uid
  1149. config.agent_type = trace.agent_type or "default"
  1150. persist_run_config_snapshot(
  1151. trace.context,
  1152. RunConfigSnapshotV1.from_run_config(
  1153. config,
  1154. memory_identity=None,
  1155. ),
  1156. )
  1157. await self.store.update_trace(trace_id, context=trace.context)
  1158. return config
  1159. async def test_missing_report_becomes_protocol_error_after_two_corrections(self):
  1160. calls = 0
  1161. async def llm_call(**kwargs):
  1162. nonlocal calls
  1163. calls += 1
  1164. return {
  1165. "content": "I am done",
  1166. "tool_calls": [],
  1167. "finish_reason": "stop",
  1168. }
  1169. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1170. config = await self.bind_snapshot(
  1171. self.trace_id,
  1172. RunConfig(
  1173. trace_id=self.trace_id,
  1174. max_iterations=8,
  1175. knowledge=KnowledgeConfig(
  1176. enable_extraction=False,
  1177. enable_completion_extraction=False,
  1178. enable_injection=False,
  1179. ),
  1180. ),
  1181. )
  1182. result = await runner.run_result(messages=[], config=config)
  1183. self.assertEqual("failed", result["status"])
  1184. trace = await self.store.get_trace(self.trace_id)
  1185. report = ensure_task_protocol(trace.context)["task_report"]
  1186. self.assertEqual("protocol_error", report["outcome"])
  1187. self.assertGreaterEqual(calls, 3)
  1188. async def test_valid_report_allows_child_to_finish(self):
  1189. calls = 0
  1190. async def llm_call(**kwargs):
  1191. nonlocal calls
  1192. calls += 1
  1193. messages = kwargs.get("messages") or []
  1194. if messages and "recursive_validation_protocol" in messages[0]["content"]:
  1195. return passed_validator_response(kwargs)
  1196. if calls == 1:
  1197. return {
  1198. "content": "",
  1199. "tool_calls": [{
  1200. "id": "submit-1",
  1201. "type": "function",
  1202. "function": {
  1203. "name": "submit_task_report",
  1204. "arguments": json.dumps({
  1205. "task_report": report_payload(),
  1206. }),
  1207. },
  1208. }],
  1209. "finish_reason": "tool_calls",
  1210. }
  1211. return {
  1212. "content": "Structured report submitted.",
  1213. "tool_calls": [],
  1214. "finish_reason": "stop",
  1215. }
  1216. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1217. config = await self.bind_snapshot(
  1218. self.trace_id,
  1219. RunConfig(
  1220. trace_id=self.trace_id,
  1221. max_iterations=4,
  1222. knowledge=KnowledgeConfig(
  1223. enable_extraction=False,
  1224. enable_completion_extraction=False,
  1225. enable_injection=False,
  1226. ),
  1227. ),
  1228. )
  1229. result = await runner.run_result(messages=[], config=config)
  1230. self.assertEqual("completed", result["status"])
  1231. trace = await self.store.get_trace(self.trace_id)
  1232. self.assertEqual(
  1233. "satisfied",
  1234. ensure_task_protocol(trace.context)["task_report"]["outcome"],
  1235. )
  1236. async def test_lifecycle_tool_batch_is_rejected_before_execution(self):
  1237. root_id = "batch-gate-root"
  1238. await create_recursive_root(self.store, root_id)
  1239. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  1240. calls = 0
  1241. async def llm_call(**kwargs):
  1242. nonlocal calls
  1243. calls += 1
  1244. messages = kwargs.get("messages") or []
  1245. if messages and "recursive_validation_protocol" in messages[0]["content"]:
  1246. return passed_validator_response(kwargs)
  1247. if calls == 1:
  1248. return {
  1249. "content": "",
  1250. "tool_calls": [
  1251. {
  1252. "id": "agent-1",
  1253. "type": "function",
  1254. "function": {
  1255. "name": "agent",
  1256. "arguments": json.dumps({
  1257. "task_brief": BRIEF,
  1258. }),
  1259. },
  1260. },
  1261. {
  1262. "id": "context-1",
  1263. "type": "function",
  1264. "function": {
  1265. "name": "get_current_context",
  1266. "arguments": "{}",
  1267. },
  1268. },
  1269. ],
  1270. "finish_reason": "tool_calls",
  1271. }
  1272. return {
  1273. "content": "Corrected the invalid batch.",
  1274. "tool_calls": [],
  1275. "finish_reason": "stop",
  1276. }
  1277. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1278. config = await self.bind_snapshot(
  1279. root_id,
  1280. RunConfig(
  1281. trace_id=root_id,
  1282. max_iterations=3,
  1283. knowledge=KnowledgeConfig(
  1284. enable_extraction=False,
  1285. enable_completion_extraction=False,
  1286. enable_injection=False,
  1287. ),
  1288. ),
  1289. )
  1290. result = await runner.run_result(messages=[], config=config)
  1291. self.assertEqual("completed", result["status"])
  1292. children = await self.store.list_traces(
  1293. parent_trace_id=root_id,
  1294. created_by_tool="agent",
  1295. limit=10,
  1296. )
  1297. self.assertEqual([], children)
  1298. tool_messages = [
  1299. message
  1300. for message in await self.store.get_trace_messages(root_id)
  1301. if message.role == "tool"
  1302. ]
  1303. self.assertEqual(2, len(tool_messages))
  1304. self.assertTrue(all(
  1305. "only tool call" in str(message.content)
  1306. for message in tool_messages
  1307. ))
  1308. async def test_rewind_clears_protocol_state_after_cutoff(self):
  1309. trace = await self.store.get_trace(self.trace_id)
  1310. state = ensure_task_protocol(trace.context)
  1311. original_brief = TaskBrief.model_validate(state["task_brief"])
  1312. revised_brief = original_brief.model_copy(
  1313. update={"objective": "revised after the rewind cutoff"}
  1314. )
  1315. replace_task_brief(
  1316. state,
  1317. revised_brief,
  1318. effective_at_sequence=8,
  1319. )
  1320. late_snapshot = create_context_snapshot(
  1321. kind="reviewed_task_result",
  1322. summary="granted after cutoff",
  1323. source_trace_id="late-child",
  1324. root_trace_id="root",
  1325. uid="user-1",
  1326. content={"result": "must be removed by rewind"},
  1327. granted_at_sequence=8,
  1328. )
  1329. add_context_snapshot(
  1330. trace.context,
  1331. late_snapshot,
  1332. root_task_anchor=ROOT_ANCHOR,
  1333. task_brief=revised_brief,
  1334. )
  1335. report = TaskReport(
  1336. child_trace_id=self.trace_id,
  1337. **report_payload(),
  1338. ).model_dump()
  1339. state["task_report"] = report
  1340. state["task_report_submitted_at_sequence"] = 5
  1341. state["pending_reviews"] = {
  1342. "grandchild": {
  1343. "goal_id": "1",
  1344. "task_report": report,
  1345. "validation_result": validation_payload("grandchild"),
  1346. "received_at_sequence": 6,
  1347. }
  1348. }
  1349. state["reviews"] = [{"reviewed_at_sequence": 7}]
  1350. state["next_actions"] = [{"created_at_sequence": 8}]
  1351. await self.store.update_trace(self.trace_id, context=trace.context)
  1352. tree = GoalTree(
  1353. mission="rewind",
  1354. goals=[Goal(id="1", description="active", status="pending_review")],
  1355. current_id="1",
  1356. )
  1357. await self.store.update_goal_tree(self.trace_id, tree)
  1358. for sequence in (1, 2):
  1359. await self.store.add_message(Message.create(
  1360. trace_id=self.trace_id,
  1361. role="user",
  1362. sequence=sequence,
  1363. content=f"message-{sequence}",
  1364. ))
  1365. runner = AgentRunner(trace_store=self.store)
  1366. cutoff = await runner._rewind(self.trace_id, 2, tree)
  1367. self.assertEqual(3, cutoff)
  1368. rewound = await self.store.get_trace(self.trace_id)
  1369. state = ensure_task_protocol(rewound.context)
  1370. self.assertIsNone(state["task_report"])
  1371. self.assertEqual(1, len(state["report_history"]))
  1372. self.assertEqual({}, state["pending_reviews"])
  1373. self.assertEqual([], state["reviews"])
  1374. self.assertEqual([], state["next_actions"])
  1375. self.assertEqual(1, state["task_brief_version"])
  1376. self.assertEqual(original_brief.model_dump(), state["task_brief"])
  1377. self.assertEqual([], state["task_brief_history"])
  1378. self.assertEqual({}, rewound.context["context_access"]["snapshots"])
  1379. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  1380. self.assertEqual("pending", rewound_tree.find("1").status)
  1381. async def test_rewind_restores_preserved_pending_review_projection(self):
  1382. trace = await self.store.get_trace(self.trace_id)
  1383. state = ensure_task_protocol(trace.context)
  1384. report = TaskReport(
  1385. child_trace_id="preserved-grandchild",
  1386. **report_payload(),
  1387. ).model_dump()
  1388. state["pending_reviews"] = {
  1389. "preserved-grandchild": {
  1390. "goal_id": "1",
  1391. "task_report": report,
  1392. "validation_result": validation_payload(
  1393. "preserved-grandchild"
  1394. ),
  1395. "received_at_sequence": 2,
  1396. }
  1397. }
  1398. await self.store.update_trace(self.trace_id, context=trace.context)
  1399. tree = GoalTree(
  1400. mission="rewind",
  1401. goals=[Goal(id="1", description="review", status="pending_review")],
  1402. current_id="1",
  1403. )
  1404. await self.store.update_goal_tree(self.trace_id, tree)
  1405. for sequence in (1, 2):
  1406. await self.store.add_message(Message.create(
  1407. trace_id=self.trace_id,
  1408. role="user",
  1409. sequence=sequence,
  1410. content=f"message-{sequence}",
  1411. ))
  1412. runner = AgentRunner(trace_store=self.store)
  1413. await runner._rewind(self.trace_id, 2, tree)
  1414. rewound = await self.store.get_trace(self.trace_id)
  1415. self.assertIn(
  1416. "preserved-grandchild",
  1417. ensure_task_protocol(rewound.context)["pending_reviews"],
  1418. )
  1419. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  1420. self.assertEqual("pending_review", rewound_tree.find("1").status)
  1421. async def test_rewind_rebuilds_root_validation_history_and_cache(self):
  1422. root_id = "root-validation-rewind"
  1423. state = new_task_protocol()
  1424. passed = validation_payload(root_id, outcome="passed")
  1425. failed = validation_payload(root_id, outcome="failed")
  1426. state["root_validation_history"] = [
  1427. {**passed, "validated_at_sequence": 2},
  1428. {**failed, "validated_at_sequence": 5},
  1429. ]
  1430. state["root_validation_attempts"] = 2
  1431. state["root_validation_passed"] = False
  1432. state["task_report_validation"] = {
  1433. "plan_hash": failed["plan_hash"],
  1434. "aggregate_result": failed,
  1435. "scope_results": failed["scope_results"],
  1436. "validated_at_sequence": 5,
  1437. }
  1438. await create_recursive_root(self.store, root_id, state=state)
  1439. tree = GoalTree(
  1440. mission="root",
  1441. goals=[Goal(id="1", description="root", status="in_progress")],
  1442. current_id="1",
  1443. )
  1444. await self.store.update_goal_tree(root_id, tree)
  1445. for sequence in (1, 2, 3):
  1446. await self.store.add_message(Message.create(
  1447. trace_id=root_id,
  1448. role="user",
  1449. sequence=sequence,
  1450. content=f"message-{sequence}",
  1451. ))
  1452. await AgentRunner(trace_store=self.store)._rewind(root_id, 3, tree)
  1453. rewound = await self.store.get_trace(root_id)
  1454. state = ensure_task_protocol(rewound.context)
  1455. self.assertEqual(1, state["root_validation_attempts"])
  1456. self.assertTrue(state["root_validation_passed"])
  1457. self.assertEqual(
  1458. ["passed"],
  1459. [item["outcome"] for item in state["root_validation_history"]],
  1460. )
  1461. self.assertIsNone(state["task_report_validation"])
  1462. async def test_root_cannot_finish_with_unconsumed_next_action(self):
  1463. root_id = "next-action-gate-root"
  1464. state = new_task_protocol()
  1465. state["next_actions"] = [{
  1466. "decision": "ACCEPT_REFINE",
  1467. "child_trace_id": "reviewed-child",
  1468. "goal_id": None,
  1469. "task_brief": BRIEF,
  1470. "created_at_sequence": 1,
  1471. }]
  1472. await create_recursive_root(self.store, root_id, state=state)
  1473. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  1474. calls = 0
  1475. async def llm_call(**kwargs):
  1476. nonlocal calls
  1477. calls += 1
  1478. return {
  1479. "content": "Trying to finish without the approved action.",
  1480. "tool_calls": [],
  1481. "finish_reason": "stop",
  1482. }
  1483. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1484. config = await self.bind_snapshot(
  1485. root_id,
  1486. RunConfig(
  1487. trace_id=root_id,
  1488. max_iterations=5,
  1489. knowledge=KnowledgeConfig(
  1490. enable_extraction=False,
  1491. enable_completion_extraction=False,
  1492. enable_injection=False,
  1493. ),
  1494. ),
  1495. )
  1496. result = await runner.run_result(messages=[], config=config)
  1497. self.assertEqual("failed", result["status"])
  1498. self.assertEqual(3, calls)
  1499. root = await self.store.get_trace(root_id)
  1500. self.assertEqual(1, len(ensure_task_protocol(root.context)["next_actions"]))
  1501. class RuntimeProtocolSchemaTest(unittest.TestCase):
  1502. def setUp(self):
  1503. self.runner = AgentRunner(tool_registry=get_tool_registry())
  1504. def names(self, schemas):
  1505. return {schema["function"]["name"] for schema in schemas}
  1506. def test_legacy_and_recursive_runtime_schemas_are_isolated(self):
  1507. legacy = Trace(
  1508. trace_id="legacy",
  1509. mode="agent",
  1510. context={"agent_mode": "legacy", "agent_mode_revision": 1},
  1511. )
  1512. legacy_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), legacy)
  1513. self.assertNotIn("submit_task_report", self.names(legacy_schemas))
  1514. self.assertNotIn("review_task_result", self.names(legacy_schemas))
  1515. legacy_agent = next(
  1516. item for item in legacy_schemas if item["function"]["name"] == "agent"
  1517. )
  1518. properties = legacy_agent["function"]["parameters"]["properties"]
  1519. self.assertIn("task", properties)
  1520. self.assertNotIn("task_brief", properties)
  1521. child = Trace(
  1522. trace_id="child",
  1523. mode="agent",
  1524. parent_trace_id="root",
  1525. context={
  1526. "agent_mode": "recursive",
  1527. "agent_mode_revision": 2,
  1528. "task_protocol": new_task_protocol(BRIEF),
  1529. },
  1530. )
  1531. child_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1532. self.assertIn("submit_task_report", self.names(child_schemas))
  1533. self.assertNotIn("review_task_result", self.names(child_schemas))
  1534. self.assertNotIn("evaluate", self.names(child_schemas))
  1535. recursive_agent = next(
  1536. item for item in child_schemas if item["function"]["name"] == "agent"
  1537. )
  1538. recursive_properties = recursive_agent["function"]["parameters"]["properties"]
  1539. self.assertNotIn("messages", recursive_properties)
  1540. brief_object = recursive_properties["task_brief"]["oneOf"][0]
  1541. self.assertEqual(
  1542. {
  1543. "objective",
  1544. "reason",
  1545. "completion_criteria",
  1546. "expected_outputs",
  1547. },
  1548. set(brief_object["required"]),
  1549. )
  1550. state = ensure_task_protocol(child.context)
  1551. state["pending_reviews"]["grandchild"] = {"task_report": {}}
  1552. review_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1553. self.assertEqual(
  1554. {"review_task_result", "read_context_ref"},
  1555. self.names(review_schemas),
  1556. )
  1557. def test_protocol_tools_publish_nested_object_schemas(self):
  1558. schemas = {
  1559. schema["function"]["name"]: schema
  1560. for schema in get_tool_registry().get_schemas([
  1561. "submit_task_report",
  1562. "review_task_result",
  1563. ])
  1564. }
  1565. report_schema = schemas["submit_task_report"]["function"]["parameters"]
  1566. task_report = report_schema["properties"]["task_report"]
  1567. self.assertEqual("object", task_report["type"])
  1568. self.assertIn("validation", task_report["properties"])
  1569. self.assertIn("next_step_suggestion", task_report["properties"])
  1570. review_schema = schemas["review_task_result"]["function"]["parameters"]
  1571. self.assertEqual(
  1572. [
  1573. "ACCEPT_REFINE",
  1574. "REJECT_REFINE",
  1575. "REVISE_CHILD",
  1576. "DESCEND_AGAIN",
  1577. "REPLAN_CURRENT",
  1578. "ASCEND",
  1579. "FAIL",
  1580. ],
  1581. review_schema["properties"]["decision"]["enum"],
  1582. )
  1583. self.assertEqual(
  1584. "object",
  1585. review_schema["properties"]["approved_next_task"]["type"],
  1586. )
  1587. if __name__ == "__main__":
  1588. unittest.main()