test_task_protocol.py 67 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764
  1. import json
  2. import tempfile
  3. import unittest
  4. from unittest.mock import AsyncMock, patch
  5. from pydantic import ValidationError
  6. from cyber_agent.core.agent_mode import RECURSIVE_CAPABILITY_TOOLS_CONTEXT_KEY
  7. from cyber_agent.core.resource_budget import (
  8. RESOURCE_BUDGET_CONTEXT_KEY,
  9. ResourceBudget,
  10. ResourceBudgetController,
  11. )
  12. from cyber_agent.core.runner import AgentRunner, RunConfig
  13. from cyber_agent.core.run_snapshot import (
  14. RunConfigSnapshotV1,
  15. persist_run_config_snapshot,
  16. )
  17. from cyber_agent.core.context_policy import (
  18. add_context_snapshot,
  19. create_context_snapshot,
  20. persist_root_task_anchor,
  21. )
  22. from cyber_agent.core.task_protocol import (
  23. TaskProgress,
  24. TaskBrief,
  25. TaskReport,
  26. TaskReview,
  27. append_task_progress_revision,
  28. ensure_task_protocol,
  29. initialize_task_progress,
  30. new_task_protocol,
  31. replace_task_brief,
  32. task_contract_ref,
  33. )
  34. from cyber_agent.core.task_protocol_service import TaskProtocolService
  35. from cyber_agent.tools import get_tool_registry
  36. from cyber_agent.tools.builtin.knowledge import KnowledgeConfig
  37. from cyber_agent.tools.builtin.subagent import _record_pending_task_reports, agent
  38. from cyber_agent.tools.builtin.task_protocol import (
  39. review_task_result,
  40. submit_task_report,
  41. update_task_progress,
  42. )
  43. from cyber_agent.trace.goal_models import Goal, GoalTree
  44. from cyber_agent.trace.goal_tool import goal_tool
  45. from cyber_agent.trace.models import Message, Trace
  46. from cyber_agent.trace.store import FileSystemTraceStore
  47. from cyber_agent.core.validation import (
  48. ScopeValidationResult,
  49. ValidationCheck,
  50. ValidationPolicy,
  51. ValidationResult,
  52. ValidationRun,
  53. ValidatorSettings,
  54. persist_validation_policy,
  55. )
  56. BRIEF = {
  57. "objective": "research one concrete question",
  58. "reason": "the parent needs one independently checked result",
  59. "completion_criteria": ["provide one checked conclusion"],
  60. "expected_outputs": ["one checked conclusion"],
  61. "constraints": ["do not guess"],
  62. }
  63. ROOT_ANCHOR = {
  64. "objective": "finish the root task",
  65. "completion_criteria": ["finish the root task"],
  66. "constraints": ["do not guess"],
  67. }
  68. def with_root_anchor(context):
  69. persist_root_task_anchor(context, ROOT_ANCHOR)
  70. state = ensure_task_protocol(context)
  71. if state.get("task_progress_head_revision") is None:
  72. initialize_task_progress(
  73. state,
  74. root_task_anchor_hash=context.get("root_task_anchor_hash"),
  75. )
  76. current = state.get("task_progress_head_revision")
  77. if current == 1:
  78. append_task_progress_revision(
  79. state,
  80. TaskProgress(phase="ready_to_submit"),
  81. expected_revision=current,
  82. effective_at_sequence=0,
  83. contract_ref=task_contract_ref(
  84. state,
  85. root_task_anchor_hash=context.get("root_task_anchor_hash"),
  86. ),
  87. )
  88. return context
  89. def report_payload(*, outcome="satisfied"):
  90. return {
  91. "summary": "checked result",
  92. "outcome": outcome,
  93. "validation": {
  94. "hard_passed": outcome == "satisfied",
  95. "open_issues": [] if outcome == "satisfied" else ["still failing"],
  96. },
  97. "next_step_suggestion": {
  98. "direction": "NONE",
  99. "reason": "No additional task is needed",
  100. },
  101. "outputs": [{"kind": "note", "value": "result"}],
  102. "evidence": [{"kind": "test", "value": "passed"}],
  103. "remaining_issues": [] if outcome == "satisfied" else ["still failing"],
  104. }
  105. def validation_payload(child_trace_id, *, outcome="passed"):
  106. plan_hash = "a" * 64
  107. if outcome == "passed":
  108. issues = []
  109. retry_from = None
  110. elif outcome == "failed":
  111. issues = ["task needs revision"]
  112. retry_from = "task_definition"
  113. else:
  114. issues = ["validator could not evaluate the task"]
  115. retry_from = None
  116. scope_result = ScopeValidationResult(
  117. validator_trace_id=f"{child_trace_id}@validator-fixture",
  118. scope="task",
  119. outcome=outcome,
  120. checks=[ValidationCheck(
  121. check_id="task.fixture",
  122. status=("unknown" if outcome == "error" else outcome),
  123. issue=None if outcome == "passed" else issues[0],
  124. )],
  125. reason="fixture validation",
  126. retry_from=retry_from,
  127. plan_hash=plan_hash,
  128. )
  129. return ValidationResult(
  130. evaluated_trace_id=child_trace_id,
  131. outcome=outcome,
  132. scope_results=[scope_result],
  133. issues=issues,
  134. retry_from=retry_from,
  135. plan_hash=plan_hash,
  136. ).model_dump(mode="json")
  137. def passed_validator_response(kwargs):
  138. """按当前 ValidationPlan 返回完整的通过结果。"""
  139. messages = kwargs.get("messages") or []
  140. packet = json.loads(messages[1]["content"])
  141. scope = packet["validation_scope"]
  142. checks = [
  143. {
  144. "check_id": item["check_id"],
  145. "status": "passed",
  146. "evidence_refs": [],
  147. "issue": None,
  148. }
  149. for item in packet["validation_plan"]["checks"]
  150. if item["scope"] == scope
  151. ]
  152. return {
  153. "content": json.dumps({
  154. "outcome": "passed",
  155. "scope": scope,
  156. "checks": checks,
  157. "reason": "all planned checks passed",
  158. "retry_from": None,
  159. }),
  160. "tool_calls": [],
  161. "finish_reason": "stop",
  162. }
  163. async def create_recursive_root(store, trace_id, *, state=None, uid="user-1"):
  164. budget = ResourceBudget()
  165. context = {
  166. "agent_mode": "recursive",
  167. "agent_mode_revision": 3,
  168. "agent_depth": 0,
  169. "root_trace_id": trace_id,
  170. RESOURCE_BUDGET_CONTEXT_KEY: budget.to_dict(),
  171. "task_protocol": state or new_task_protocol(),
  172. }
  173. with_root_anchor(context)
  174. persist_validation_policy(
  175. context,
  176. ValidationPolicy(),
  177. ValidatorSettings(),
  178. )
  179. await store.create_trace(Trace(
  180. trace_id=trace_id,
  181. mode="agent",
  182. task="root",
  183. uid=uid,
  184. model="fake-model",
  185. context=context,
  186. ))
  187. await ResourceBudgetController(store).initialize(
  188. trace_id,
  189. budget,
  190. initial_agents=1,
  191. )
  192. return await store.get_trace(trace_id)
  193. class FakeToolRegistry:
  194. def get_tool_names(self):
  195. return [
  196. "agent",
  197. "evaluate",
  198. "bash_command",
  199. "goal",
  200. "read_file",
  201. "update_task_progress",
  202. "submit_task_report",
  203. "review_task_result",
  204. "read_context_ref",
  205. ]
  206. class ReportingRunner:
  207. def __init__(self, store):
  208. self.store = store
  209. self.tools = FakeToolRegistry()
  210. self.debug = False
  211. self.stdin_check = None
  212. self._cancel_events = {}
  213. self.configs = []
  214. self.resource_budget = ResourceBudgetController(store)
  215. self.validation_count = 0
  216. async def _resource_budget_for_trace(self, trace_id):
  217. trace = await self.store.get_trace(trace_id)
  218. root_id = trace.context["root_trace_id"]
  219. root = trace if root_id == trace_id else await self.store.get_trace(root_id)
  220. return root_id, ResourceBudget.from_dict(
  221. root.context[RESOURCE_BUDGET_CONTEXT_KEY]
  222. )
  223. async def validate_recursive_trace(
  224. self,
  225. evaluated_trace_id,
  226. *,
  227. scope,
  228. deterministic_failure=None,
  229. **_kwargs,
  230. ):
  231. evaluated = await self.store.get_trace(evaluated_trace_id)
  232. child_state = ensure_task_protocol(evaluated.context)
  233. cached = child_state.get("task_report_validation")
  234. if isinstance(cached, dict) and cached.get("aggregate_result"):
  235. result = ValidationResult.model_validate(cached["aggregate_result"])
  236. return ValidationRun(
  237. result=result,
  238. trace_ids=[
  239. item.validator_trace_id for item in result.scope_results
  240. ],
  241. cached=True,
  242. )
  243. self.validation_count += 1
  244. validator_trace_id = (
  245. f"{evaluated_trace_id}@validator-fixture-{self.validation_count}"
  246. )
  247. if deterministic_failure:
  248. outcome = deterministic_failure.get("outcome", "error")
  249. issues = deterministic_failure.get("issues") or [
  250. deterministic_failure.get("reason", "validation failed")
  251. ]
  252. retry_from = deterministic_failure.get("retry_from")
  253. else:
  254. outcome = "passed"
  255. issues = []
  256. retry_from = None
  257. result = ValidationResult.model_validate(
  258. validation_payload(evaluated_trace_id, outcome=outcome)
  259. )
  260. child_state["task_report_validation"] = {
  261. "validation_plan": None,
  262. "plan_hash": result.plan_hash,
  263. "scope_results": [
  264. item.model_dump(mode="json") for item in result.scope_results
  265. ],
  266. "aggregate_result": result.model_dump(mode="json"),
  267. "validated_at_sequence": evaluated.head_sequence,
  268. }
  269. await self.store.update_trace(
  270. evaluated_trace_id,
  271. context=evaluated.context,
  272. )
  273. await self.store.create_trace(Trace(
  274. trace_id=validator_trace_id,
  275. mode="agent",
  276. task=f"validate {evaluated_trace_id}",
  277. parent_trace_id=evaluated_trace_id,
  278. uid=evaluated.uid,
  279. status="completed",
  280. tools=[],
  281. context=with_root_anchor({
  282. "created_by_tool": "validator",
  283. "root_trace_id": evaluated.context["root_trace_id"],
  284. "agent_depth": evaluated.context["agent_depth"],
  285. }),
  286. ))
  287. return ValidationRun(result=result, trace_ids=[validator_trace_id])
  288. async def run_result(self, messages, config, on_event=None):
  289. self.configs.append(config)
  290. trace = await self.store.get_trace(config.trace_id)
  291. current_revision = ensure_task_protocol(trace.context)[
  292. "task_progress_head_revision"
  293. ]
  294. progressed = await update_task_progress(
  295. expected_revision=current_revision,
  296. progress={
  297. "phase": "ready_to_submit",
  298. "questions": [],
  299. "blockers": [],
  300. "findings": [],
  301. "hypotheses": [],
  302. "work_items": [],
  303. "decision_rationale": "The fixture completed its assigned work.",
  304. },
  305. context={
  306. "store": self.store,
  307. "task_protocol_service": TaskProtocolService(self.store),
  308. "trace_id": config.trace_id,
  309. "sequence": 2,
  310. },
  311. )
  312. if progressed["status"] != "completed":
  313. raise AssertionError(progressed)
  314. submitted = await submit_task_report(
  315. task_report=report_payload(),
  316. context={
  317. "store": self.store,
  318. "trace_id": config.trace_id,
  319. "sequence": 3,
  320. },
  321. )
  322. if submitted["status"] != "completed":
  323. raise AssertionError(submitted)
  324. await self.store.update_trace(config.trace_id, status="completed")
  325. return {
  326. "status": "completed",
  327. "summary": "child submitted TaskReport",
  328. "saved_knowledge_ids": [],
  329. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  330. }
  331. class FailedAfterReportingRunner(ReportingRunner):
  332. async def run_result(self, messages, config, on_event=None):
  333. result = await super().run_result(messages, config, on_event)
  334. await self.store.update_trace(
  335. config.trace_id,
  336. status="failed",
  337. error_message="runner failed after report submission",
  338. )
  339. return {
  340. **result,
  341. "status": "failed",
  342. "error": "runner failed after report submission",
  343. }
  344. class ForgedIdentityRunner(ReportingRunner):
  345. async def run_result(self, messages, config, on_event=None):
  346. trace = await self.store.get_trace(config.trace_id)
  347. state = ensure_task_protocol(trace.context)
  348. state["task_report"] = TaskReport(
  349. child_trace_id="someone-elses-trace",
  350. **report_payload(),
  351. ).model_dump()
  352. await self.store.update_trace(
  353. config.trace_id,
  354. context=trace.context,
  355. status="completed",
  356. )
  357. return {
  358. "status": "completed",
  359. "summary": "forged report identity",
  360. "saved_knowledge_ids": [],
  361. "stats": {"total_messages": 1, "total_tokens": 1, "total_cost": 0.0},
  362. }
  363. class RecursiveTaskProtocolTest(unittest.IsolatedAsyncioTestCase):
  364. async def asyncSetUp(self):
  365. self.temp_dir = tempfile.TemporaryDirectory()
  366. self.store = FileSystemTraceStore(self.temp_dir.name)
  367. self.runner = ReportingRunner(self.store)
  368. self.root_id = "recursive-root"
  369. await create_recursive_root(self.store, self.root_id)
  370. self.tree = GoalTree(
  371. mission="root",
  372. goals=[Goal(id="1", description="delegate", status="in_progress")],
  373. current_id="1",
  374. )
  375. await self.store.update_goal_tree(self.root_id, self.tree)
  376. async def asyncTearDown(self):
  377. self.temp_dir.cleanup()
  378. def context(self, trace_id=None, goal_id="1", tree=None):
  379. return {
  380. "store": self.store,
  381. "trace_id": trace_id or self.root_id,
  382. "goal_id": goal_id,
  383. "runner": self.runner,
  384. "goal_tree": tree or self.tree,
  385. "sequence": 10,
  386. RECURSIVE_CAPABILITY_TOOLS_CONTEXT_KEY: self.runner.tools.get_tool_names(),
  387. }
  388. async def children(self):
  389. return await self.store.list_traces(
  390. parent_trace_id=self.root_id,
  391. created_by_tool="agent",
  392. limit=20,
  393. )
  394. async def test_models_validate_and_round_trip(self):
  395. brief = TaskBrief.model_validate(BRIEF)
  396. self.assertEqual(
  397. {
  398. **BRIEF,
  399. "parent_findings": [],
  400. "context": {},
  401. "context_refs": [],
  402. "validation_scopes": [],
  403. },
  404. brief.model_dump(exclude_none=True),
  405. )
  406. with self.assertRaises(ValidationError):
  407. TaskBrief(objective="", completion_criteria=[])
  408. with self.assertRaises(ValidationError):
  409. TaskReview(
  410. parent_trace_id="p",
  411. child_trace_id="c",
  412. decision="ACCEPT_REFINE",
  413. reason="continue",
  414. )
  415. invalid_suggestion = report_payload()
  416. invalid_suggestion["next_step_suggestion"] = {
  417. "direction": "NONE",
  418. "reason": "No next task should be attached",
  419. "suggested_next_task": BRIEF,
  420. }
  421. with self.assertRaises(ValidationError):
  422. TaskReport(child_trace_id="child", **invalid_suggestion)
  423. root_submission = await submit_task_report(
  424. task_report=report_payload(),
  425. context={"store": self.store, "trace_id": self.root_id},
  426. )
  427. self.assertEqual("failed", root_submission["status"])
  428. self.assertIn("Root Agent", root_submission["error"])
  429. async def test_child_report_blocks_new_child_until_parent_review(self):
  430. result = await agent(task_brief=BRIEF, context=self.context())
  431. self.assertEqual("completed", result["status"])
  432. child_id = result["sub_trace_id"]
  433. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  434. parent = await self.store.get_trace(self.root_id)
  435. state = ensure_task_protocol(parent.context)
  436. self.assertEqual([child_id], list(state["pending_reviews"]))
  437. tree = await self.store.get_goal_tree(self.root_id)
  438. self.assertEqual("pending_review", tree.find("1").status)
  439. restarted_runner = AgentRunner(tool_registry=get_tool_registry())
  440. restored = await self.store.get_trace(self.root_id)
  441. restored_names = {
  442. schema["function"]["name"]
  443. for schema in restarted_runner._get_runtime_tool_schemas(
  444. RunConfig(), restored
  445. )
  446. }
  447. self.assertEqual(
  448. {"review_task_result", "read_context_ref"},
  449. restored_names,
  450. )
  451. blocked = await agent(task_brief=BRIEF, context=self.context())
  452. self.assertEqual("failed", blocked["status"])
  453. self.assertIn("Review pending", blocked["error"])
  454. async def test_child_inherits_brief_constraints_not_parent_runtime_context(self):
  455. parent = await self.store.get_trace(self.root_id)
  456. parent.context["parent_private_runtime"] = {"secret": "must-not-leak"}
  457. await self.store.update_trace(self.root_id, context=parent.context)
  458. result = await agent(task_brief=BRIEF, context=self.context())
  459. child = await self.store.get_trace(result["sub_trace_id"])
  460. self.assertNotIn("parent_private_runtime", child.context)
  461. child_brief = ensure_task_protocol(child.context)["task_brief"]
  462. self.assertEqual(["do not guess"], child_brief["constraints"])
  463. child_config_context = self.runner.configs[-1].context
  464. self.assertNotIn("parent_private_runtime", child_config_context)
  465. async def test_same_task_report_version_reuses_its_validator_result(self):
  466. result = await agent(task_brief=BRIEF, context=self.context())
  467. child_id = result["sub_trace_id"]
  468. self.assertEqual(1, self.runner.validation_count)
  469. parent = await self.store.get_trace(self.root_id)
  470. await _record_pending_task_reports(
  471. self.store,
  472. self.runner,
  473. parent,
  474. "1",
  475. [(child_id, {"status": "completed", "summary": "same report"})],
  476. 10,
  477. )
  478. self.assertEqual(1, self.runner.validation_count)
  479. validators = await self.store.list_traces(
  480. parent_trace_id=child_id,
  481. created_by_tool="validator",
  482. limit=10,
  483. )
  484. self.assertEqual(1, len(validators))
  485. async def test_recursive_messages_are_rejected(self):
  486. result = await agent(
  487. task_brief=json.dumps(BRIEF),
  488. messages=json.dumps([{
  489. "role": "system",
  490. "content": "controlled child context",
  491. }]),
  492. context=self.context(),
  493. )
  494. self.assertEqual("failed", result["status"])
  495. self.assertIn("does not accept messages", result["error"])
  496. async def test_messages_reject_mixed_dimensions(self):
  497. mixed_messages = [
  498. {"role": "system", "content": "shared context"},
  499. [{"role": "user", "content": "per-agent context"}],
  500. ]
  501. for messages in (mixed_messages, json.dumps(mixed_messages)):
  502. with self.subTest(messages_type=type(messages).__name__):
  503. result = await agent(
  504. task_brief=BRIEF,
  505. messages=messages,
  506. context=self.context(),
  507. )
  508. self.assertEqual("failed", result["status"])
  509. self.assertIn("without mixed dimensions", result["error"])
  510. async def test_no_focused_goal_still_persists_and_reviews_report(self):
  511. result = await agent(
  512. task_brief=BRIEF,
  513. context=self.context(goal_id=None),
  514. )
  515. parent = await self.store.get_trace(self.root_id)
  516. entry = ensure_task_protocol(parent.context)["pending_reviews"][
  517. result["sub_trace_id"]
  518. ]
  519. self.assertIsNone(entry["goal_id"])
  520. reviewed = await review_task_result(
  521. child_trace_id=result["sub_trace_id"],
  522. decision="ASCEND",
  523. reason="Standalone delegation is complete",
  524. context=self.context(goal_id=None),
  525. )
  526. self.assertEqual("completed", reviewed["status"])
  527. async def test_accept_refine_approves_exactly_one_next_child(self):
  528. first = await agent(task_brief=BRIEF, context=self.context())
  529. next_brief = {
  530. "objective": "check the independent gap",
  531. "reason": "the first report identified a separate gap",
  532. "completion_criteria": ["close the gap"],
  533. "expected_outputs": ["a checked gap analysis"],
  534. }
  535. parent = await self.store.get_trace(self.root_id)
  536. ensure_task_protocol(parent.context)["protocol_correction_attempts"] = 2
  537. await self.store.update_trace(self.root_id, context=parent.context)
  538. reviewed = await review_task_result(
  539. child_trace_id=first["sub_trace_id"],
  540. decision="ACCEPT_REFINE",
  541. reason="The gap belongs to this parent",
  542. approved_next_task=json.dumps(next_brief),
  543. context=self.context(),
  544. )
  545. self.assertEqual("completed", reviewed["status"])
  546. self.assertIsNotNone(reviewed["context_ref"])
  547. self.assertIn(
  548. reviewed["context_ref"],
  549. reviewed["next_action_task_brief"]["context_refs"],
  550. )
  551. parent = await self.store.get_trace(self.root_id)
  552. self.assertEqual(
  553. 0,
  554. ensure_task_protocol(parent.context)["protocol_correction_attempts"],
  555. )
  556. schemas = AgentRunner(
  557. tool_registry=get_tool_registry()
  558. )._get_runtime_tool_schemas(RunConfig(), parent)
  559. self.assertEqual(
  560. {"agent", "read_context_ref"},
  561. {schema["function"]["name"] for schema in schemas},
  562. )
  563. wrong = await agent(task_brief=BRIEF, context=self.context())
  564. self.assertEqual("failed", wrong["status"])
  565. self.assertIn("approved next task", wrong["error"])
  566. second = await agent(
  567. task_brief=reviewed["next_action_task_brief"],
  568. context=self.context(),
  569. )
  570. self.assertEqual("completed", second["status"])
  571. self.assertEqual(2, len(await self.children()))
  572. second_child = await self.store.get_trace(second["sub_trace_id"])
  573. self.assertIn(
  574. reviewed["context_ref"]["ref_id"],
  575. second_child.context["context_access"]["snapshots"],
  576. )
  577. parent = await self.store.get_trace(self.root_id)
  578. self.assertEqual([], ensure_task_protocol(parent.context)["next_actions"])
  579. async def test_review_rejects_an_unusable_context_ref_without_consuming_report(self):
  580. first = await agent(task_brief=BRIEF, context=self.context())
  581. invalid_next = {
  582. "objective": "use an unauthorized context ref",
  583. "reason": "exercise approval validation",
  584. "completion_criteria": ["must not be approved"],
  585. "expected_outputs": ["nothing"],
  586. "context_refs": [{
  587. "ref_id": "ctx_000000000000000000000000",
  588. "version": "0" * 64,
  589. }],
  590. }
  591. reviewed = await review_task_result(
  592. child_trace_id=first["sub_trace_id"],
  593. decision="ACCEPT_REFINE",
  594. reason="try an invalid transfer",
  595. approved_next_task=invalid_next,
  596. context=self.context(),
  597. )
  598. self.assertEqual("failed", reviewed["status"])
  599. self.assertIn("not authorized", reviewed["error"])
  600. parent = await self.store.get_trace(self.root_id)
  601. state = ensure_task_protocol(parent.context)
  602. self.assertIn(first["sub_trace_id"], state["pending_reviews"])
  603. self.assertEqual([], state["reviews"])
  604. async def test_oversized_review_snapshot_warns_without_blocking_review(self):
  605. first = await agent(task_brief=BRIEF, context=self.context())
  606. parent = await self.store.get_trace(self.root_id)
  607. state = ensure_task_protocol(parent.context)
  608. state["pending_reviews"][first["sub_trace_id"]]["task_report"]["outputs"] = [
  609. {"payload": "x" * 16_000}
  610. ]
  611. await self.store.update_trace(self.root_id, context=parent.context)
  612. reviewed = await review_task_result(
  613. child_trace_id=first["sub_trace_id"],
  614. decision="ASCEND",
  615. reason="The task passed but its transferable snapshot is too large",
  616. context=self.context(),
  617. )
  618. self.assertEqual("completed", reviewed["status"])
  619. self.assertIsNone(reviewed["context_ref"])
  620. self.assertIn("16000", reviewed["context_ref_warning"])
  621. parent = await self.store.get_trace(self.root_id)
  622. self.assertEqual({}, ensure_task_protocol(parent.context)["pending_reviews"])
  623. async def test_batch_context_refs_are_validated_before_any_trace_is_created(self):
  624. invalid_ref_brief = {
  625. **BRIEF,
  626. "objective": "second child with an unauthorized reference",
  627. "context_refs": [{
  628. "ref_id": "ctx_000000000000000000000000",
  629. "version": "0" * 64,
  630. }],
  631. }
  632. result = await agent(
  633. task_brief=[BRIEF, invalid_ref_brief],
  634. context=self.context(),
  635. )
  636. self.assertEqual("failed", result["status"])
  637. self.assertIn("not authorized", result["error"])
  638. self.assertEqual([], await self.children())
  639. async def test_batch_reports_are_reviewed_one_by_one(self):
  640. briefs = [
  641. BRIEF,
  642. {
  643. "objective": "research second question",
  644. "reason": "the batch needs a second independent result",
  645. "completion_criteria": ["provide second conclusion"],
  646. "expected_outputs": ["a second conclusion"],
  647. },
  648. ]
  649. result = await agent(task_brief=briefs, context=self.context())
  650. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  651. self.assertEqual(2, len(result["task_reports"]))
  652. premature = await review_task_result(
  653. child_trace_id=child_ids[0],
  654. decision="ASCEND",
  655. reason="Cannot finish while another report is pending",
  656. context=self.context(),
  657. )
  658. self.assertEqual("failed", premature["status"])
  659. self.assertIn("final pending report", premature["error"])
  660. first_review = await review_task_result(
  661. child_trace_id=child_ids[0],
  662. decision="REJECT_REFINE",
  663. reason="No extra task needed for this report",
  664. context=self.context(),
  665. )
  666. self.assertEqual(1, first_review["pending_review_count"])
  667. tree = await self.store.get_goal_tree(self.root_id)
  668. self.assertEqual("pending_review", tree.find("1").status)
  669. final_review = await review_task_result(
  670. child_trace_id=child_ids[1],
  671. decision="ASCEND",
  672. reason="All completion criteria are satisfied",
  673. context=self.context(),
  674. )
  675. self.assertEqual(0, final_review["pending_review_count"])
  676. tree = await self.store.get_goal_tree(self.root_id)
  677. self.assertEqual("completed", tree.find("1").status)
  678. duplicate = await review_task_result(
  679. child_trace_id=child_ids[1],
  680. decision="ASCEND",
  681. reason="Duplicate review must fail",
  682. context=self.context(),
  683. )
  684. self.assertEqual("failed", duplicate["status"])
  685. self.assertIn("no pending TaskReport", duplicate["error"])
  686. async def test_terminal_review_waits_for_queued_next_action(self):
  687. result = await agent(
  688. task_brief=[
  689. BRIEF,
  690. {
  691. "objective": "second batch child",
  692. "reason": "exercise queued batch behavior",
  693. "completion_criteria": ["finish the second child"],
  694. "expected_outputs": ["the second child result"],
  695. },
  696. ],
  697. context=self.context(),
  698. )
  699. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  700. next_brief = {
  701. "objective": "approved follow-up",
  702. "reason": "the parent approved one follow-up",
  703. "completion_criteria": ["finish the approved follow-up"],
  704. "expected_outputs": ["the approved follow-up result"],
  705. }
  706. accepted = await review_task_result(
  707. child_trace_id=child_ids[0],
  708. decision="ACCEPT_REFINE",
  709. reason="A separate follow-up is required",
  710. approved_next_task=next_brief,
  711. context=self.context(),
  712. )
  713. self.assertEqual("completed", accepted["status"])
  714. blocked = await review_task_result(
  715. child_trace_id=child_ids[1],
  716. decision="ASCEND",
  717. reason="Cannot ascend before the accepted follow-up",
  718. context=self.context(),
  719. )
  720. self.assertEqual("failed", blocked["status"])
  721. self.assertIn("approved next action", blocked["error"])
  722. tree = await self.store.get_goal_tree(self.root_id)
  723. self.assertEqual("pending_review", tree.find("1").status)
  724. final_non_terminal = await review_task_result(
  725. child_trace_id=child_ids[1],
  726. decision="REJECT_REFINE",
  727. reason="This report adds no other follow-up",
  728. context=self.context(),
  729. )
  730. self.assertEqual("completed", final_non_terminal["status"])
  731. tree = await self.store.get_goal_tree(self.root_id)
  732. self.assertEqual("in_progress", tree.find("1").status)
  733. async def test_runner_failure_overrides_stale_satisfied_report(self):
  734. self.runner = FailedAfterReportingRunner(self.store)
  735. result = await agent(task_brief=BRIEF, context=self.context())
  736. self.assertEqual("failed", result["status"])
  737. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  738. child = await self.store.get_trace(result["sub_trace_id"])
  739. child_state = ensure_task_protocol(child.context)
  740. self.assertEqual(1, len(child_state["report_history"]))
  741. self.assertEqual("satisfied", child_state["report_history"][0]["outcome"])
  742. invalid_ascend = await review_task_result(
  743. child_trace_id=result["sub_trace_id"],
  744. decision="ASCEND",
  745. reason="A failed execution cannot ascend",
  746. context=self.context(),
  747. )
  748. self.assertEqual("failed", invalid_ascend["status"])
  749. self.assertIn("only allow", invalid_ascend["error"])
  750. async def test_submit_and_review_reject_child_anchor_that_differs_from_root(self):
  751. child_context = with_root_anchor({
  752. "agent_mode": "recursive",
  753. "agent_mode_revision": 3,
  754. "agent_depth": 1,
  755. "root_trace_id": self.root_id,
  756. "created_by_tool": "agent",
  757. "task_protocol": new_task_protocol(BRIEF),
  758. })
  759. persist_root_task_anchor(child_context, {
  760. **ROOT_ANCHOR,
  761. "objective": "self-consistent but unauthorized root objective",
  762. })
  763. child_id = "mismatched-anchor-child"
  764. await self.store.create_trace(Trace(
  765. trace_id=child_id,
  766. mode="agent",
  767. task="child",
  768. parent_trace_id=self.root_id,
  769. parent_goal_id="1",
  770. uid="user-1",
  771. model="fake-model",
  772. context=child_context,
  773. ))
  774. submitted = await submit_task_report(
  775. task_report=report_payload(),
  776. context={"store": self.store, "trace_id": child_id},
  777. )
  778. self.assertEqual("failed", submitted["status"])
  779. self.assertIn("does not match", submitted["error"])
  780. parent = await self.store.get_trace(self.root_id)
  781. parent_state = ensure_task_protocol(parent.context)
  782. parent_state["pending_reviews"][child_id] = {
  783. "goal_id": "1",
  784. "task_report": TaskReport(
  785. child_trace_id=child_id,
  786. **report_payload(),
  787. ).model_dump(),
  788. "validation_result": validation_payload(child_id),
  789. "received_at_sequence": 1,
  790. }
  791. await self.store.update_trace(self.root_id, context=parent.context)
  792. reviewed = await review_task_result(
  793. child_trace_id=child_id,
  794. decision="ASCEND",
  795. reason="must not accept a different root anchor",
  796. context=self.context(),
  797. )
  798. self.assertEqual("failed", reviewed["status"])
  799. self.assertIn("does not match", reviewed["error"])
  800. parent = await self.store.get_trace(self.root_id)
  801. state = ensure_task_protocol(parent.context)
  802. self.assertIn(child_id, state["pending_reviews"])
  803. self.assertEqual([], state["reviews"])
  804. async def test_review_rejects_wrong_owner_even_if_report_is_injected(self):
  805. first = await agent(task_brief=BRIEF, context=self.context())
  806. child_id = first["sub_trace_id"]
  807. await self.store.update_trace(child_id, uid="another-user")
  808. rejected = await review_task_result(
  809. child_trace_id=child_id,
  810. decision="ASCEND",
  811. reason="Owner mismatch must be rejected",
  812. context=self.context(),
  813. )
  814. self.assertEqual("failed", rejected["status"])
  815. self.assertIn("same owner", rejected["error"])
  816. async def test_report_identity_must_match_direct_child(self):
  817. self.runner = ForgedIdentityRunner(self.store)
  818. result = await agent(task_brief=BRIEF, context=self.context())
  819. child_id = result["sub_trace_id"]
  820. self.assertEqual("protocol_error", result["task_report"]["outcome"])
  821. self.assertEqual(child_id, result["task_report"]["child_trace_id"])
  822. parent = await self.store.get_trace(self.root_id)
  823. pending_report = ensure_task_protocol(parent.context)["pending_reviews"][
  824. child_id
  825. ]["task_report"]
  826. pending_report["child_trace_id"] = "corrupted-after-recording"
  827. await self.store.update_trace(self.root_id, context=parent.context)
  828. rejected = await review_task_result(
  829. child_trace_id=child_id,
  830. decision="FAIL",
  831. reason="Corrupted identity cannot be reviewed",
  832. context=self.context(),
  833. )
  834. self.assertEqual("failed", rejected["status"])
  835. self.assertIn("does not match", rejected["error"])
  836. async def test_remote_agent_is_rejected_by_recursive_revision_2(self):
  837. remote_call = AsyncMock(return_value={
  838. "status": "completed",
  839. "summary": "remote result",
  840. })
  841. with patch(
  842. "cyber_agent.tools.builtin.subagent._run_remote_agent",
  843. new=remote_call,
  844. ):
  845. result = await agent(
  846. task="remote free text",
  847. agent_type="remote_librarian",
  848. context=self.context(),
  849. )
  850. self.assertEqual("failed", result["status"])
  851. self.assertIn("does not support remote agents", result["error"])
  852. remote_call.assert_not_awaited()
  853. async def test_revise_child_reuses_trace_and_preserves_report_history(self):
  854. first = await agent(task_brief=BRIEF, context=self.context())
  855. child_id = first["sub_trace_id"]
  856. reviewed = await review_task_result(
  857. child_trace_id=child_id,
  858. decision="REVISE_CHILD",
  859. reason="Please strengthen the evidence",
  860. context=self.context(),
  861. )
  862. self.assertEqual("completed", reviewed["status"])
  863. cleared_child = await self.store.get_trace(child_id)
  864. self.assertEqual({}, cleared_child.context["context_access"]["snapshots"])
  865. wrong = await agent(
  866. task_brief={
  867. "objective": "silently replace the reviewed task",
  868. "reason": "attempt a mismatched revision",
  869. "completion_criteria": ["this must be rejected"],
  870. "expected_outputs": ["a rejected replacement"],
  871. },
  872. continue_from=child_id,
  873. context=self.context(),
  874. )
  875. self.assertEqual("failed", wrong["status"])
  876. self.assertIn("approved revision", wrong["error"])
  877. revised_brief = reviewed["next_action_task_brief"]
  878. revised = await agent(
  879. task_brief=revised_brief,
  880. continue_from=child_id,
  881. context=self.context(),
  882. )
  883. self.assertEqual(child_id, revised["sub_trace_id"])
  884. self.assertTrue(revised["continue_from"])
  885. self.assertEqual(1, len(await self.children()))
  886. child = await self.store.get_trace(child_id)
  887. self.assertEqual(1, len(ensure_task_protocol(child.context)["report_history"]))
  888. self.assertEqual(
  889. TaskBrief.model_validate(revised_brief).model_dump(),
  890. ensure_task_protocol(child.context)["task_brief"],
  891. )
  892. self.assertEqual(2, ensure_task_protocol(child.context)["task_brief_version"])
  893. self.assertEqual(
  894. 1,
  895. len(ensure_task_protocol(child.context)["task_brief_history"]),
  896. )
  897. async def test_review_cannot_reserve_more_children_than_quota(self):
  898. briefs = [
  899. {
  900. "objective": f"batch child {index}",
  901. "reason": "fill the local child quota",
  902. "completion_criteria": [f"finish child {index}"],
  903. "expected_outputs": [f"result for child {index}"],
  904. }
  905. for index in range(5)
  906. ]
  907. result = await agent(task_brief=briefs, context=self.context())
  908. child_ids = [item["trace_id"] for item in result["sub_trace_ids"]]
  909. next_brief = {
  910. "objective": "one allowed reserved child",
  911. "reason": "use the final available child slot",
  912. "completion_criteria": ["use the final available slot"],
  913. "expected_outputs": ["one final child result"],
  914. }
  915. first = await review_task_result(
  916. child_trace_id=child_ids[0],
  917. decision="ACCEPT_REFINE",
  918. reason="Reserve the sixth and final child slot",
  919. approved_next_task=next_brief,
  920. context=self.context(),
  921. )
  922. self.assertEqual("completed", first["status"])
  923. overflow = await review_task_result(
  924. child_trace_id=child_ids[1],
  925. decision="DESCEND_AGAIN",
  926. reason="A seventh child must not be approved",
  927. approved_next_task={
  928. "objective": "overflow child",
  929. "reason": "exercise quota rejection",
  930. "completion_criteria": ["must never be scheduled"],
  931. "expected_outputs": ["no output because creation is rejected"],
  932. },
  933. context=self.context(),
  934. )
  935. self.assertEqual("failed", overflow["status"])
  936. self.assertIn("reserved=1", overflow["error"])
  937. parent = await self.store.get_trace(self.root_id)
  938. state = ensure_task_protocol(parent.context)
  939. self.assertEqual(1, len(state["next_actions"]))
  940. self.assertIn(child_ids[1], state["pending_reviews"])
  941. async def test_recursive_goal_completion_does_not_cascade_or_select_sibling(self):
  942. tree = GoalTree(mission="recursive")
  943. parent = tree.add_goals(["parent"])[0]
  944. first, second = tree.add_goals(["first", "second"], parent_id=parent.id)
  945. tree.focus(parent.id)
  946. tree.focus(first.id)
  947. tree.complete(
  948. first.id,
  949. "done",
  950. auto_advance=False,
  951. cascade_parent=False,
  952. )
  953. self.assertEqual(parent.id, tree.current_id)
  954. self.assertEqual("pending", second.status)
  955. self.assertNotEqual("completed", parent.status)
  956. async def test_rewind_before_review_restores_report_and_review_gate(self):
  957. delegated = await agent(task_brief=BRIEF, context=self.context())
  958. child_id = delegated["sub_trace_id"]
  959. reviewed = await review_task_result(
  960. child_trace_id=child_id,
  961. decision="ASCEND",
  962. reason="Initially accepted before rewind",
  963. context={**self.context(), "sequence": 20},
  964. )
  965. self.assertEqual("completed", reviewed["status"])
  966. for sequence in (1, 15):
  967. await self.store.add_message(Message.create(
  968. trace_id=self.root_id,
  969. role="user",
  970. sequence=sequence,
  971. content=f"message-{sequence}",
  972. ))
  973. tree = await self.store.get_goal_tree(self.root_id)
  974. runner = AgentRunner(
  975. trace_store=self.store,
  976. tool_registry=get_tool_registry(),
  977. )
  978. await runner._rewind(self.root_id, 15, tree)
  979. parent = await self.store.get_trace(self.root_id)
  980. state = ensure_task_protocol(parent.context)
  981. self.assertIn(child_id, state["pending_reviews"])
  982. self.assertEqual([], state["reviews"])
  983. restored_tree = await self.store.get_goal_tree(self.root_id)
  984. self.assertEqual("pending_review", restored_tree.find("1").status)
  985. schemas = runner._get_runtime_tool_schemas(RunConfig(), parent)
  986. self.assertEqual(
  987. {"review_task_result", "read_context_ref"},
  988. {schema["function"]["name"] for schema in schemas},
  989. )
  990. async def test_grandchild_report_is_reviewed_only_by_direct_child(self):
  991. child_id = "manual-child"
  992. child_context = with_root_anchor({
  993. "agent_mode": "recursive",
  994. "agent_mode_revision": 3,
  995. "agent_depth": 1,
  996. "root_trace_id": self.root_id,
  997. "created_by_tool": "agent",
  998. "task_protocol": new_task_protocol(BRIEF),
  999. })
  1000. await self.store.create_trace(Trace(
  1001. trace_id=child_id,
  1002. mode="agent",
  1003. task="child",
  1004. parent_trace_id=self.root_id,
  1005. uid="user-1",
  1006. model="fake-model",
  1007. context=child_context,
  1008. ))
  1009. child_tree = GoalTree(
  1010. mission="child",
  1011. goals=[Goal(id="1", description="delegate grandchild", status="in_progress")],
  1012. current_id="1",
  1013. )
  1014. await self.store.update_goal_tree(child_id, child_tree)
  1015. grandchild = await agent(
  1016. task_brief=BRIEF,
  1017. context=self.context(trace_id=child_id, tree=child_tree),
  1018. )
  1019. child = await self.store.get_trace(child_id)
  1020. child_state = ensure_task_protocol(child.context)
  1021. self.assertIn(grandchild["sub_trace_id"], child_state["pending_reviews"])
  1022. root = await self.store.get_trace(self.root_id)
  1023. self.assertEqual({}, ensure_task_protocol(root.context)["pending_reviews"])
  1024. blocked = await agent(
  1025. task_brief=BRIEF,
  1026. context=self.context(trace_id=child_id, tree=child_tree),
  1027. )
  1028. self.assertEqual("failed", blocked["status"])
  1029. reviewed = await review_task_result(
  1030. child_trace_id=grandchild["sub_trace_id"],
  1031. decision="ASCEND",
  1032. reason="Grandchild satisfied the child criteria",
  1033. context=self.context(trace_id=child_id, tree=child_tree),
  1034. )
  1035. self.assertEqual("completed", reviewed["status"])
  1036. submitted = await submit_task_report(
  1037. task_report=json.dumps(report_payload()),
  1038. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  1039. )
  1040. self.assertEqual("completed", submitted["status"])
  1041. async def test_child_must_execute_approved_action_before_own_report(self):
  1042. child_id = "child-with-approved-action"
  1043. child_context = with_root_anchor({
  1044. "agent_mode": "recursive",
  1045. "agent_mode_revision": 3,
  1046. "agent_depth": 1,
  1047. "root_trace_id": self.root_id,
  1048. "created_by_tool": "agent",
  1049. "task_protocol": new_task_protocol(BRIEF),
  1050. })
  1051. await self.store.create_trace(Trace(
  1052. trace_id=child_id,
  1053. mode="agent",
  1054. task="child",
  1055. parent_trace_id=self.root_id,
  1056. uid="user-1",
  1057. model="fake-model",
  1058. context=child_context,
  1059. ))
  1060. child_tree = GoalTree(
  1061. mission="child",
  1062. goals=[Goal(id="1", description="delegate", status="in_progress")],
  1063. current_id="1",
  1064. )
  1065. await self.store.update_goal_tree(child_id, child_tree)
  1066. child_tool_context = self.context(trace_id=child_id, tree=child_tree)
  1067. first = await agent(task_brief=BRIEF, context=child_tool_context)
  1068. next_brief = {
  1069. "objective": "check a second grandchild task",
  1070. "reason": "the child approved another direct task",
  1071. "completion_criteria": ["provide a second checked conclusion"],
  1072. "expected_outputs": ["a second checked conclusion"],
  1073. }
  1074. reviewed = await review_task_result(
  1075. child_trace_id=first["sub_trace_id"],
  1076. decision="ACCEPT_REFINE",
  1077. reason="A separate gap remains",
  1078. approved_next_task=next_brief,
  1079. context=child_tool_context,
  1080. )
  1081. self.assertEqual("completed", reviewed["status"])
  1082. blocked_report = await submit_task_report(
  1083. task_report=report_payload(),
  1084. context={"store": self.store, "trace_id": child_id, "sequence": 20},
  1085. )
  1086. self.assertEqual("failed", blocked_report["status"])
  1087. self.assertIn("approved next action", blocked_report["error"])
  1088. second = await agent(
  1089. task_brief=reviewed["next_action_task_brief"],
  1090. context=child_tool_context,
  1091. )
  1092. self.assertEqual("completed", second["status"])
  1093. child = await self.store.get_trace(child_id)
  1094. state = ensure_task_protocol(child.context)
  1095. self.assertEqual([], state["next_actions"])
  1096. self.assertIn(second["sub_trace_id"], state["pending_reviews"])
  1097. async def test_goal_tool_cannot_bypass_queued_next_action(self):
  1098. root = await self.store.get_trace(self.root_id)
  1099. state = ensure_task_protocol(root.context)
  1100. state["next_actions"] = [{
  1101. "decision": "ACCEPT_REFINE",
  1102. "child_trace_id": "reviewed-child",
  1103. "goal_id": "1",
  1104. "task_brief": BRIEF,
  1105. "created_at_sequence": 5,
  1106. }]
  1107. await self.store.update_trace(self.root_id, context=root.context)
  1108. result = await goal_tool(
  1109. tree=self.tree,
  1110. store=self.store,
  1111. trace_id=self.root_id,
  1112. done="attempt to bypass the approved action",
  1113. )
  1114. self.assertIn("已批准的下一步待执行", result)
  1115. persisted_tree = await self.store.get_goal_tree(self.root_id)
  1116. self.assertEqual("in_progress", persisted_tree.find("1").status)
  1117. async def test_failed_report_allows_only_revise_or_fail(self):
  1118. child_id = "failed-child"
  1119. failed_report = TaskReport(
  1120. child_trace_id=child_id,
  1121. **report_payload(outcome="failed"),
  1122. )
  1123. child_context = with_root_anchor({
  1124. "agent_mode": "recursive",
  1125. "agent_mode_revision": 3,
  1126. "agent_depth": 1,
  1127. "root_trace_id": self.root_id,
  1128. "created_by_tool": "agent",
  1129. "task_protocol": new_task_protocol(BRIEF),
  1130. })
  1131. child_context["task_protocol"]["task_report"] = failed_report.model_dump()
  1132. failed_validation = validation_payload(child_id, outcome="failed")
  1133. child_context["task_protocol"]["task_report_validation"] = {
  1134. "validation_plan": None,
  1135. "plan_hash": failed_validation["plan_hash"],
  1136. "scope_results": failed_validation["scope_results"],
  1137. "aggregate_result": failed_validation,
  1138. "validated_at_sequence": 2,
  1139. }
  1140. await self.store.create_trace(Trace(
  1141. trace_id=child_id,
  1142. mode="agent",
  1143. task="failed",
  1144. parent_trace_id=self.root_id,
  1145. parent_goal_id="1",
  1146. uid="user-1",
  1147. context=child_context,
  1148. ))
  1149. root = await self.store.get_trace(self.root_id)
  1150. state = ensure_task_protocol(root.context)
  1151. state["pending_reviews"][child_id] = {
  1152. "goal_id": "1",
  1153. "task_report": failed_report.model_dump(),
  1154. "validation_result": failed_validation,
  1155. "received_at_sequence": 2,
  1156. }
  1157. await self.store.update_trace(self.root_id, context=root.context)
  1158. rejected = await review_task_result(
  1159. child_trace_id=child_id,
  1160. decision="ASCEND",
  1161. reason="incorrect",
  1162. context=self.context(),
  1163. )
  1164. self.assertEqual("failed", rejected["status"])
  1165. self.assertIn("only allow", rejected["error"])
  1166. class RunnerCompletionGateTest(unittest.IsolatedAsyncioTestCase):
  1167. async def asyncSetUp(self):
  1168. self.temp_dir = tempfile.TemporaryDirectory()
  1169. self.store = FileSystemTraceStore(self.temp_dir.name)
  1170. self.trace_id = "gate-child"
  1171. await create_recursive_root(self.store, "root")
  1172. controller = ResourceBudgetController(self.store)
  1173. budget = ResourceBudget()
  1174. await controller.reserve_agents("root", budget, 1)
  1175. await self.store.create_trace(Trace(
  1176. trace_id=self.trace_id,
  1177. mode="agent",
  1178. task="must report",
  1179. parent_trace_id="root",
  1180. uid="user-1",
  1181. context=with_root_anchor({
  1182. "agent_mode": "recursive",
  1183. "agent_mode_revision": 3,
  1184. "agent_depth": 1,
  1185. "root_trace_id": "root",
  1186. "created_by_tool": "agent",
  1187. "task_protocol": new_task_protocol(BRIEF),
  1188. }),
  1189. ))
  1190. await self.store.update_goal_tree(self.trace_id, GoalTree(mission="must report"))
  1191. async def asyncTearDown(self):
  1192. self.temp_dir.cleanup()
  1193. async def bind_snapshot(self, trace_id, config):
  1194. """Make manually-created Recursive fixtures equivalent to new traces."""
  1195. trace = await self.store.get_trace(trace_id)
  1196. config.model = trace.model or config.model
  1197. config.uid = trace.uid
  1198. config.agent_type = trace.agent_type or "default"
  1199. persist_run_config_snapshot(
  1200. trace.context,
  1201. RunConfigSnapshotV1.from_run_config(
  1202. config,
  1203. memory_identity=None,
  1204. ),
  1205. )
  1206. await self.store.update_trace(trace_id, context=trace.context)
  1207. return config
  1208. async def test_missing_report_becomes_protocol_error_after_two_corrections(self):
  1209. calls = 0
  1210. async def llm_call(**kwargs):
  1211. nonlocal calls
  1212. calls += 1
  1213. return {
  1214. "content": "I am done",
  1215. "tool_calls": [],
  1216. "finish_reason": "stop",
  1217. }
  1218. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1219. config = await self.bind_snapshot(
  1220. self.trace_id,
  1221. RunConfig(
  1222. trace_id=self.trace_id,
  1223. max_iterations=8,
  1224. knowledge=KnowledgeConfig(
  1225. enable_extraction=False,
  1226. enable_completion_extraction=False,
  1227. enable_injection=False,
  1228. ),
  1229. ),
  1230. )
  1231. result = await runner.run_result(messages=[], config=config)
  1232. self.assertEqual("failed", result["status"])
  1233. trace = await self.store.get_trace(self.trace_id)
  1234. report = ensure_task_protocol(trace.context)["task_report"]
  1235. self.assertEqual("protocol_error", report["outcome"])
  1236. self.assertGreaterEqual(calls, 3)
  1237. async def test_valid_report_allows_child_to_finish(self):
  1238. calls = 0
  1239. async def llm_call(**kwargs):
  1240. nonlocal calls
  1241. calls += 1
  1242. messages = kwargs.get("messages") or []
  1243. if messages and "recursive_validation_protocol" in messages[0]["content"]:
  1244. return passed_validator_response(kwargs)
  1245. if calls == 1:
  1246. return {
  1247. "content": "",
  1248. "tool_calls": [{
  1249. "id": "submit-1",
  1250. "type": "function",
  1251. "function": {
  1252. "name": "submit_task_report",
  1253. "arguments": json.dumps({
  1254. "task_report": report_payload(),
  1255. }),
  1256. },
  1257. }],
  1258. "finish_reason": "tool_calls",
  1259. }
  1260. return {
  1261. "content": "Structured report submitted.",
  1262. "tool_calls": [],
  1263. "finish_reason": "stop",
  1264. }
  1265. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1266. config = await self.bind_snapshot(
  1267. self.trace_id,
  1268. RunConfig(
  1269. trace_id=self.trace_id,
  1270. max_iterations=4,
  1271. knowledge=KnowledgeConfig(
  1272. enable_extraction=False,
  1273. enable_completion_extraction=False,
  1274. enable_injection=False,
  1275. ),
  1276. ),
  1277. )
  1278. result = await runner.run_result(messages=[], config=config)
  1279. self.assertEqual("completed", result["status"])
  1280. trace = await self.store.get_trace(self.trace_id)
  1281. self.assertEqual(
  1282. "satisfied",
  1283. ensure_task_protocol(trace.context)["task_report"]["outcome"],
  1284. )
  1285. async def test_lifecycle_tool_batch_is_rejected_before_execution(self):
  1286. root_id = "batch-gate-root"
  1287. await create_recursive_root(self.store, root_id)
  1288. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  1289. calls = 0
  1290. async def llm_call(**kwargs):
  1291. nonlocal calls
  1292. calls += 1
  1293. messages = kwargs.get("messages") or []
  1294. if messages and "recursive_validation_protocol" in messages[0]["content"]:
  1295. return passed_validator_response(kwargs)
  1296. if calls == 1:
  1297. return {
  1298. "content": "",
  1299. "tool_calls": [
  1300. {
  1301. "id": "agent-1",
  1302. "type": "function",
  1303. "function": {
  1304. "name": "agent",
  1305. "arguments": json.dumps({
  1306. "task_brief": BRIEF,
  1307. }),
  1308. },
  1309. },
  1310. {
  1311. "id": "context-1",
  1312. "type": "function",
  1313. "function": {
  1314. "name": "get_current_context",
  1315. "arguments": "{}",
  1316. },
  1317. },
  1318. ],
  1319. "finish_reason": "tool_calls",
  1320. }
  1321. return {
  1322. "content": "Corrected the invalid batch.",
  1323. "tool_calls": [],
  1324. "finish_reason": "stop",
  1325. }
  1326. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1327. config = await self.bind_snapshot(
  1328. root_id,
  1329. RunConfig(
  1330. trace_id=root_id,
  1331. max_iterations=3,
  1332. knowledge=KnowledgeConfig(
  1333. enable_extraction=False,
  1334. enable_completion_extraction=False,
  1335. enable_injection=False,
  1336. ),
  1337. ),
  1338. )
  1339. result = await runner.run_result(messages=[], config=config)
  1340. self.assertEqual("completed", result["status"])
  1341. children = await self.store.list_traces(
  1342. parent_trace_id=root_id,
  1343. created_by_tool="agent",
  1344. limit=10,
  1345. )
  1346. self.assertEqual([], children)
  1347. tool_messages = [
  1348. message
  1349. for message in await self.store.get_trace_messages(root_id)
  1350. if message.role == "tool"
  1351. ]
  1352. self.assertEqual(2, len(tool_messages))
  1353. self.assertTrue(all(
  1354. "only tool call" in str(message.content)
  1355. for message in tool_messages
  1356. ))
  1357. async def test_rewind_clears_protocol_state_after_cutoff(self):
  1358. trace = await self.store.get_trace(self.trace_id)
  1359. state = ensure_task_protocol(trace.context)
  1360. original_brief = TaskBrief.model_validate(state["task_brief"])
  1361. revised_brief = original_brief.model_copy(
  1362. update={"objective": "revised after the rewind cutoff"}
  1363. )
  1364. replace_task_brief(
  1365. state,
  1366. revised_brief,
  1367. effective_at_sequence=8,
  1368. )
  1369. late_snapshot = create_context_snapshot(
  1370. kind="reviewed_task_result",
  1371. summary="granted after cutoff",
  1372. source_trace_id="late-child",
  1373. root_trace_id="root",
  1374. uid="user-1",
  1375. content={"result": "must be removed by rewind"},
  1376. granted_at_sequence=8,
  1377. )
  1378. add_context_snapshot(
  1379. trace.context,
  1380. late_snapshot,
  1381. root_task_anchor=ROOT_ANCHOR,
  1382. task_brief=revised_brief,
  1383. )
  1384. report = TaskReport(
  1385. child_trace_id=self.trace_id,
  1386. **report_payload(),
  1387. ).model_dump()
  1388. state["task_report"] = report
  1389. state["task_report_submitted_at_sequence"] = 5
  1390. state["pending_reviews"] = {
  1391. "grandchild": {
  1392. "goal_id": "1",
  1393. "task_report": report,
  1394. "validation_result": validation_payload("grandchild"),
  1395. "received_at_sequence": 6,
  1396. }
  1397. }
  1398. state["reviews"] = [{"reviewed_at_sequence": 7}]
  1399. state["next_actions"] = [{"created_at_sequence": 8}]
  1400. await self.store.update_trace(self.trace_id, context=trace.context)
  1401. tree = GoalTree(
  1402. mission="rewind",
  1403. goals=[Goal(id="1", description="active", status="pending_review")],
  1404. current_id="1",
  1405. )
  1406. await self.store.update_goal_tree(self.trace_id, tree)
  1407. for sequence in (1, 2):
  1408. await self.store.add_message(Message.create(
  1409. trace_id=self.trace_id,
  1410. role="user",
  1411. sequence=sequence,
  1412. content=f"message-{sequence}",
  1413. ))
  1414. runner = AgentRunner(trace_store=self.store)
  1415. cutoff = await runner._rewind(self.trace_id, 2, tree)
  1416. self.assertEqual(3, cutoff)
  1417. rewound = await self.store.get_trace(self.trace_id)
  1418. state = ensure_task_protocol(rewound.context)
  1419. self.assertIsNone(state["task_report"])
  1420. self.assertEqual(1, len(state["report_history"]))
  1421. self.assertEqual({}, state["pending_reviews"])
  1422. self.assertEqual([], state["reviews"])
  1423. self.assertEqual([], state["next_actions"])
  1424. self.assertEqual(1, state["task_brief_version"])
  1425. self.assertEqual(original_brief.model_dump(), state["task_brief"])
  1426. self.assertEqual([], state["task_brief_history"])
  1427. self.assertEqual({}, rewound.context["context_access"]["snapshots"])
  1428. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  1429. self.assertEqual("pending", rewound_tree.find("1").status)
  1430. async def test_rewind_restores_preserved_pending_review_projection(self):
  1431. trace = await self.store.get_trace(self.trace_id)
  1432. state = ensure_task_protocol(trace.context)
  1433. report = TaskReport(
  1434. child_trace_id="preserved-grandchild",
  1435. **report_payload(),
  1436. ).model_dump()
  1437. state["pending_reviews"] = {
  1438. "preserved-grandchild": {
  1439. "goal_id": "1",
  1440. "task_report": report,
  1441. "validation_result": validation_payload(
  1442. "preserved-grandchild"
  1443. ),
  1444. "received_at_sequence": 2,
  1445. }
  1446. }
  1447. await self.store.update_trace(self.trace_id, context=trace.context)
  1448. tree = GoalTree(
  1449. mission="rewind",
  1450. goals=[Goal(id="1", description="review", status="pending_review")],
  1451. current_id="1",
  1452. )
  1453. await self.store.update_goal_tree(self.trace_id, tree)
  1454. for sequence in (1, 2):
  1455. await self.store.add_message(Message.create(
  1456. trace_id=self.trace_id,
  1457. role="user",
  1458. sequence=sequence,
  1459. content=f"message-{sequence}",
  1460. ))
  1461. runner = AgentRunner(trace_store=self.store)
  1462. await runner._rewind(self.trace_id, 2, tree)
  1463. rewound = await self.store.get_trace(self.trace_id)
  1464. self.assertIn(
  1465. "preserved-grandchild",
  1466. ensure_task_protocol(rewound.context)["pending_reviews"],
  1467. )
  1468. rewound_tree = await self.store.get_goal_tree(self.trace_id)
  1469. self.assertEqual("pending_review", rewound_tree.find("1").status)
  1470. async def test_rewind_rebuilds_root_validation_history_and_cache(self):
  1471. root_id = "root-validation-rewind"
  1472. state = new_task_protocol()
  1473. passed = validation_payload(root_id, outcome="passed")
  1474. failed = validation_payload(root_id, outcome="failed")
  1475. state["root_validation_history"] = [
  1476. {**passed, "validated_at_sequence": 2},
  1477. {**failed, "validated_at_sequence": 5},
  1478. ]
  1479. state["root_validation_attempts"] = 2
  1480. state["root_validation_passed"] = False
  1481. state["task_report_validation"] = {
  1482. "plan_hash": failed["plan_hash"],
  1483. "aggregate_result": failed,
  1484. "scope_results": failed["scope_results"],
  1485. "validated_at_sequence": 5,
  1486. }
  1487. await create_recursive_root(self.store, root_id, state=state)
  1488. tree = GoalTree(
  1489. mission="root",
  1490. goals=[Goal(id="1", description="root", status="in_progress")],
  1491. current_id="1",
  1492. )
  1493. await self.store.update_goal_tree(root_id, tree)
  1494. for sequence in (1, 2, 3):
  1495. await self.store.add_message(Message.create(
  1496. trace_id=root_id,
  1497. role="user",
  1498. sequence=sequence,
  1499. content=f"message-{sequence}",
  1500. ))
  1501. await AgentRunner(trace_store=self.store)._rewind(root_id, 3, tree)
  1502. rewound = await self.store.get_trace(root_id)
  1503. state = ensure_task_protocol(rewound.context)
  1504. self.assertEqual(1, state["root_validation_attempts"])
  1505. self.assertTrue(state["root_validation_passed"])
  1506. self.assertEqual(
  1507. ["passed"],
  1508. [item["outcome"] for item in state["root_validation_history"]],
  1509. )
  1510. self.assertIsNone(state["task_report_validation"])
  1511. async def test_root_cannot_finish_with_unconsumed_next_action(self):
  1512. root_id = "next-action-gate-root"
  1513. state = new_task_protocol()
  1514. state["next_actions"] = [{
  1515. "decision": "ACCEPT_REFINE",
  1516. "child_trace_id": "reviewed-child",
  1517. "goal_id": None,
  1518. "task_brief": BRIEF,
  1519. "created_at_sequence": 1,
  1520. }]
  1521. await create_recursive_root(self.store, root_id, state=state)
  1522. await self.store.update_goal_tree(root_id, GoalTree(mission="root"))
  1523. calls = 0
  1524. async def llm_call(**kwargs):
  1525. nonlocal calls
  1526. calls += 1
  1527. return {
  1528. "content": "Trying to finish without the approved action.",
  1529. "tool_calls": [],
  1530. "finish_reason": "stop",
  1531. }
  1532. runner = AgentRunner(trace_store=self.store, llm_call=llm_call)
  1533. config = await self.bind_snapshot(
  1534. root_id,
  1535. RunConfig(
  1536. trace_id=root_id,
  1537. max_iterations=5,
  1538. knowledge=KnowledgeConfig(
  1539. enable_extraction=False,
  1540. enable_completion_extraction=False,
  1541. enable_injection=False,
  1542. ),
  1543. ),
  1544. )
  1545. result = await runner.run_result(messages=[], config=config)
  1546. self.assertEqual("failed", result["status"])
  1547. self.assertEqual(3, calls)
  1548. root = await self.store.get_trace(root_id)
  1549. self.assertEqual(1, len(ensure_task_protocol(root.context)["next_actions"]))
  1550. class RuntimeProtocolSchemaTest(unittest.TestCase):
  1551. def setUp(self):
  1552. self.runner = AgentRunner(tool_registry=get_tool_registry())
  1553. def names(self, schemas):
  1554. return {schema["function"]["name"] for schema in schemas}
  1555. def test_legacy_and_recursive_runtime_schemas_are_isolated(self):
  1556. legacy = Trace(
  1557. trace_id="legacy",
  1558. mode="agent",
  1559. context={"agent_mode": "legacy", "agent_mode_revision": 1},
  1560. )
  1561. legacy_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), legacy)
  1562. self.assertNotIn("submit_task_report", self.names(legacy_schemas))
  1563. self.assertNotIn("review_task_result", self.names(legacy_schemas))
  1564. legacy_agent = next(
  1565. item for item in legacy_schemas if item["function"]["name"] == "agent"
  1566. )
  1567. properties = legacy_agent["function"]["parameters"]["properties"]
  1568. self.assertIn("task", properties)
  1569. self.assertNotIn("task_brief", properties)
  1570. child = Trace(
  1571. trace_id="child",
  1572. mode="agent",
  1573. parent_trace_id="root",
  1574. context={
  1575. "agent_mode": "recursive",
  1576. "agent_mode_revision": 3,
  1577. "task_protocol": new_task_protocol(BRIEF),
  1578. },
  1579. )
  1580. child_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1581. self.assertIn("submit_task_report", self.names(child_schemas))
  1582. self.assertNotIn("review_task_result", self.names(child_schemas))
  1583. self.assertNotIn("evaluate", self.names(child_schemas))
  1584. recursive_agent = next(
  1585. item for item in child_schemas if item["function"]["name"] == "agent"
  1586. )
  1587. recursive_properties = recursive_agent["function"]["parameters"]["properties"]
  1588. self.assertNotIn("messages", recursive_properties)
  1589. brief_object = recursive_properties["task_brief"]["oneOf"][0]
  1590. self.assertEqual(
  1591. {
  1592. "objective",
  1593. "reason",
  1594. "completion_criteria",
  1595. "expected_outputs",
  1596. },
  1597. set(brief_object["required"]),
  1598. )
  1599. state = ensure_task_protocol(child.context)
  1600. state["pending_reviews"]["grandchild"] = {"task_report": {}}
  1601. review_schemas = self.runner._get_runtime_tool_schemas(RunConfig(), child)
  1602. self.assertEqual(
  1603. {"review_task_result", "read_context_ref"},
  1604. self.names(review_schemas),
  1605. )
  1606. def test_protocol_tools_publish_nested_object_schemas(self):
  1607. schemas = {
  1608. schema["function"]["name"]: schema
  1609. for schema in get_tool_registry().get_schemas([
  1610. "submit_task_report",
  1611. "review_task_result",
  1612. ])
  1613. }
  1614. report_schema = schemas["submit_task_report"]["function"]["parameters"]
  1615. task_report = report_schema["properties"]["task_report"]
  1616. self.assertEqual("object", task_report["type"])
  1617. self.assertIn("validation", task_report["properties"])
  1618. self.assertIn("next_step_suggestion", task_report["properties"])
  1619. review_schema = schemas["review_task_result"]["function"]["parameters"]
  1620. self.assertEqual(
  1621. [
  1622. "ACCEPT_REFINE",
  1623. "REJECT_REFINE",
  1624. "REVISE_CHILD",
  1625. "DESCEND_AGAIN",
  1626. "REPLAN_CURRENT",
  1627. "ASCEND",
  1628. "FAIL",
  1629. ],
  1630. review_schema["properties"]["decision"]["enum"],
  1631. )
  1632. self.assertEqual(
  1633. "object",
  1634. review_schema["properties"]["approved_next_task"]["type"],
  1635. )
  1636. if __name__ == "__main__":
  1637. unittest.main()