|
@@ -1,5 +1,4 @@
|
|
|
import asyncio
|
|
import asyncio
|
|
|
-import os
|
|
|
|
|
import tempfile
|
|
import tempfile
|
|
|
import unittest
|
|
import unittest
|
|
|
from unittest.mock import AsyncMock, patch
|
|
from unittest.mock import AsyncMock, patch
|
|
@@ -49,30 +48,40 @@ class FakeRunner:
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
-class SubAgentRecursionTest(unittest.IsolatedAsyncioTestCase):
|
|
|
|
|
|
|
+class SubAgentModeTest(unittest.IsolatedAsyncioTestCase):
|
|
|
async def asyncSetUp(self):
|
|
async def asyncSetUp(self):
|
|
|
self.temp_dir = tempfile.TemporaryDirectory()
|
|
self.temp_dir = tempfile.TemporaryDirectory()
|
|
|
self.store = FileSystemTraceStore(self.temp_dir.name)
|
|
self.store = FileSystemTraceStore(self.temp_dir.name)
|
|
|
self.runner = FakeRunner(self.store)
|
|
self.runner = FakeRunner(self.store)
|
|
|
- self.previous_recursion_setting = os.environ.get("AGENT_RECURSION_ENABLED")
|
|
|
|
|
- os.environ["AGENT_RECURSION_ENABLED"] = "true"
|
|
|
|
|
self.root_id = await self._create_root("root")
|
|
self.root_id = await self._create_root("root")
|
|
|
|
|
|
|
|
async def asyncTearDown(self):
|
|
async def asyncTearDown(self):
|
|
|
- if self.previous_recursion_setting is None:
|
|
|
|
|
- os.environ.pop("AGENT_RECURSION_ENABLED", None)
|
|
|
|
|
- else:
|
|
|
|
|
- os.environ["AGENT_RECURSION_ENABLED"] = self.previous_recursion_setting
|
|
|
|
|
self.temp_dir.cleanup()
|
|
self.temp_dir.cleanup()
|
|
|
|
|
|
|
|
- async def _create_root(self, trace_id, uid="user-1"):
|
|
|
|
|
|
|
+ async def _create_root(
|
|
|
|
|
+ self,
|
|
|
|
|
+ trace_id,
|
|
|
|
|
+ uid="user-1",
|
|
|
|
|
+ *,
|
|
|
|
|
+ agent_mode="recursive",
|
|
|
|
|
+ include_mode=True,
|
|
|
|
|
+ ):
|
|
|
|
|
+ context = {
|
|
|
|
|
+ "agent_depth": 0,
|
|
|
|
|
+ "root_trace_id": trace_id,
|
|
|
|
|
+ }
|
|
|
|
|
+ if include_mode:
|
|
|
|
|
+ context.update({
|
|
|
|
|
+ "agent_mode": agent_mode,
|
|
|
|
|
+ "agent_mode_revision": 1,
|
|
|
|
|
+ })
|
|
|
trace = Trace(
|
|
trace = Trace(
|
|
|
trace_id=trace_id,
|
|
trace_id=trace_id,
|
|
|
mode="agent",
|
|
mode="agent",
|
|
|
task=f"task-{trace_id}",
|
|
task=f"task-{trace_id}",
|
|
|
uid=uid,
|
|
uid=uid,
|
|
|
model="fake-model",
|
|
model="fake-model",
|
|
|
- context={},
|
|
|
|
|
|
|
+ context=context,
|
|
|
)
|
|
)
|
|
|
await self.store.create_trace(trace)
|
|
await self.store.create_trace(trace)
|
|
|
await self.store.update_goal_tree(trace_id, GoalTree(mission=trace.task))
|
|
await self.store.update_goal_tree(trace_id, GoalTree(mission=trace.task))
|
|
@@ -97,9 +106,9 @@ class SubAgentRecursionTest(unittest.IsolatedAsyncioTestCase):
|
|
|
limit=20,
|
|
limit=20,
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
- async def test_recursion_disabled_preserves_single_level_behavior(self):
|
|
|
|
|
- os.environ["AGENT_RECURSION_ENABLED"] = "false"
|
|
|
|
|
- first = await self._spawn_one(self.root_id)
|
|
|
|
|
|
|
+ async def test_legacy_preserves_single_level_behavior(self):
|
|
|
|
|
+ legacy_root = await self._create_root("legacy-root", agent_mode="legacy")
|
|
|
|
|
+ first = await self._spawn_one(legacy_root)
|
|
|
self.assertEqual("completed", first["status"])
|
|
self.assertEqual("completed", first["status"])
|
|
|
|
|
|
|
|
child_id = first["sub_trace_id"]
|
|
child_id = first["sub_trace_id"]
|
|
@@ -108,7 +117,17 @@ class SubAgentRecursionTest(unittest.IsolatedAsyncioTestCase):
|
|
|
|
|
|
|
|
blocked = await self._spawn_one(child_id, "grandchild")
|
|
blocked = await self._spawn_one(child_id, "grandchild")
|
|
|
self.assertEqual("failed", blocked["status"])
|
|
self.assertEqual("failed", blocked["status"])
|
|
|
- self.assertIn("recursion is disabled", blocked["error"])
|
|
|
|
|
|
|
+ self.assertIn("depth limit reached", blocked["error"])
|
|
|
|
|
+ self.assertIn("mode=legacy", blocked["error"])
|
|
|
|
|
+
|
|
|
|
|
+ async def test_legacy_has_no_direct_child_quota(self):
|
|
|
|
|
+ legacy_root = await self._create_root("legacy-unlimited", agent_mode="legacy")
|
|
|
|
|
+ result = await agent(
|
|
|
|
|
+ task=[f"legacy-child-{index}" for index in range(8)],
|
|
|
|
|
+ context=self._context(legacy_root),
|
|
|
|
|
+ )
|
|
|
|
|
+ self.assertEqual("completed", result["status"])
|
|
|
|
|
+ self.assertEqual(8, len(await self._local_children(legacy_root)))
|
|
|
|
|
|
|
|
async def test_five_subagent_levels_and_sixth_level_rejected(self):
|
|
async def test_five_subagent_levels_and_sixth_level_rejected(self):
|
|
|
parent_id = self.root_id
|
|
parent_id = self.root_id
|
|
@@ -181,6 +200,24 @@ class SubAgentRecursionTest(unittest.IsolatedAsyncioTestCase):
|
|
|
self.assertEqual(2, len(sub_trace_ids))
|
|
self.assertEqual(2, len(sub_trace_ids))
|
|
|
self.assertEqual(2, len({entry["trace_id"] for entry in sub_trace_ids}))
|
|
self.assertEqual(2, len({entry["trace_id"] for entry in sub_trace_ids}))
|
|
|
|
|
|
|
|
|
|
+ async def test_legacy_goal_overwrites_sub_traces_with_latest_call(self):
|
|
|
|
|
+ legacy_root = await self._create_root("legacy-goal", agent_mode="legacy")
|
|
|
|
|
+ tree = await self.store.get_goal_tree(legacy_root)
|
|
|
|
|
+ tree.goals = [Goal(id="1", description="delegate twice")]
|
|
|
|
|
+ await self.store.update_goal_tree(legacy_root, tree)
|
|
|
|
|
+ context = self._context(legacy_root, goal_id="1")
|
|
|
|
|
+
|
|
|
|
|
+ first = await agent(task="first", context=context)
|
|
|
|
|
+ second = await agent(task="second", context=context)
|
|
|
|
|
+
|
|
|
|
|
+ updated_tree = await self.store.get_goal_tree(legacy_root)
|
|
|
|
|
+ sub_trace_ids = updated_tree.find("1").sub_trace_ids
|
|
|
|
|
+ self.assertEqual(
|
|
|
|
|
+ [{"trace_id": second["sub_trace_id"], "mission": "second"}],
|
|
|
|
|
+ sub_trace_ids,
|
|
|
|
|
+ )
|
|
|
|
|
+ self.assertNotEqual(first["sub_trace_id"], second["sub_trace_id"])
|
|
|
|
|
+
|
|
|
async def test_concurrent_batches_never_exceed_limit(self):
|
|
async def test_concurrent_batches_never_exceed_limit(self):
|
|
|
results = await asyncio.gather(
|
|
results = await asyncio.gather(
|
|
|
agent(
|
|
agent(
|
|
@@ -231,6 +268,56 @@ class SubAgentRecursionTest(unittest.IsolatedAsyncioTestCase):
|
|
|
self.assertEqual("failed", wrong_owner["status"])
|
|
self.assertEqual("failed", wrong_owner["status"])
|
|
|
self.assertIn("owner", wrong_owner["error"])
|
|
self.assertIn("owner", wrong_owner["error"])
|
|
|
|
|
|
|
|
|
|
+ async def test_continue_from_requires_matching_persisted_mode(self):
|
|
|
|
|
+ created = await self._spawn_one(self.root_id)
|
|
|
|
|
+ child_id = created["sub_trace_id"]
|
|
|
|
|
+ child = await self.store.get_trace(child_id)
|
|
|
|
|
+ child.context["agent_mode"] = "legacy"
|
|
|
|
|
+ await self.store.update_trace(child_id, context=child.context)
|
|
|
|
|
+
|
|
|
|
|
+ result = await agent(
|
|
|
|
|
+ task="continue",
|
|
|
|
|
+ continue_from=child_id,
|
|
|
|
|
+ context=self._context(self.root_id),
|
|
|
|
|
+ )
|
|
|
|
|
+ self.assertEqual("failed", result["status"])
|
|
|
|
|
+ self.assertIn("mode does not match", result["error"])
|
|
|
|
|
+
|
|
|
|
|
+ async def test_continue_from_rejects_cross_tool_traces(self):
|
|
|
|
|
+ evaluated = await evaluate(messages=[], context=self._context(self.root_id))
|
|
|
|
|
+ rejected_agent = await agent(
|
|
|
|
|
+ task="wrong tool",
|
|
|
|
|
+ continue_from=evaluated["sub_trace_id"],
|
|
|
|
|
+ context=self._context(self.root_id),
|
|
|
|
|
+ )
|
|
|
|
|
+ self.assertEqual("failed", rejected_agent["status"])
|
|
|
|
|
+ self.assertIn("created by the agent tool", rejected_agent["error"])
|
|
|
|
|
+
|
|
|
|
|
+ created = await self._spawn_one(self.root_id)
|
|
|
|
|
+ rejected_evaluator = await evaluate(
|
|
|
|
|
+ messages=[],
|
|
|
|
|
+ continue_from=created["sub_trace_id"],
|
|
|
|
|
+ context=self._context(self.root_id),
|
|
|
|
|
+ )
|
|
|
|
|
+ self.assertEqual("failed", rejected_evaluator["status"])
|
|
|
|
|
+ self.assertIn("created by evaluate", rejected_evaluator["error"])
|
|
|
|
|
+
|
|
|
|
|
+ async def test_continue_from_rejects_forged_depth_or_root_lineage(self):
|
|
|
|
|
+ created = await self._spawn_one(self.root_id)
|
|
|
|
|
+ child_id = created["sub_trace_id"]
|
|
|
|
|
+ child = await self.store.get_trace(child_id)
|
|
|
|
|
+ child.context["agent_depth"] = 0
|
|
|
|
|
+ child.context["root_trace_id"] = "wrong-root"
|
|
|
|
|
+ await self.store.update_trace(child_id, context=child.context)
|
|
|
|
|
+
|
|
|
|
|
+ result = await agent(
|
|
|
|
|
+ task="continue forged child",
|
|
|
|
|
+ continue_from=child_id,
|
|
|
|
|
+ context=self._context(self.root_id),
|
|
|
|
|
+ )
|
|
|
|
|
+ self.assertEqual("failed", result["status"])
|
|
|
|
|
+ self.assertIn("lineage does not match", result["error"])
|
|
|
|
|
+
|
|
|
async def test_evaluator_never_receives_recursive_or_shell_tools(self):
|
|
async def test_evaluator_never_receives_recursive_or_shell_tools(self):
|
|
|
await evaluate(messages=[], context=self._context(self.root_id))
|
|
await evaluate(messages=[], context=self._context(self.root_id))
|
|
|
config = self.runner.configs[-1]
|
|
config = self.runner.configs[-1]
|
|
@@ -252,29 +339,16 @@ class SubAgentRecursionTest(unittest.IsolatedAsyncioTestCase):
|
|
|
self.assertEqual("completed", result["status"])
|
|
self.assertEqual("completed", result["status"])
|
|
|
self.assertEqual(6, len(await self._local_children(self.root_id)))
|
|
self.assertEqual(6, len(await self._local_children(self.root_id)))
|
|
|
|
|
|
|
|
- async def test_legacy_trace_depth_is_recovered_from_parent_links(self):
|
|
|
|
|
- legacy_child_id = "legacy-child"
|
|
|
|
|
- await self.store.create_trace(
|
|
|
|
|
- Trace(
|
|
|
|
|
- trace_id=legacy_child_id,
|
|
|
|
|
- mode="agent",
|
|
|
|
|
- task="legacy child",
|
|
|
|
|
- parent_trace_id=self.root_id,
|
|
|
|
|
- uid="user-1",
|
|
|
|
|
- model="fake-model",
|
|
|
|
|
- context={"created_by_tool": "agent"},
|
|
|
|
|
- )
|
|
|
|
|
- )
|
|
|
|
|
- await self.store.update_goal_tree(
|
|
|
|
|
- legacy_child_id,
|
|
|
|
|
- GoalTree(mission="legacy child"),
|
|
|
|
|
|
|
+ async def test_trace_without_mode_is_treated_as_legacy(self):
|
|
|
|
|
+ historical_root = await self._create_root(
|
|
|
|
|
+ "historical-root",
|
|
|
|
|
+ include_mode=False,
|
|
|
)
|
|
)
|
|
|
-
|
|
|
|
|
- result = await self._spawn_one(legacy_child_id, "grandchild")
|
|
|
|
|
- self.assertEqual("completed", result["status"])
|
|
|
|
|
- grandchild = await self.store.get_trace(result["sub_trace_id"])
|
|
|
|
|
- self.assertEqual(2, grandchild.context["agent_depth"])
|
|
|
|
|
- self.assertEqual(self.root_id, grandchild.context["root_trace_id"])
|
|
|
|
|
|
|
+ child = await self._spawn_one(historical_root, "legacy child")
|
|
|
|
|
+ self.assertEqual("completed", child["status"])
|
|
|
|
|
+ child_trace = await self.store.get_trace(child["sub_trace_id"])
|
|
|
|
|
+ self.assertEqual("legacy", child_trace.context["agent_mode"])
|
|
|
|
|
+ self.assertNotIn("agent", self.runner.configs[-1].tools)
|
|
|
|
|
|
|
|
async def test_remote_agent_is_not_counted_as_local_child(self):
|
|
async def test_remote_agent_is_not_counted_as_local_child(self):
|
|
|
child = await self._spawn_one(self.root_id)
|
|
child = await self._spawn_one(self.root_id)
|