|
|
@@ -0,0 +1,401 @@
|
|
|
+import asyncio
|
|
|
+from collections import deque
|
|
|
+
|
|
|
+import pytest
|
|
|
+
|
|
|
+from agent.orchestration.config import OrchestrationConfig
|
|
|
+from agent.orchestration.coordinator import TaskConflict, TaskCoordinator
|
|
|
+from agent.orchestration.models import (
|
|
|
+ AgentRole,
|
|
|
+ ArtifactRef,
|
|
|
+ AttemptSubmission,
|
|
|
+ CriterionResult,
|
|
|
+ DecisionAction,
|
|
|
+ TaskStatus,
|
|
|
+ ValidationVerdict,
|
|
|
+)
|
|
|
+from agent.orchestration.protocols import ValidatorRunResult, WorkerRunResult
|
|
|
+from agent.orchestration.store import FileSystemArtifactStore, FileSystemTaskStore, TraceEventSink
|
|
|
+from agent.trace.goal_models import GoalTree
|
|
|
+from agent.trace.models import Trace
|
|
|
+from agent.trace.store import FileSystemTraceStore
|
|
|
+from agent.core.runner import AgentRunner
|
|
|
+from agent.orchestration.wiring import wire_orchestration
|
|
|
+
|
|
|
+
|
|
|
+class FakeExecutor:
|
|
|
+ def __init__(self, verdicts, submit_worker=True, submit_validator=True, delay=0):
|
|
|
+ self.verdicts = deque(verdicts)
|
|
|
+ self.submit_worker = submit_worker
|
|
|
+ self.submit_validator = submit_validator
|
|
|
+ self.delay = delay
|
|
|
+ self.coordinator = None
|
|
|
+ self.active = 0
|
|
|
+ self.max_active = 0
|
|
|
+
|
|
|
+ async def run_worker(self, context):
|
|
|
+ self.active += 1
|
|
|
+ self.max_active = max(self.max_active, self.active)
|
|
|
+ if self.delay:
|
|
|
+ await asyncio.sleep(self.delay)
|
|
|
+ try:
|
|
|
+ if self.submit_worker:
|
|
|
+ await self.coordinator.submit_attempt(
|
|
|
+ {
|
|
|
+ **context,
|
|
|
+ "role": AgentRole.WORKER.value,
|
|
|
+ "trace_id": context["worker_trace_id"],
|
|
|
+ "tool_call_id": f"submit-{context['attempt_id']}",
|
|
|
+ },
|
|
|
+ AttemptSubmission(
|
|
|
+ summary="done",
|
|
|
+ artifact_refs=[ArtifactRef(uri=f"memory://{context['attempt_id']}", version="1")],
|
|
|
+ ),
|
|
|
+ )
|
|
|
+ return WorkerRunResult(context["worker_trace_id"], "completed")
|
|
|
+ finally:
|
|
|
+ self.active -= 1
|
|
|
+
|
|
|
+ async def run_validator(self, context):
|
|
|
+ verdict = self.verdicts.popleft() if self.verdicts else ValidationVerdict.PASSED
|
|
|
+ if self.submit_validator:
|
|
|
+ criteria = [
|
|
|
+ CriterionResult(
|
|
|
+ criterion_id=item["criterion_id"], verdict=verdict,
|
|
|
+ reason="checked",
|
|
|
+ )
|
|
|
+ for item in context["task_spec"]["acceptance_criteria"]
|
|
|
+ ]
|
|
|
+ await self.coordinator.submit_validation(
|
|
|
+ {
|
|
|
+ **context,
|
|
|
+ "role": AgentRole.VALIDATOR.value,
|
|
|
+ "trace_id": context["validator_trace_id"],
|
|
|
+ "tool_call_id": f"validate-{context['validation_id']}",
|
|
|
+ },
|
|
|
+ verdict,
|
|
|
+ criteria,
|
|
|
+ "independent report",
|
|
|
+ [], [], [], "accept" if verdict == ValidationVerdict.PASSED else "replan",
|
|
|
+ )
|
|
|
+ return ValidatorRunResult(context["validator_trace_id"], "completed")
|
|
|
+
|
|
|
+
|
|
|
+async def make_coordinator(tmp_path, executor):
|
|
|
+ trace_store = FileSystemTraceStore(str(tmp_path))
|
|
|
+ await trace_store.create_trace(Trace(trace_id="root", mode="agent", task="mission", agent_role="planner"))
|
|
|
+ await trace_store.update_goal_tree("root", GoalTree(mission="mission"))
|
|
|
+ task_store = FileSystemTaskStore(str(tmp_path))
|
|
|
+ coordinator = TaskCoordinator(
|
|
|
+ task_store,
|
|
|
+ FileSystemArtifactStore(str(tmp_path)),
|
|
|
+ trace_store,
|
|
|
+ OrchestrationConfig(max_parallel_tasks=4),
|
|
|
+ TraceEventSink(str(tmp_path)),
|
|
|
+ executor,
|
|
|
+ )
|
|
|
+ executor.coordinator = coordinator
|
|
|
+ await coordinator.ensure_ledger("root", "mission")
|
|
|
+ return coordinator, task_store, trace_store
|
|
|
+
|
|
|
+
|
|
|
+async def create_task(coordinator, objective="task", parent_task_id=None):
|
|
|
+ result = await coordinator.create_tasks(
|
|
|
+ "root",
|
|
|
+ [{
|
|
|
+ "objective": objective,
|
|
|
+ "acceptance_criteria": [{"criterion_id": "c1", "description": "must pass", "hard": True}],
|
|
|
+ }],
|
|
|
+ parent_task_id=parent_task_id,
|
|
|
+ )
|
|
|
+ return result["tasks"][0]["task_id"]
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_passed_validation_requires_planner_accept(tmp_path):
|
|
|
+ executor = FakeExecutor([ValidationVerdict.PASSED])
|
|
|
+ coordinator, store, _ = await make_coordinator(tmp_path, executor)
|
|
|
+ task_id = await create_task(coordinator)
|
|
|
+ cycle = (await coordinator.dispatch_tasks("root", [task_id]))[0]
|
|
|
+ assert cycle.task_status == TaskStatus.AWAITING_DECISION
|
|
|
+ ledger = await store.load("root")
|
|
|
+ assert ledger.tasks[task_id].status == TaskStatus.AWAITING_DECISION
|
|
|
+
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, cycle.validation_id, DecisionAction.ACCEPT,
|
|
|
+ {"reason": "all hard criteria passed"}, "decision-1",
|
|
|
+ )
|
|
|
+ assert (await store.load("root")).tasks[task_id].status == TaskStatus.COMPLETED
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_failed_or_inconclusive_cannot_be_accepted(tmp_path):
|
|
|
+ executor = FakeExecutor([ValidationVerdict.FAILED])
|
|
|
+ coordinator, store, _ = await make_coordinator(tmp_path, executor)
|
|
|
+ task_id = await create_task(coordinator)
|
|
|
+ cycle = (await coordinator.dispatch_tasks("root", [task_id]))[0]
|
|
|
+ with pytest.raises(TaskConflict, match="passed"):
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, cycle.validation_id, DecisionAction.ACCEPT,
|
|
|
+ {"reason": "override"}, "bad-accept",
|
|
|
+ )
|
|
|
+ assert (await store.load("root")).tasks[task_id].status == TaskStatus.AWAITING_DECISION
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_repair_is_limited_to_once_per_spec_version(tmp_path):
|
|
|
+ executor = FakeExecutor([ValidationVerdict.FAILED, ValidationVerdict.FAILED])
|
|
|
+ coordinator, store, _ = await make_coordinator(tmp_path, executor)
|
|
|
+ task_id = await create_task(coordinator)
|
|
|
+ first = (await coordinator.dispatch_tasks("root", [task_id]))[0]
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, first.validation_id, DecisionAction.REPAIR,
|
|
|
+ {"reason": "small local correction"}, "repair-1",
|
|
|
+ )
|
|
|
+ second = (await coordinator.dispatch_tasks("root", [task_id]))[0]
|
|
|
+ ledger = await store.load("root")
|
|
|
+ attempts = [ledger.attempts[x] for x in ledger.tasks[task_id].attempt_ids]
|
|
|
+ assert attempts[0].worker_trace_id == attempts[1].worker_trace_id
|
|
|
+ with pytest.raises(TaskConflict, match="limit"):
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, second.validation_id, DecisionAction.REPAIR,
|
|
|
+ {"reason": "try again"}, "repair-2",
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_worker_without_submit_attempt_needs_replan(tmp_path):
|
|
|
+ executor = FakeExecutor([], submit_worker=False)
|
|
|
+ coordinator, store, _ = await make_coordinator(tmp_path, executor)
|
|
|
+ task_id = await create_task(coordinator)
|
|
|
+ result = (await coordinator.dispatch_tasks("root", [task_id]))[0]
|
|
|
+ ledger = await store.load("root")
|
|
|
+ assert result.task_status == TaskStatus.NEEDS_REPLAN
|
|
|
+ assert ledger.tasks[task_id].status == TaskStatus.NEEDS_REPLAN
|
|
|
+ assert ledger.attempts[result.attempt_id].status.value == "failed"
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_validator_without_submit_is_error_then_revalidates_with_new_trace(tmp_path):
|
|
|
+ executor = FakeExecutor([ValidationVerdict.PASSED], submit_validator=False)
|
|
|
+ coordinator, store, _ = await make_coordinator(tmp_path, executor)
|
|
|
+ task_id = await create_task(coordinator)
|
|
|
+ first = (await coordinator.dispatch_tasks("root", [task_id]))[0]
|
|
|
+ ledger = await store.load("root")
|
|
|
+ first_validation = ledger.validations[first.validation_id]
|
|
|
+ assert first.task_status == TaskStatus.NEEDS_REPLAN
|
|
|
+ assert first_validation.status.value == "error"
|
|
|
+ assert first_validation.verdict is None
|
|
|
+
|
|
|
+ executor.submit_validator = True
|
|
|
+ executor.verdicts.append(ValidationVerdict.PASSED)
|
|
|
+ second = await coordinator.revalidate_attempt(
|
|
|
+ "root", task_id, first.attempt_id, "revalidate-1"
|
|
|
+ )
|
|
|
+ assert second.task_status == TaskStatus.AWAITING_DECISION
|
|
|
+ assert second.validation_id != first.validation_id
|
|
|
+ assert second.validation.validator_trace_id != first_validation.validator_trace_id
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_retry_uses_new_trace_and_revise_invalidates_old_validation(tmp_path):
|
|
|
+ executor = FakeExecutor([
|
|
|
+ ValidationVerdict.FAILED,
|
|
|
+ ValidationVerdict.INCONCLUSIVE,
|
|
|
+ ValidationVerdict.PASSED,
|
|
|
+ ])
|
|
|
+ coordinator, store, _ = await make_coordinator(tmp_path, executor)
|
|
|
+ task_id = await create_task(coordinator)
|
|
|
+ first = (await coordinator.dispatch_tasks("root", [task_id]))[0]
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, first.validation_id, DecisionAction.RETRY,
|
|
|
+ {"reason": "use a fresh worker"}, "retry-1",
|
|
|
+ )
|
|
|
+ second = (await coordinator.dispatch_tasks("root", [task_id]))[0]
|
|
|
+ ledger = await store.load("root")
|
|
|
+ first_attempt = ledger.attempts[first.attempt_id]
|
|
|
+ second_attempt = ledger.attempts[second.attempt_id]
|
|
|
+ assert first_attempt.worker_trace_id != second_attempt.worker_trace_id
|
|
|
+
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, second.validation_id, DecisionAction.REVISE,
|
|
|
+ {
|
|
|
+ "reason": "clarify criterion",
|
|
|
+ "objective": "revised task",
|
|
|
+ "acceptance_criteria": [{"criterion_id": "c1", "description": "revised", "hard": True}],
|
|
|
+ },
|
|
|
+ "revise-1",
|
|
|
+ )
|
|
|
+ ledger = await store.load("root")
|
|
|
+ assert ledger.tasks[task_id].current_spec_version == 2
|
|
|
+ third = (await coordinator.dispatch_tasks("root", [task_id]))[0]
|
|
|
+ with pytest.raises(TaskConflict, match="obsolete"):
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, first.validation_id, DecisionAction.ACCEPT,
|
|
|
+ {"reason": "stale"}, "stale-accept",
|
|
|
+ )
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, third.validation_id, DecisionAction.ACCEPT,
|
|
|
+ {"reason": "current validation passed"}, "current-accept",
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_block_unblock_cancel_and_supersede(tmp_path):
|
|
|
+ executor = FakeExecutor([])
|
|
|
+ coordinator, store, _ = await make_coordinator(tmp_path, executor)
|
|
|
+ task_id = await create_task(coordinator, "blocked")
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, None, DecisionAction.BLOCK,
|
|
|
+ {"reason": "external dependency"}, "block-1",
|
|
|
+ )
|
|
|
+ assert (await store.load("root")).tasks[task_id].blocked_reason == "external dependency"
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, None, DecisionAction.UNBLOCK, {}, "unblock-1"
|
|
|
+ )
|
|
|
+ assert (await store.load("root")).tasks[task_id].status == TaskStatus.NEEDS_REPLAN
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", task_id, None, DecisionAction.CANCEL,
|
|
|
+ {"reason": "no longer needed"}, "cancel-1",
|
|
|
+ )
|
|
|
+ assert (await store.load("root")).tasks[task_id].status == TaskStatus.CANCELLED
|
|
|
+
|
|
|
+ old_id = await create_task(coordinator, "old")
|
|
|
+ result = await coordinator.decide_task(
|
|
|
+ "root", old_id, None, DecisionAction.SUPERSEDE,
|
|
|
+ {"reason": "replace spec", "replacement": {"objective": "replacement"}},
|
|
|
+ "supersede-1",
|
|
|
+ )
|
|
|
+ replacement_id = result["payload"]["replacement_task_id"]
|
|
|
+ ledger = await store.load("root")
|
|
|
+ assert ledger.tasks[old_id].status == TaskStatus.SUPERSEDED
|
|
|
+ assert ledger.tasks[replacement_id].status == TaskStatus.PENDING
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_insert_after_keeps_stable_ids_and_updates_display_order(tmp_path):
|
|
|
+ executor = FakeExecutor([])
|
|
|
+ coordinator, store, _ = await make_coordinator(tmp_path, executor)
|
|
|
+ first = await create_task(coordinator, "first")
|
|
|
+ third = await create_task(coordinator, "third")
|
|
|
+ inserted = await coordinator.create_tasks(
|
|
|
+ "root",
|
|
|
+ [{"objective": "second"}],
|
|
|
+ placement={"after_task_id": first},
|
|
|
+ idempotency_key="insert",
|
|
|
+ )
|
|
|
+ second = inserted["tasks"][0]["task_id"]
|
|
|
+ ledger = await store.load("root")
|
|
|
+ assert ledger.tasks[first].display_path == "1"
|
|
|
+ assert ledger.tasks[second].display_path == "2"
|
|
|
+ assert ledger.tasks[third].display_path == "3"
|
|
|
+ assert len({first, second, third}) == 3
|
|
|
+ repeated = await coordinator.create_tasks(
|
|
|
+ "root",
|
|
|
+ [{"objective": "second"}],
|
|
|
+ placement={"after_task_id": first},
|
|
|
+ idempotency_key="insert",
|
|
|
+ )
|
|
|
+ assert repeated["tasks"][0]["task_id"] == second
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_split_children_do_not_auto_complete_parent(tmp_path):
|
|
|
+ executor = FakeExecutor([ValidationVerdict.FAILED, ValidationVerdict.PASSED, ValidationVerdict.PASSED])
|
|
|
+ coordinator, store, _ = await make_coordinator(tmp_path, executor)
|
|
|
+ parent_id = await create_task(coordinator, "parent")
|
|
|
+ first = (await coordinator.dispatch_tasks("root", [parent_id]))[0]
|
|
|
+ decision = await coordinator.decide_task(
|
|
|
+ "root", parent_id, first.validation_id, DecisionAction.SPLIT,
|
|
|
+ {
|
|
|
+ "reason": "split work",
|
|
|
+ "tasks": [
|
|
|
+ {"objective": "child one", "acceptance_criteria": [{"criterion_id": "c1", "description": "pass"}]},
|
|
|
+ {"objective": "child two", "acceptance_criteria": [{"criterion_id": "c1", "description": "pass"}]},
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ "split-1",
|
|
|
+ )
|
|
|
+ child_ids = decision["payload"]["child_task_ids"]
|
|
|
+ cycles = await coordinator.dispatch_tasks("root", child_ids)
|
|
|
+ for child_id, cycle in zip(child_ids, cycles):
|
|
|
+ await coordinator.decide_task(
|
|
|
+ "root", child_id, cycle.validation_id, DecisionAction.ACCEPT,
|
|
|
+ {"reason": "passed"}, f"accept-{child_id}",
|
|
|
+ )
|
|
|
+ ledger = await store.load("root")
|
|
|
+ assert all(ledger.tasks[x].status == TaskStatus.COMPLETED for x in child_ids)
|
|
|
+ assert ledger.tasks[parent_id].status == TaskStatus.NEEDS_REPLAN
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_parallel_tasks_are_isolated_and_bounded(tmp_path):
|
|
|
+ executor = FakeExecutor([ValidationVerdict.PASSED] * 4, delay=0.02)
|
|
|
+ coordinator, store, _ = await make_coordinator(tmp_path, executor)
|
|
|
+ task_ids = [await create_task(coordinator, f"task-{i}") for i in range(4)]
|
|
|
+ results = await coordinator.dispatch_tasks("root", task_ids, idempotency_key="batch")
|
|
|
+ assert [x.task_id for x in results] == task_ids
|
|
|
+ assert executor.max_active <= 4
|
|
|
+ assert all(x.task_status == TaskStatus.AWAITING_DECISION for x in results)
|
|
|
+ ledger = await store.load("root")
|
|
|
+ assert len({ledger.attempts[ledger.tasks[x].attempt_ids[-1]].worker_trace_id for x in task_ids}) == 4
|
|
|
+
|
|
|
+ repeated = await coordinator.dispatch_tasks("root", task_ids, idempotency_key="batch")
|
|
|
+ assert [x.attempt_id for x in repeated] == [x.attempt_id for x in results]
|
|
|
+
|
|
|
+
|
|
|
+@pytest.mark.asyncio
|
|
|
+async def test_real_runner_local_executor_creates_independent_terminal_traces(tmp_path):
|
|
|
+ import json
|
|
|
+
|
|
|
+ async def fake_llm(messages, tools, **kwargs):
|
|
|
+ names = {item["function"]["name"] for item in tools or []}
|
|
|
+ if "submit_attempt" in names:
|
|
|
+ arguments = {
|
|
|
+ "summary": "implemented",
|
|
|
+ "artifact_refs": [{"uri": "memory://result", "version": "1"}],
|
|
|
+ "evidence_refs": [],
|
|
|
+ }
|
|
|
+ tool_name = "submit_attempt"
|
|
|
+ elif "submit_validation" in names:
|
|
|
+ arguments = {
|
|
|
+ "verdict": "passed",
|
|
|
+ "criterion_results": [{"criterion_id": "c1", "verdict": "passed", "reason": "verified"}],
|
|
|
+ "summary": "independent pass",
|
|
|
+ "evidence_refs": [],
|
|
|
+ "unverified_claims": [],
|
|
|
+ "risks": [],
|
|
|
+ "recommendation": "accept",
|
|
|
+ }
|
|
|
+ tool_name = "submit_validation"
|
|
|
+ else:
|
|
|
+ raise AssertionError(f"unexpected tools: {names}")
|
|
|
+ return {
|
|
|
+ "content": "",
|
|
|
+ "tool_calls": [{
|
|
|
+ "id": f"call-{tool_name}",
|
|
|
+ "type": "function",
|
|
|
+ "function": {"name": tool_name, "arguments": json.dumps(arguments)},
|
|
|
+ }],
|
|
|
+ "finish_reason": "tool_calls",
|
|
|
+ }
|
|
|
+
|
|
|
+ trace_store = FileSystemTraceStore(str(tmp_path))
|
|
|
+ await trace_store.create_trace(Trace(trace_id="root", mode="agent", task="mission", agent_role="planner"))
|
|
|
+ await trace_store.update_goal_tree("root", GoalTree(mission="mission"))
|
|
|
+ runner = AgentRunner(trace_store=trace_store, llm_call=fake_llm)
|
|
|
+ coordinator = wire_orchestration(
|
|
|
+ runner,
|
|
|
+ FileSystemTaskStore(str(tmp_path)),
|
|
|
+ FileSystemArtifactStore(str(tmp_path)),
|
|
|
+ )
|
|
|
+ await coordinator.ensure_ledger("root", "mission")
|
|
|
+ task_id = await create_task(coordinator)
|
|
|
+ cycle = (await coordinator.dispatch_tasks("root", [task_id]))[0]
|
|
|
+ ledger = await coordinator.task_store.load("root")
|
|
|
+ worker = await trace_store.get_trace(ledger.attempts[cycle.attempt_id].worker_trace_id)
|
|
|
+ validator = await trace_store.get_trace(cycle.validation.validator_trace_id)
|
|
|
+ assert cycle.task_status == TaskStatus.AWAITING_DECISION
|
|
|
+ assert worker.agent_role == "worker" and worker.result_summary
|
|
|
+ assert validator.agent_role == "validator" and validator.result_summary
|
|
|
+ assert worker.trace_id != validator.trace_id
|