|
|
@@ -11,7 +11,8 @@ from dataclasses import asdict, is_dataclass
|
|
|
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
|
|
|
from uuid import uuid4
|
|
|
|
|
|
-from agent.failures import FailureDetail
|
|
|
+from agent.failures import FailureDetail, ToolExecutionError
|
|
|
+from agent.trace.models import Trace
|
|
|
|
|
|
from .config import OrchestrationConfig
|
|
|
from .models import (
|
|
|
@@ -74,6 +75,12 @@ from .validation_policy import (
|
|
|
)
|
|
|
from ._decision_engine import DecisionEngine
|
|
|
from ._task_context import render_task_context
|
|
|
+from .context_provider import (
|
|
|
+ RoleContextProvider,
|
|
|
+ RoleContextRequest,
|
|
|
+ TaskContextProvider,
|
|
|
+)
|
|
|
+from .deterministic_worker import DeterministicWorker, DeterministicWorkerContext
|
|
|
from ._task_graph import (
|
|
|
TERMINAL_TASK_STATUSES,
|
|
|
TaskGraph,
|
|
|
@@ -102,6 +109,9 @@ class TaskCoordinator:
|
|
|
validation_policy: Optional[ValidationPolicy] = None,
|
|
|
deterministic_validator: Optional[DeterministicValidator] = None,
|
|
|
evidence_provider: Optional[EvidenceProvider] = None,
|
|
|
+ task_context_provider: Optional[TaskContextProvider] = None,
|
|
|
+ role_context_provider: Optional[RoleContextProvider] = None,
|
|
|
+ deterministic_worker: Optional[DeterministicWorker] = None,
|
|
|
) -> None:
|
|
|
self.task_store = task_store
|
|
|
self.artifact_store = artifact_store
|
|
|
@@ -114,6 +124,9 @@ class TaskCoordinator:
|
|
|
)
|
|
|
self.deterministic_validator = deterministic_validator
|
|
|
self.evidence_provider = evidence_provider
|
|
|
+ self.task_context_provider = task_context_provider
|
|
|
+ self.role_context_provider = role_context_provider
|
|
|
+ self.deterministic_worker = deterministic_worker
|
|
|
self._locks: Dict[str, asyncio.Lock] = {}
|
|
|
self._task_graph = TaskGraph()
|
|
|
self._decision_engine = DecisionEngine(
|
|
|
@@ -157,7 +170,9 @@ class TaskCoordinator:
|
|
|
try:
|
|
|
ledger = await self.task_store.load(root_trace_id)
|
|
|
if ledger.root_trace_id != root_trace_id:
|
|
|
- raise OrchestrationError("Task ledger root trace identity is invalid")
|
|
|
+ raise OrchestrationError(
|
|
|
+ "Task ledger root trace identity is invalid"
|
|
|
+ )
|
|
|
self._root_task(ledger)
|
|
|
return ledger
|
|
|
except TaskStoreNotFound:
|
|
|
@@ -173,7 +188,9 @@ class TaskCoordinator:
|
|
|
if not isinstance(root_task_spec, dict):
|
|
|
raise ValueError("root_task_spec must be an object")
|
|
|
unknown = set(root_task_spec) - {
|
|
|
- "objective", "acceptance_criteria", "context_refs",
|
|
|
+ "objective",
|
|
|
+ "acceptance_criteria",
|
|
|
+ "context_refs",
|
|
|
}
|
|
|
if unknown:
|
|
|
raise ValueError(
|
|
|
@@ -222,9 +239,7 @@ class TaskCoordinator:
|
|
|
root = self._root_task(ledger)
|
|
|
nonterminal_descendants = self._nonterminal_descendants(ledger, root)
|
|
|
if root.status == TaskStatus.COMPLETED and nonterminal_descendants:
|
|
|
- raise OrchestrationError(
|
|
|
- "Completed root task has non-terminal descendants"
|
|
|
- )
|
|
|
+ raise OrchestrationError("Completed root task has non-terminal descendants")
|
|
|
result_summary = None
|
|
|
if root.status == TaskStatus.COMPLETED:
|
|
|
_, attempt, _ = self._accepted_result_binding(ledger, root)
|
|
|
@@ -236,7 +251,9 @@ class TaskCoordinator:
|
|
|
"status": task.status.value,
|
|
|
"objective": task.current_spec.objective,
|
|
|
}
|
|
|
- for task in sorted(ledger.tasks.values(), key=lambda item: _path_key(item.display_path))
|
|
|
+ for task in sorted(
|
|
|
+ ledger.tasks.values(), key=lambda item: _path_key(item.display_path)
|
|
|
+ )
|
|
|
if task.status not in TERMINAL_TASK_STATUSES
|
|
|
]
|
|
|
return {
|
|
|
@@ -279,6 +296,8 @@ class TaskCoordinator:
|
|
|
|
|
|
ledger = await self.task_store.load(root_trace_id)
|
|
|
self._root_task(ledger)
|
|
|
+ if self.task_context_provider is not None:
|
|
|
+ return await self.task_context_provider.render(root_trace_id, ledger)
|
|
|
return render_task_context(ledger)
|
|
|
|
|
|
async def progress_snapshot(self, root_trace_id: str) -> Dict[str, Any]:
|
|
|
@@ -340,7 +359,11 @@ class TaskCoordinator:
|
|
|
for ref in attempt.submission.artifact_refs
|
|
|
),
|
|
|
"snapshot_sha256": (
|
|
|
- (await self.artifact_store.get(root_trace_id, attempt.snapshot_id)).sha256
|
|
|
+ (
|
|
|
+ await self.artifact_store.get(
|
|
|
+ root_trace_id, attempt.snapshot_id
|
|
|
+ )
|
|
|
+ ).sha256
|
|
|
if attempt.snapshot_id
|
|
|
else None
|
|
|
),
|
|
|
@@ -395,7 +418,10 @@ class TaskCoordinator:
|
|
|
ledger = await self.task_store.load(root_trace_id)
|
|
|
if command_id and command_id in ledger.command_records:
|
|
|
command = ledger.command_records[command_id]
|
|
|
- if command.operation != event_type or command.fingerprint != fingerprint:
|
|
|
+ if (
|
|
|
+ command.operation != event_type
|
|
|
+ or command.fingerprint != fingerprint
|
|
|
+ ):
|
|
|
raise TaskConflict(
|
|
|
"Idempotency key is bound to a different operation or request"
|
|
|
)
|
|
|
@@ -483,8 +509,13 @@ class TaskCoordinator:
|
|
|
target = _task(ledger, after_task_id)
|
|
|
if target.task_id == ledger.root_task_id:
|
|
|
raise ValueError("The root task cannot have siblings")
|
|
|
- if parent_task_id is not None and target.parent_task_id != parent_task_id:
|
|
|
- raise ValueError("after_task_id must be a sibling under parent_task_id")
|
|
|
+ if (
|
|
|
+ parent_task_id is not None
|
|
|
+ and target.parent_task_id != parent_task_id
|
|
|
+ ):
|
|
|
+ raise ValueError(
|
|
|
+ "after_task_id must be a sibling under parent_task_id"
|
|
|
+ )
|
|
|
effective_parent_id = target.parent_task_id
|
|
|
else:
|
|
|
effective_parent_id = parent_task_id or ledger.root_task_id
|
|
|
@@ -503,13 +534,23 @@ class TaskCoordinator:
|
|
|
f"{parent.status.value}"
|
|
|
)
|
|
|
siblings = sorted(
|
|
|
- (t for t in ledger.tasks.values() if t.parent_task_id == effective_parent_id),
|
|
|
+ (
|
|
|
+ t
|
|
|
+ for t in ledger.tasks.values()
|
|
|
+ if t.parent_task_id == effective_parent_id
|
|
|
+ ),
|
|
|
key=lambda item: _path_key(item.display_path),
|
|
|
)
|
|
|
if after_task_id:
|
|
|
- target_index = next(i for i, sibling in enumerate(siblings) if sibling.task_id == after_task_id)
|
|
|
+ target_index = next(
|
|
|
+ i
|
|
|
+ for i, sibling in enumerate(siblings)
|
|
|
+ if sibling.task_id == after_task_id
|
|
|
+ )
|
|
|
start = target_index + 2
|
|
|
- for index, sibling in enumerate(siblings[target_index + 1:], start=start + len(drafts)):
|
|
|
+ for index, sibling in enumerate(
|
|
|
+ siblings[target_index + 1 :], start=start + len(drafts)
|
|
|
+ ):
|
|
|
self._rebase_task_path(
|
|
|
ledger,
|
|
|
sibling,
|
|
|
@@ -517,7 +558,11 @@ class TaskCoordinator:
|
|
|
)
|
|
|
else:
|
|
|
start = len(siblings) + 1
|
|
|
- parent_path = ledger.tasks[effective_parent_id].display_path if effective_parent_id else ""
|
|
|
+ parent_path = (
|
|
|
+ ledger.tasks[effective_parent_id].display_path
|
|
|
+ if effective_parent_id
|
|
|
+ else ""
|
|
|
+ )
|
|
|
created: List[str] = []
|
|
|
for offset, draft in enumerate(drafts):
|
|
|
index = start + offset
|
|
|
@@ -716,29 +761,35 @@ class TaskCoordinator:
|
|
|
reserved = await self._create_attempt(
|
|
|
operation.root_trace_id,
|
|
|
task_id,
|
|
|
- presets[index] if index < len(presets) else self.config.default_worker_preset,
|
|
|
+ presets[index]
|
|
|
+ if index < len(presets)
|
|
|
+ else self.config.default_worker_preset,
|
|
|
None,
|
|
|
operation_id=operation.operation_id,
|
|
|
execution_epoch=operation.execution_epoch,
|
|
|
)
|
|
|
attempt_id = reserved["attempt_id"]
|
|
|
except Exception as exc:
|
|
|
- results.append(await self._cycle_error_result(
|
|
|
- operation.root_trace_id,
|
|
|
- task_id,
|
|
|
- None,
|
|
|
- f"Attempt reservation failed during resume: "
|
|
|
- f"{type(exc).__name__}: {exc}",
|
|
|
- ))
|
|
|
+ results.append(
|
|
|
+ await self._cycle_error_result(
|
|
|
+ operation.root_trace_id,
|
|
|
+ task_id,
|
|
|
+ None,
|
|
|
+ f"Attempt reservation failed during resume: "
|
|
|
+ f"{type(exc).__name__}: {exc}",
|
|
|
+ )
|
|
|
+ )
|
|
|
continue
|
|
|
- results.append(await self.advance_cycle(
|
|
|
- operation.root_trace_id,
|
|
|
- task_id,
|
|
|
- attempt_id,
|
|
|
- operation_id=operation.operation_id,
|
|
|
- execution_epoch=operation.execution_epoch,
|
|
|
- deadline=operation.deadline_at,
|
|
|
- ))
|
|
|
+ results.append(
|
|
|
+ await self.advance_cycle(
|
|
|
+ operation.root_trace_id,
|
|
|
+ task_id,
|
|
|
+ attempt_id,
|
|
|
+ operation_id=operation.operation_id,
|
|
|
+ execution_epoch=operation.execution_epoch,
|
|
|
+ deadline=operation.deadline_at,
|
|
|
+ )
|
|
|
+ )
|
|
|
return results
|
|
|
|
|
|
async def _operation_cycle_results(
|
|
|
@@ -762,11 +813,15 @@ class TaskCoordinator:
|
|
|
task_id,
|
|
|
attempt_ids[index],
|
|
|
)
|
|
|
- if operation.status in {
|
|
|
- OperationStatus.PENDING,
|
|
|
- OperationStatus.RUNNING,
|
|
|
- OperationStatus.STOP_REQUESTED,
|
|
|
- } and not result.error:
|
|
|
+ if (
|
|
|
+ operation.status
|
|
|
+ in {
|
|
|
+ OperationStatus.PENDING,
|
|
|
+ OperationStatus.RUNNING,
|
|
|
+ OperationStatus.STOP_REQUESTED,
|
|
|
+ }
|
|
|
+ and not result.error
|
|
|
+ ):
|
|
|
result = TaskCycleResult(
|
|
|
task_id=result.task_id,
|
|
|
task_status=result.task_status,
|
|
|
@@ -892,13 +947,17 @@ class TaskCoordinator:
|
|
|
# could corrupt a pre-existing run owned by another dispatch.
|
|
|
attempt_id = None
|
|
|
early_results[index] = await self._cycle_error_result(
|
|
|
- root_trace_id, task_id, attempt_id,
|
|
|
+ root_trace_id,
|
|
|
+ task_id,
|
|
|
+ attempt_id,
|
|
|
f"Attempt reservation failed: {type(exc).__name__}: {exc}",
|
|
|
)
|
|
|
|
|
|
semaphore = asyncio.Semaphore(self.config.max_parallel_tasks)
|
|
|
|
|
|
- async def run_one(index: int, task_id: str, attempt_id: str) -> Tuple[int, TaskCycleResult]:
|
|
|
+ async def run_one(
|
|
|
+ index: int, task_id: str, attempt_id: str
|
|
|
+ ) -> Tuple[int, TaskCycleResult]:
|
|
|
async with semaphore:
|
|
|
try:
|
|
|
result = await self.advance_cycle(
|
|
|
@@ -925,7 +984,10 @@ class TaskCoordinator:
|
|
|
return index, result
|
|
|
|
|
|
completed = await asyncio.gather(
|
|
|
- *(run_one(index, task_id, attempt_id) for index, task_id, attempt_id in reservations)
|
|
|
+ *(
|
|
|
+ run_one(index, task_id, attempt_id)
|
|
|
+ for index, task_id, attempt_id in reservations
|
|
|
+ )
|
|
|
)
|
|
|
indexed_results = dict(early_results)
|
|
|
indexed_results.update(completed)
|
|
|
@@ -952,7 +1014,9 @@ class TaskCoordinator:
|
|
|
validation_id = candidate_id
|
|
|
break
|
|
|
error = validation.error if validation else (attempt.error if attempt else None)
|
|
|
- failure = validation.failure if validation else (attempt.failure if attempt else None)
|
|
|
+ failure = (
|
|
|
+ validation.failure if validation else (attempt.failure if attempt else None)
|
|
|
+ )
|
|
|
if task.status in {
|
|
|
TaskStatus.RUNNING,
|
|
|
TaskStatus.AWAITING_VALIDATION,
|
|
|
@@ -1113,13 +1177,16 @@ class TaskCoordinator:
|
|
|
)
|
|
|
task = _task(ledger, task_id)
|
|
|
if task.status not in (TaskStatus.PENDING, TaskStatus.NEEDS_REPLAN):
|
|
|
- raise TaskConflict(f"Task {task_id} cannot dispatch from {task.status.value}")
|
|
|
+ raise TaskConflict(
|
|
|
+ f"Task {task_id} cannot dispatch from {task.status.value}"
|
|
|
+ )
|
|
|
if self._nonterminal_descendants(ledger, task):
|
|
|
raise TaskConflict(
|
|
|
f"Task {task_id} cannot dispatch while descendants are non-terminal"
|
|
|
)
|
|
|
active = [
|
|
|
- ledger.attempts[x] for x in task.attempt_ids
|
|
|
+ ledger.attempts[x]
|
|
|
+ for x in task.attempt_ids
|
|
|
if ledger.attempts[x].status == AttemptStatus.RUNNING
|
|
|
]
|
|
|
if active:
|
|
|
@@ -1229,9 +1296,7 @@ class TaskCoordinator:
|
|
|
worker_result = WorkerRunResult(
|
|
|
trace_id=attempt.worker_trace_id,
|
|
|
status="failed",
|
|
|
- error=(
|
|
|
- "Legacy running attempt has no frozen child input binding"
|
|
|
- ),
|
|
|
+ error=("Legacy running attempt has no frozen child input binding"),
|
|
|
execution_stats=ExecutionStats(
|
|
|
failure_code=FailureCode.PROTOCOL_VIOLATION
|
|
|
),
|
|
|
@@ -1287,9 +1352,7 @@ class TaskCoordinator:
|
|
|
else None
|
|
|
),
|
|
|
"repair_feedback": (
|
|
|
- prior_feedback
|
|
|
- if attempt.execution_mode == "repair"
|
|
|
- else None
|
|
|
+ prior_feedback if attempt.execution_mode == "repair" else None
|
|
|
),
|
|
|
"prior_feedback": prior_feedback,
|
|
|
"accepted_child_results": accepted_child_results,
|
|
|
@@ -1299,11 +1362,109 @@ class TaskCoordinator:
|
|
|
}
|
|
|
if worker_result is None:
|
|
|
try:
|
|
|
- worker_result = await asyncio.wait_for(
|
|
|
- self.executor.run_worker(worker_context), # type: ignore[union-attr]
|
|
|
- timeout=stage_timeout(
|
|
|
- deadline,
|
|
|
- self.config.worker_timeout_seconds,
|
|
|
+ role_context = await self._build_role_context(
|
|
|
+ RoleContextRequest(
|
|
|
+ role=AgentRole.WORKER,
|
|
|
+ root_trace_id=root_trace_id,
|
|
|
+ task_id=task_id,
|
|
|
+ spec_version=task.current_spec_version,
|
|
|
+ task_spec=worker_context["task_spec"],
|
|
|
+ attempt_id=attempt_id,
|
|
|
+ ledger_revision=ledger.revision,
|
|
|
+ accepted_child_results=tuple(accepted_child_results),
|
|
|
+ metadata={
|
|
|
+ "operation_id": operation_id,
|
|
|
+ "execution_epoch": execution_epoch,
|
|
|
+ },
|
|
|
+ )
|
|
|
+ )
|
|
|
+ worker_context["role_context"] = role_context
|
|
|
+ deterministic_context = DeterministicWorkerContext(
|
|
|
+ root_trace_id=root_trace_id,
|
|
|
+ task=task,
|
|
|
+ attempt=attempt,
|
|
|
+ ledger_revision=ledger.revision,
|
|
|
+ accepted_child_results=tuple(accepted_child_results),
|
|
|
+ operation_id=operation_id,
|
|
|
+ execution_epoch=execution_epoch,
|
|
|
+ role_context=role_context,
|
|
|
+ )
|
|
|
+ timeout = stage_timeout(
|
|
|
+ deadline, self.config.worker_timeout_seconds
|
|
|
+ )
|
|
|
+ if (
|
|
|
+ self.deterministic_worker is not None
|
|
|
+ and await self.deterministic_worker.supports(
|
|
|
+ deterministic_context
|
|
|
+ )
|
|
|
+ ):
|
|
|
+ deterministic_result = await asyncio.wait_for(
|
|
|
+ self.deterministic_worker.execute(deterministic_context),
|
|
|
+ timeout=timeout,
|
|
|
+ )
|
|
|
+ if self.trace_store is not None:
|
|
|
+ existing_trace = await self.trace_store.get_trace(
|
|
|
+ attempt.worker_trace_id
|
|
|
+ )
|
|
|
+ if existing_trace is None:
|
|
|
+ await self.trace_store.create_trace(
|
|
|
+ Trace(
|
|
|
+ trace_id=attempt.worker_trace_id,
|
|
|
+ mode="agent",
|
|
|
+ task=task.current_spec.objective,
|
|
|
+ agent_type=attempt.worker_preset,
|
|
|
+ agent_role=AgentRole.WORKER.value,
|
|
|
+ parent_trace_id=root_trace_id,
|
|
|
+ status="completed",
|
|
|
+ total_messages=0,
|
|
|
+ total_tokens=0,
|
|
|
+ result_summary=deterministic_result.summary,
|
|
|
+ context={
|
|
|
+ "task_id": task_id,
|
|
|
+ "spec_version": task.current_spec_version,
|
|
|
+ "deterministic_worker": True,
|
|
|
+ },
|
|
|
+ )
|
|
|
+ )
|
|
|
+ await self.submit_attempt(
|
|
|
+ {
|
|
|
+ "role": AgentRole.WORKER.value,
|
|
|
+ "root_trace_id": root_trace_id,
|
|
|
+ "task_id": task_id,
|
|
|
+ "attempt_id": attempt_id,
|
|
|
+ "spec_version": task.current_spec_version,
|
|
|
+ "trace_id": attempt.worker_trace_id,
|
|
|
+ "tool_call_id": f"deterministic-worker:{attempt_id}",
|
|
|
+ "operation_id": operation_id,
|
|
|
+ "execution_epoch": execution_epoch,
|
|
|
+ },
|
|
|
+ AttemptSubmission(
|
|
|
+ summary=deterministic_result.summary,
|
|
|
+ artifact_refs=list(deterministic_result.artifact_refs),
|
|
|
+ evidence_refs=list(deterministic_result.evidence_refs),
|
|
|
+ ),
|
|
|
+ )
|
|
|
+ worker_result = WorkerRunResult(
|
|
|
+ trace_id=attempt.worker_trace_id,
|
|
|
+ status="completed",
|
|
|
+ summary=deterministic_result.summary,
|
|
|
+ execution_stats=ExecutionStats(
|
|
|
+ total_tokens=0, total_cost=0.0
|
|
|
+ ),
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ worker_result = await asyncio.wait_for(
|
|
|
+ self.executor.run_worker(worker_context), # type: ignore[union-attr]
|
|
|
+ timeout=timeout,
|
|
|
+ )
|
|
|
+ except ToolExecutionError as exc:
|
|
|
+ worker_result = WorkerRunResult(
|
|
|
+ trace_id=attempt.worker_trace_id,
|
|
|
+ status="failed",
|
|
|
+ error=exc.failure.message,
|
|
|
+ failure=exc.failure,
|
|
|
+ execution_stats=ExecutionStats(
|
|
|
+ failure_code=FailureCode.TOOL_FAILURE
|
|
|
),
|
|
|
)
|
|
|
except asyncio.TimeoutError:
|
|
|
@@ -1311,7 +1472,9 @@ class TaskCoordinator:
|
|
|
trace_id=attempt.worker_trace_id,
|
|
|
status="expired",
|
|
|
error="Worker execution timed out",
|
|
|
- execution_stats=ExecutionStats(failure_code=FailureCode.TIMEOUT),
|
|
|
+ execution_stats=ExecutionStats(
|
|
|
+ failure_code=FailureCode.TIMEOUT
|
|
|
+ ),
|
|
|
)
|
|
|
except Exception as exc:
|
|
|
worker_result = WorkerRunResult(
|
|
|
@@ -1522,10 +1685,36 @@ class TaskCoordinator:
|
|
|
timeout=timeout,
|
|
|
)
|
|
|
else:
|
|
|
+ validator_context["role_context"] = await self._build_role_context(
|
|
|
+ RoleContextRequest(
|
|
|
+ role=AgentRole.VALIDATOR,
|
|
|
+ root_trace_id=root_trace_id,
|
|
|
+ task_id=task_id,
|
|
|
+ spec_version=task.current_spec_version,
|
|
|
+ task_spec=validator_context["task_spec"],
|
|
|
+ attempt_id=attempt_id,
|
|
|
+ ledger_revision=ledger.revision,
|
|
|
+ snapshot_id=snapshot.snapshot_id,
|
|
|
+ artifact_snapshot=validator_context["artifact_snapshot"],
|
|
|
+ metadata={
|
|
|
+ "validation_id": validation_id,
|
|
|
+ "operation_id": operation_id,
|
|
|
+ "execution_epoch": execution_epoch,
|
|
|
+ },
|
|
|
+ )
|
|
|
+ )
|
|
|
validator_result = await asyncio.wait_for(
|
|
|
self.executor.run_validator(validator_context), # type: ignore[union-attr]
|
|
|
timeout=timeout,
|
|
|
)
|
|
|
+ except ToolExecutionError as exc:
|
|
|
+ validator_result = ValidatorRunResult(
|
|
|
+ trace_id=validation.validator_trace_id,
|
|
|
+ status="error",
|
|
|
+ error=exc.failure.message,
|
|
|
+ failure=exc.failure,
|
|
|
+ execution_stats=ExecutionStats(failure_code=FailureCode.TOOL_FAILURE),
|
|
|
+ )
|
|
|
except asyncio.TimeoutError:
|
|
|
validator_result = ValidatorRunResult(
|
|
|
trace_id=validation.validator_trace_id,
|
|
|
@@ -1538,9 +1727,7 @@ class TaskCoordinator:
|
|
|
trace_id=validation.validator_trace_id,
|
|
|
status="error",
|
|
|
error=f"Validator executor error: {exc}",
|
|
|
- execution_stats=ExecutionStats(
|
|
|
- failure_code=FailureCode.EXECUTOR_ERROR
|
|
|
- ),
|
|
|
+ execution_stats=ExecutionStats(failure_code=FailureCode.EXECUTOR_ERROR),
|
|
|
)
|
|
|
ledger = await self.task_store.load(root_trace_id)
|
|
|
validation = ledger.validations[validation_id]
|
|
|
@@ -1593,6 +1780,14 @@ class TaskCoordinator:
|
|
|
failure=validation.failure,
|
|
|
)
|
|
|
|
|
|
+ async def _build_role_context(self, request: RoleContextRequest) -> Dict[str, Any]:
|
|
|
+ if self.role_context_provider is None:
|
|
|
+ return {}
|
|
|
+ value = await self.role_context_provider.build(request)
|
|
|
+ if not isinstance(value, dict):
|
|
|
+ value = dict(value)
|
|
|
+ return _plain(value)
|
|
|
+
|
|
|
async def _reuse_semantic_validation(
|
|
|
self,
|
|
|
context: ValidationContext,
|
|
|
@@ -1642,7 +1837,9 @@ class TaskCoordinator:
|
|
|
},
|
|
|
prior.verdict,
|
|
|
tuple(prior.criterion_results),
|
|
|
- f"semantic_validation_reused_from={prior.validation_id};{prior.summary}"[:500],
|
|
|
+ f"semantic_validation_reused_from={prior.validation_id};{prior.summary}"[
|
|
|
+ :500
|
|
|
+ ],
|
|
|
tuple(prior.evidence_refs),
|
|
|
tuple(prior.unverified_claims),
|
|
|
tuple(prior.risks),
|
|
|
@@ -1727,15 +1924,16 @@ class TaskCoordinator:
|
|
|
) -> ValidatorRunResult:
|
|
|
if self.deterministic_validator is None:
|
|
|
raise RuntimeError("No DeterministicValidator is configured")
|
|
|
- result: DeterministicValidationResult = await self.deterministic_validator.validate(
|
|
|
- context,
|
|
|
- validation.validation_plan,
|
|
|
+ result: DeterministicValidationResult = (
|
|
|
+ await self.deterministic_validator.validate(
|
|
|
+ context,
|
|
|
+ validation.validation_plan,
|
|
|
+ )
|
|
|
)
|
|
|
returned_rule_ids = [item.rule_id for item in result.rule_results]
|
|
|
- if (
|
|
|
- len(set(returned_rule_ids)) != len(returned_rule_ids)
|
|
|
- or set(returned_rule_ids) != set(validation.validation_plan.rule_ids)
|
|
|
- ):
|
|
|
+ if len(set(returned_rule_ids)) != len(returned_rule_ids) or set(
|
|
|
+ returned_rule_ids
|
|
|
+ ) != set(validation.validation_plan.rule_ids):
|
|
|
raise ValueError(
|
|
|
"DeterministicValidator results must match the frozen rule_ids exactly"
|
|
|
)
|
|
|
@@ -1761,9 +1959,7 @@ class TaskCoordinator:
|
|
|
)
|
|
|
)
|
|
|
evidence_refs = [
|
|
|
- ref
|
|
|
- for item in result.rule_results
|
|
|
- for ref in item.evidence_refs
|
|
|
+ ref for item in result.rule_results for ref in item.evidence_refs
|
|
|
]
|
|
|
errors = result.errors
|
|
|
summary = (
|
|
|
@@ -1789,7 +1985,9 @@ class TaskCoordinator:
|
|
|
evidence_refs,
|
|
|
errors,
|
|
|
errors,
|
|
|
- "Planner review required" if result.verdict != ValidationVerdict.PASSED else "Accept",
|
|
|
+ "Planner review required"
|
|
|
+ if result.verdict != ValidationVerdict.PASSED
|
|
|
+ else "Accept",
|
|
|
)
|
|
|
return ValidatorRunResult(
|
|
|
trace_id=validation.validator_trace_id,
|
|
|
@@ -1814,7 +2012,9 @@ class TaskCoordinator:
|
|
|
operation.status != OperationStatus.RUNNING
|
|
|
or operation.execution_epoch != execution_epoch
|
|
|
):
|
|
|
- raise TaskConflict("Execution epoch is stale or operation is stopping")
|
|
|
+ raise TaskConflict(
|
|
|
+ "Execution epoch is stale or operation is stopping"
|
|
|
+ )
|
|
|
if attempt_id:
|
|
|
stage = ledger.attempts[attempt_id]
|
|
|
elif validation_id:
|
|
|
@@ -1890,13 +2090,22 @@ class TaskCoordinator:
|
|
|
raise OrchestrationError("Attempt does not belong to task")
|
|
|
if attempt.worker_trace_id != actor_context.get("trace_id"):
|
|
|
raise OrchestrationError("Worker trace does not own this attempt")
|
|
|
- if attempt.spec_version != int(actor_context.get("spec_version")):
|
|
|
- raise OrchestrationError("Worker submitted against the wrong TaskSpec version")
|
|
|
+ actor_spec_version = actor_context.get("spec_version")
|
|
|
+ if isinstance(actor_spec_version, bool) or not isinstance(
|
|
|
+ actor_spec_version, int
|
|
|
+ ):
|
|
|
+ raise OrchestrationError("Worker context has no valid TaskSpec version")
|
|
|
+ if attempt.spec_version != actor_spec_version:
|
|
|
+ raise OrchestrationError(
|
|
|
+ "Worker submitted against the wrong TaskSpec version"
|
|
|
+ )
|
|
|
self._assert_execution_owner(ledger, attempt, actor_context)
|
|
|
if attempt.status != AttemptStatus.RUNNING or task.status != TaskStatus.RUNNING:
|
|
|
raise TaskConflict("Attempt can only be submitted once while running")
|
|
|
|
|
|
- snapshot = await self.artifact_store.freeze(root_trace_id, attempt_id, submission)
|
|
|
+ snapshot = await self.artifact_store.freeze(
|
|
|
+ root_trace_id, attempt_id, submission
|
|
|
+ )
|
|
|
|
|
|
def mutate(current: TaskLedger) -> Dict[str, Any]:
|
|
|
current_task = _task(current, task_id)
|
|
|
@@ -1953,7 +2162,10 @@ class TaskCoordinator:
|
|
|
)
|
|
|
task = _task(ledger, task_id)
|
|
|
attempt = ledger.attempts[attempt_id]
|
|
|
- if task.status != TaskStatus.AWAITING_VALIDATION or attempt.status != AttemptStatus.SUBMITTED:
|
|
|
+ if (
|
|
|
+ task.status != TaskStatus.AWAITING_VALIDATION
|
|
|
+ or attempt.status != AttemptStatus.SUBMITTED
|
|
|
+ ):
|
|
|
raise TaskConflict("Validation requires a submitted attempt")
|
|
|
validation = ValidationReport(
|
|
|
validation_id=new_id(),
|
|
|
@@ -2020,7 +2232,9 @@ class TaskCoordinator:
|
|
|
if plan.validator_preset is not None and not plan.validator_preset.strip():
|
|
|
raise ValueError("Agent validator_preset cannot be blank")
|
|
|
if not plan.validator_preset:
|
|
|
- plan = replace(plan, validator_preset=self.config.default_validator_preset)
|
|
|
+ plan = replace(
|
|
|
+ plan, validator_preset=self.config.default_validator_preset
|
|
|
+ )
|
|
|
if plan.mode == ValidationMode.DETERMINISTIC:
|
|
|
criteria = {
|
|
|
item.criterion_id: item
|
|
|
@@ -2082,14 +2296,21 @@ class TaskCoordinator:
|
|
|
return replay
|
|
|
task = _task(ledger, task_id)
|
|
|
validation = ledger.validations.get(validation_id)
|
|
|
- if not validation or validation.task_id != task_id or validation.attempt_id != attempt_id:
|
|
|
+ if (
|
|
|
+ not validation
|
|
|
+ or validation.task_id != task_id
|
|
|
+ or validation.attempt_id != attempt_id
|
|
|
+ ):
|
|
|
raise OrchestrationError("Validation identity does not match task/attempt")
|
|
|
if validation.validator_trace_id != actor_context.get("trace_id"):
|
|
|
raise OrchestrationError("Validator trace does not own this validation")
|
|
|
if validation.snapshot_id != snapshot_id:
|
|
|
raise OrchestrationError("Validator must evaluate the fixed snapshot")
|
|
|
self._assert_execution_owner(ledger, validation, actor_context)
|
|
|
- if validation.status != ValidationRunStatus.RUNNING or task.status != TaskStatus.VALIDATING:
|
|
|
+ if (
|
|
|
+ validation.status != ValidationRunStatus.RUNNING
|
|
|
+ or task.status != TaskStatus.VALIDATING
|
|
|
+ ):
|
|
|
raise TaskConflict("Validation can only be submitted once while running")
|
|
|
self._validate_report(task, verdict, criterion_results)
|
|
|
|
|
|
@@ -2196,7 +2417,9 @@ class TaskCoordinator:
|
|
|
validation = self._owned_evidence_validation(ledger, actor_context, request)
|
|
|
self._assert_execution_owner(ledger, validation, actor_context)
|
|
|
if validation.status != ValidationRunStatus.RUNNING:
|
|
|
- raise TaskConflict("Evidence can only be queried during a running validation")
|
|
|
+ raise TaskConflict(
|
|
|
+ "Evidence can only be queried during a running validation"
|
|
|
+ )
|
|
|
await self._validate_evidence_snapshot(request)
|
|
|
return tool_call_id, validation
|
|
|
|
|
|
@@ -2213,7 +2436,9 @@ class TaskCoordinator:
|
|
|
"validation_id": request.validation_id,
|
|
|
}
|
|
|
if any(actor_context.get(key) != value for key, value in expected.items()):
|
|
|
- raise EvidenceOwnershipError("Evidence request does not match Validator scope")
|
|
|
+ raise EvidenceOwnershipError(
|
|
|
+ "Evidence request does not match Validator scope"
|
|
|
+ )
|
|
|
|
|
|
@staticmethod
|
|
|
def _owned_evidence_validation(
|
|
|
@@ -2240,7 +2465,9 @@ class TaskCoordinator:
|
|
|
):
|
|
|
raise EvidenceOwnershipError("Evidence snapshot ownership mismatch")
|
|
|
if validation.validator_trace_id != actor_context.get("trace_id"):
|
|
|
- raise EvidenceOwnershipError("Validator trace does not own this evidence scope")
|
|
|
+ raise EvidenceOwnershipError(
|
|
|
+ "Validator trace does not own this evidence scope"
|
|
|
+ )
|
|
|
return validation
|
|
|
|
|
|
async def _validate_evidence_snapshot(self, request: EvidenceQuery) -> None:
|
|
|
@@ -2311,7 +2538,9 @@ class TaskCoordinator:
|
|
|
timeout=plan.evidence_timeout_seconds,
|
|
|
)
|
|
|
except asyncio.TimeoutError as exc:
|
|
|
- await self._fail_evidence_query(request, query_key, "Evidence provider timed out")
|
|
|
+ await self._fail_evidence_query(
|
|
|
+ request, query_key, "Evidence provider timed out"
|
|
|
+ )
|
|
|
raise EvidenceProviderError("Evidence provider timed out") from exc
|
|
|
except asyncio.CancelledError:
|
|
|
await self._fail_evidence_query(
|
|
|
@@ -2324,10 +2553,10 @@ class TaskCoordinator:
|
|
|
error = f"Evidence provider failed: {type(exc).__name__}: {exc}"
|
|
|
await self._fail_evidence_query(request, query_key, error)
|
|
|
raise EvidenceProviderError(error) from exc
|
|
|
- error = self._evidence_result_error(result, request.limit)
|
|
|
- if error:
|
|
|
- await self._fail_evidence_query(request, query_key, error)
|
|
|
- raise EvidenceProviderError(error)
|
|
|
+ result_error = self._evidence_result_error(result, request.limit)
|
|
|
+ if result_error:
|
|
|
+ await self._fail_evidence_query(request, query_key, result_error)
|
|
|
+ raise EvidenceProviderError(result_error)
|
|
|
return result
|
|
|
|
|
|
async def _fail_evidence_query(
|
|
|
@@ -2414,7 +2643,9 @@ class TaskCoordinator:
|
|
|
queries_remaining=int(record["queries_remaining"]),
|
|
|
)
|
|
|
if status == "error":
|
|
|
- raise EvidenceProviderError(str(record.get("error", "Evidence query failed")))
|
|
|
+ raise EvidenceProviderError(
|
|
|
+ str(record.get("error", "Evidence query failed"))
|
|
|
+ )
|
|
|
if asyncio.get_running_loop().time() >= deadline:
|
|
|
raise EvidenceProviderError("Evidence query is already in progress")
|
|
|
await asyncio.sleep(0.01)
|
|
|
@@ -2432,7 +2663,9 @@ class TaskCoordinator:
|
|
|
actual = {c.criterion_id: c for c in criterion_results}
|
|
|
missing = set(expected) - set(actual)
|
|
|
if missing:
|
|
|
- raise ValueError(f"Validation report is missing criteria: {sorted(missing)}")
|
|
|
+ raise ValueError(
|
|
|
+ f"Validation report is missing criteria: {sorted(missing)}"
|
|
|
+ )
|
|
|
unknown = set(actual) - set(expected)
|
|
|
if unknown:
|
|
|
raise ValueError(
|
|
|
@@ -2440,11 +2673,14 @@ class TaskCoordinator:
|
|
|
)
|
|
|
if verdict == ValidationVerdict.PASSED:
|
|
|
non_passed = [
|
|
|
- cid for cid, criterion in expected.items()
|
|
|
+ cid
|
|
|
+ for cid, criterion in expected.items()
|
|
|
if criterion.hard and actual[cid].verdict != ValidationVerdict.PASSED
|
|
|
]
|
|
|
if non_passed:
|
|
|
- raise ValueError(f"Overall passed conflicts with hard criteria: {non_passed}")
|
|
|
+ raise ValueError(
|
|
|
+ f"Overall passed conflicts with hard criteria: {non_passed}"
|
|
|
+ )
|
|
|
|
|
|
@staticmethod
|
|
|
def _assert_execution_owner(
|
|
|
@@ -2467,7 +2703,9 @@ class TaskCoordinator:
|
|
|
or actor_epoch != stage.execution_epoch
|
|
|
or actor_epoch != operation.execution_epoch
|
|
|
):
|
|
|
- raise TaskConflict("Agent execution epoch is stale or operation is stopping")
|
|
|
+ raise TaskConflict(
|
|
|
+ "Agent execution epoch is stale or operation is stopping"
|
|
|
+ )
|
|
|
|
|
|
async def decide_task(
|
|
|
self,
|
|
|
@@ -2567,16 +2805,27 @@ class TaskCoordinator:
|
|
|
attempt = ledger.attempts.get(attempt_id)
|
|
|
if not attempt or attempt.task_id != task_id or not attempt.snapshot_id:
|
|
|
raise OrchestrationError("Attempt has no immutable snapshot")
|
|
|
- prior = [ledger.validations[x] for x in task.validation_ids if ledger.validations[x].attempt_id == attempt_id]
|
|
|
+ prior = [
|
|
|
+ ledger.validations[x]
|
|
|
+ for x in task.validation_ids
|
|
|
+ if ledger.validations[x].attempt_id == attempt_id
|
|
|
+ ]
|
|
|
if not prior or prior[-1].status not in {
|
|
|
- ValidationRunStatus.ERROR, ValidationRunStatus.STOPPED, ValidationRunStatus.EXPIRED
|
|
|
+ ValidationRunStatus.ERROR,
|
|
|
+ ValidationRunStatus.STOPPED,
|
|
|
+ ValidationRunStatus.EXPIRED,
|
|
|
}:
|
|
|
- raise TaskConflict("Revalidation is only allowed after validator error/stopped/expired")
|
|
|
+ raise TaskConflict(
|
|
|
+ "Revalidation is only allowed after validator error/stopped/expired"
|
|
|
+ )
|
|
|
if task.status != TaskStatus.NEEDS_REPLAN:
|
|
|
raise TaskConflict("Task must be in needs_replan before revalidation")
|
|
|
report = ValidationReport(
|
|
|
- validation_id=new_id(), task_id=task_id, attempt_id=attempt_id,
|
|
|
- spec_version=attempt.spec_version, snapshot_id=attempt.snapshot_id,
|
|
|
+ validation_id=new_id(),
|
|
|
+ task_id=task_id,
|
|
|
+ attempt_id=attempt_id,
|
|
|
+ spec_version=attempt.spec_version,
|
|
|
+ snapshot_id=attempt.snapshot_id,
|
|
|
validator_trace_id=str(uuid4()),
|
|
|
validator_preset=plan.validator_preset or "deterministic",
|
|
|
validation_plan=plan,
|
|
|
@@ -2629,17 +2878,34 @@ class TaskCoordinator:
|
|
|
raise OrchestrationError("Prior attempt does not belong to task")
|
|
|
if attempt.spec_version != task.current_spec_version:
|
|
|
raise TaskConflict("Cannot continue a worker across TaskSpec revisions")
|
|
|
- if not task.decision_ids or ledger.decisions[task.decision_ids[-1]].action != DecisionAction.REPAIR:
|
|
|
- raise TaskConflict("Worker continuation requires the latest decision to be repair")
|
|
|
- if task.repair_count_by_version.get(str(task.current_spec_version), 0) > self.config.max_repair_continuations:
|
|
|
+ if (
|
|
|
+ not task.decision_ids
|
|
|
+ or ledger.decisions[task.decision_ids[-1]].action != DecisionAction.REPAIR
|
|
|
+ ):
|
|
|
+ raise TaskConflict(
|
|
|
+ "Worker continuation requires the latest decision to be repair"
|
|
|
+ )
|
|
|
+ if (
|
|
|
+ task.repair_count_by_version.get(str(task.current_spec_version), 0)
|
|
|
+ > self.config.max_repair_continuations
|
|
|
+ ):
|
|
|
raise TaskConflict("Repair continuation limit exceeded")
|
|
|
- trace = await self.trace_store.get_trace(attempt.worker_trace_id) if self.trace_store else None
|
|
|
+ trace = (
|
|
|
+ await self.trace_store.get_trace(attempt.worker_trace_id)
|
|
|
+ if self.trace_store
|
|
|
+ else None
|
|
|
+ )
|
|
|
if not trace or trace.agent_role != AgentRole.WORKER.value:
|
|
|
raise TaskConflict("Prior worker trace is missing or has the wrong role")
|
|
|
if trace.status not in ("completed", "stopped"):
|
|
|
- raise TaskConflict("Prior worker trace is not in a recoverable terminal state")
|
|
|
+ raise TaskConflict(
|
|
|
+ "Prior worker trace is not in a recoverable terminal state"
|
|
|
+ )
|
|
|
context = trace.context or {}
|
|
|
- if context.get("root_trace_id") != root_trace_id or context.get("task_id") != task_id:
|
|
|
+ if (
|
|
|
+ context.get("root_trace_id") != root_trace_id
|
|
|
+ or context.get("task_id") != task_id
|
|
|
+ ):
|
|
|
raise TaskConflict("Prior worker trace identity is invalid")
|
|
|
if int(context.get("spec_version", -1)) != task.current_spec_version:
|
|
|
raise TaskConflict("Prior worker trace TaskSpec version is invalid")
|
|
|
@@ -2658,7 +2924,10 @@ class TaskCoordinator:
|
|
|
operation_id: Optional[str],
|
|
|
execution_epoch: int,
|
|
|
) -> bool:
|
|
|
- if stage.operation_id != operation_id or stage.execution_epoch != execution_epoch:
|
|
|
+ if (
|
|
|
+ stage.operation_id != operation_id
|
|
|
+ or stage.execution_epoch != execution_epoch
|
|
|
+ ):
|
|
|
return False
|
|
|
if operation_id is None:
|
|
|
return True
|
|
|
@@ -2693,11 +2962,8 @@ class TaskCoordinator:
|
|
|
if isinstance(stage, TaskAttempt)
|
|
|
else stage.status == ValidationRunStatus.COMPLETED
|
|
|
)
|
|
|
- if (
|
|
|
- not terminal
|
|
|
- or not self._stage_execution_is_current(
|
|
|
- ledger, stage, operation_id, execution_epoch
|
|
|
- )
|
|
|
+ if not terminal or not self._stage_execution_is_current(
|
|
|
+ ledger, stage, operation_id, execution_epoch
|
|
|
):
|
|
|
return False
|
|
|
if stage.execution_stats is not None:
|
|
|
@@ -2716,11 +2982,8 @@ class TaskCoordinator:
|
|
|
if isinstance(current_stage, TaskAttempt)
|
|
|
else current_stage.status == ValidationRunStatus.COMPLETED
|
|
|
)
|
|
|
- if (
|
|
|
- not current_terminal
|
|
|
- or not self._stage_execution_is_current(
|
|
|
- current, current_stage, operation_id, execution_epoch
|
|
|
- )
|
|
|
+ if not current_terminal or not self._stage_execution_is_current(
|
|
|
+ current, current_stage, operation_id, execution_epoch
|
|
|
):
|
|
|
return {
|
|
|
"recorded": False,
|
|
|
@@ -2775,7 +3038,10 @@ class TaskCoordinator:
|
|
|
)
|
|
|
):
|
|
|
return {"task_id": task_id, "attempt_id": attempt_id, "ignored": True}
|
|
|
- status_map = {"stopped": AttemptStatus.STOPPED, "expired": AttemptStatus.EXPIRED}
|
|
|
+ status_map = {
|
|
|
+ "stopped": AttemptStatus.STOPPED,
|
|
|
+ "expired": AttemptStatus.EXPIRED,
|
|
|
+ }
|
|
|
attempt.status = status_map.get(run_status, AttemptStatus.FAILED)
|
|
|
attempt.error = error
|
|
|
attempt.failure = failure
|
|
|
@@ -2973,7 +3239,9 @@ class TaskCoordinator:
|
|
|
def _update_parent_after_child(ledger: TaskLedger, child: TaskRecord) -> None:
|
|
|
TaskGraph.update_parent_after_child(ledger, child)
|
|
|
|
|
|
- async def _tasks_result(self, root_trace_id: str, task_ids: Iterable[str]) -> Dict[str, Any]:
|
|
|
+ async def _tasks_result(
|
|
|
+ self, root_trace_id: str, task_ids: Iterable[str]
|
|
|
+ ) -> Dict[str, Any]:
|
|
|
ledger = await self.task_store.load(root_trace_id)
|
|
|
return {
|
|
|
"tasks": [
|
|
|
@@ -2990,7 +3258,9 @@ class TaskCoordinator:
|
|
|
}
|
|
|
|
|
|
@staticmethod
|
|
|
- def _display_path(ledger: TaskLedger, parent_task_id: Optional[str], index: int) -> str:
|
|
|
+ def _display_path(
|
|
|
+ ledger: TaskLedger, parent_task_id: Optional[str], index: int
|
|
|
+ ) -> str:
|
|
|
return TaskGraph.display_path(ledger, parent_task_id, index)
|
|
|
|
|
|
@staticmethod
|
|
|
@@ -3019,7 +3289,9 @@ def _required(context: Dict[str, Any], key: str) -> str:
|
|
|
def _require_actor(context: Dict[str, Any], expected: AgentRole) -> None:
|
|
|
actual = context.get("role")
|
|
|
if actual != expected.value:
|
|
|
- raise OrchestrationError(f"Expected actor role {expected.value}, got {actual!r}")
|
|
|
+ raise OrchestrationError(
|
|
|
+ f"Expected actor role {expected.value}, got {actual!r}"
|
|
|
+ )
|
|
|
|
|
|
|
|
|
def _plain(value: Any) -> Any:
|
SamLee