|
@@ -11,6 +11,8 @@ from dataclasses import asdict, is_dataclass
|
|
|
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
|
|
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
|
|
|
from uuid import uuid4
|
|
from uuid import uuid4
|
|
|
|
|
|
|
|
|
|
+from agent.failures import FailureDetail
|
|
|
|
|
+
|
|
|
from .config import OrchestrationConfig
|
|
from .config import OrchestrationConfig
|
|
|
from .models import (
|
|
from .models import (
|
|
|
AgentRole,
|
|
AgentRole,
|
|
@@ -681,6 +683,7 @@ class TaskCoordinator:
|
|
|
validation_id=result.validation_id,
|
|
validation_id=result.validation_id,
|
|
|
validation=result.validation,
|
|
validation=result.validation,
|
|
|
error="Dispatch with this idempotency key is still in progress",
|
|
error="Dispatch with this idempotency key is still in progress",
|
|
|
|
|
+ failure=result.failure,
|
|
|
)
|
|
)
|
|
|
if errors[index] and result.error != errors[index]:
|
|
if errors[index] and result.error != errors[index]:
|
|
|
result = TaskCycleResult(
|
|
result = TaskCycleResult(
|
|
@@ -690,6 +693,7 @@ class TaskCoordinator:
|
|
|
validation_id=result.validation_id,
|
|
validation_id=result.validation_id,
|
|
|
validation=result.validation,
|
|
validation=result.validation,
|
|
|
error=errors[index],
|
|
error=errors[index],
|
|
|
|
|
+ failure=result.failure,
|
|
|
)
|
|
)
|
|
|
results.append(result)
|
|
results.append(result)
|
|
|
return results
|
|
return results
|
|
@@ -857,6 +861,7 @@ class TaskCoordinator:
|
|
|
validation_id = candidate_id
|
|
validation_id = candidate_id
|
|
|
break
|
|
break
|
|
|
error = validation.error if validation else (attempt.error if attempt else None)
|
|
error = validation.error if validation else (attempt.error if attempt else None)
|
|
|
|
|
+ failure = validation.failure if validation else (attempt.failure if attempt else None)
|
|
|
if task.status in {
|
|
if task.status in {
|
|
|
TaskStatus.RUNNING,
|
|
TaskStatus.RUNNING,
|
|
|
TaskStatus.AWAITING_VALIDATION,
|
|
TaskStatus.AWAITING_VALIDATION,
|
|
@@ -870,6 +875,7 @@ class TaskCoordinator:
|
|
|
validation_id=validation_id,
|
|
validation_id=validation_id,
|
|
|
validation=validation,
|
|
validation=validation,
|
|
|
error=error,
|
|
error=error,
|
|
|
|
|
+ failure=failure,
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
async def _recover_cycle_failure(
|
|
async def _recover_cycle_failure(
|
|
@@ -1174,6 +1180,7 @@ class TaskCoordinator:
|
|
|
)
|
|
)
|
|
|
else:
|
|
else:
|
|
|
worker_result = None
|
|
worker_result = None
|
|
|
|
|
+ prior_feedback = self._prior_feedback(ledger, task, attempt_id)
|
|
|
worker_context = {
|
|
worker_context = {
|
|
|
"root_trace_id": root_trace_id,
|
|
"root_trace_id": root_trace_id,
|
|
|
"task_id": task_id,
|
|
"task_id": task_id,
|
|
@@ -1189,10 +1196,11 @@ class TaskCoordinator:
|
|
|
else None
|
|
else None
|
|
|
),
|
|
),
|
|
|
"repair_feedback": (
|
|
"repair_feedback": (
|
|
|
- self._repair_feedback(ledger, task)
|
|
|
|
|
|
|
+ prior_feedback
|
|
|
if attempt.execution_mode == "repair"
|
|
if attempt.execution_mode == "repair"
|
|
|
else None
|
|
else None
|
|
|
),
|
|
),
|
|
|
|
|
+ "prior_feedback": prior_feedback,
|
|
|
"accepted_child_results": accepted_child_results,
|
|
"accepted_child_results": accepted_child_results,
|
|
|
"operation_id": operation_id,
|
|
"operation_id": operation_id,
|
|
|
"execution_epoch": execution_epoch,
|
|
"execution_epoch": execution_epoch,
|
|
@@ -1247,8 +1255,15 @@ class TaskCoordinator:
|
|
|
assert worker_result is not None
|
|
assert worker_result is not None
|
|
|
stats = _failure_stats(
|
|
stats = _failure_stats(
|
|
|
worker_result.execution_stats,
|
|
worker_result.execution_stats,
|
|
|
- _failure_code(worker_result.status, protocol_failure=True),
|
|
|
|
|
- override=worker_result.status == "completed",
|
|
|
|
|
|
|
+ _failure_code(
|
|
|
|
|
+ worker_result.status,
|
|
|
|
|
+ protocol_failure=True,
|
|
|
|
|
+ failure=worker_result.failure,
|
|
|
|
|
+ ),
|
|
|
|
|
+ override=(
|
|
|
|
|
+ worker_result.status == "completed"
|
|
|
|
|
+ and worker_result.failure is None
|
|
|
|
|
+ ),
|
|
|
)
|
|
)
|
|
|
await self._mark_worker_failure(
|
|
await self._mark_worker_failure(
|
|
|
root_trace_id,
|
|
root_trace_id,
|
|
@@ -1259,16 +1274,24 @@ class TaskCoordinator:
|
|
|
stats,
|
|
stats,
|
|
|
operation_id,
|
|
operation_id,
|
|
|
execution_epoch,
|
|
execution_epoch,
|
|
|
|
|
+ failure=worker_result.failure,
|
|
|
|
|
+ worker_summary=worker_result.summary,
|
|
|
)
|
|
)
|
|
|
|
|
+ current = await self.task_store.load(root_trace_id)
|
|
|
|
|
+ current_attempt = current.attempts[attempt_id]
|
|
|
return TaskCycleResult(
|
|
return TaskCycleResult(
|
|
|
task_id=task_id,
|
|
task_id=task_id,
|
|
|
- task_status=(await self.task_store.load(root_trace_id)).tasks[task_id].status,
|
|
|
|
|
|
|
+ task_status=current.tasks[task_id].status,
|
|
|
attempt_id=attempt_id,
|
|
attempt_id=attempt_id,
|
|
|
error=(
|
|
error=(
|
|
|
- attempt.error
|
|
|
|
|
|
|
+ current_attempt.error
|
|
|
or (worker_result.error if worker_result else None)
|
|
or (worker_result.error if worker_result else None)
|
|
|
or "Worker did not produce a submitted attempt"
|
|
or "Worker did not produce a submitted attempt"
|
|
|
),
|
|
),
|
|
|
|
|
+ failure=(
|
|
|
|
|
+ current_attempt.failure
|
|
|
|
|
+ or (worker_result.failure if worker_result else None)
|
|
|
|
|
+ ),
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
validation_id = self._validation_for_attempt(ledger, task, attempt_id)
|
|
validation_id = self._validation_for_attempt(ledger, task, attempt_id)
|
|
@@ -1330,6 +1353,7 @@ class TaskCoordinator:
|
|
|
validation_id=validation_id,
|
|
validation_id=validation_id,
|
|
|
validation=validation,
|
|
validation=validation,
|
|
|
error=validation.error,
|
|
error=validation.error,
|
|
|
|
|
+ failure=validation.failure,
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
try:
|
|
try:
|
|
@@ -1438,8 +1462,15 @@ class TaskCoordinator:
|
|
|
if validation.status != ValidationRunStatus.COMPLETED:
|
|
if validation.status != ValidationRunStatus.COMPLETED:
|
|
|
stats = _failure_stats(
|
|
stats = _failure_stats(
|
|
|
validator_result.execution_stats,
|
|
validator_result.execution_stats,
|
|
|
- _failure_code(validator_result.status, protocol_failure=True),
|
|
|
|
|
- override=validator_result.status == "completed",
|
|
|
|
|
|
|
+ _failure_code(
|
|
|
|
|
+ validator_result.status,
|
|
|
|
|
+ protocol_failure=True,
|
|
|
|
|
+ failure=validator_result.failure,
|
|
|
|
|
+ ),
|
|
|
|
|
+ override=(
|
|
|
|
|
+ validator_result.status == "completed"
|
|
|
|
|
+ and validator_result.failure is None
|
|
|
|
|
+ ),
|
|
|
)
|
|
)
|
|
|
await self._mark_validation_error(
|
|
await self._mark_validation_error(
|
|
|
root_trace_id,
|
|
root_trace_id,
|
|
@@ -1450,6 +1481,7 @@ class TaskCoordinator:
|
|
|
stats,
|
|
stats,
|
|
|
operation_id,
|
|
operation_id,
|
|
|
execution_epoch,
|
|
execution_epoch,
|
|
|
|
|
+ failure=validator_result.failure,
|
|
|
)
|
|
)
|
|
|
ledger = await self.task_store.load(root_trace_id)
|
|
ledger = await self.task_store.load(root_trace_id)
|
|
|
validation = ledger.validations[validation_id]
|
|
validation = ledger.validations[validation_id]
|
|
@@ -1460,6 +1492,7 @@ class TaskCoordinator:
|
|
|
validation_id=validation_id,
|
|
validation_id=validation_id,
|
|
|
validation=validation,
|
|
validation=validation,
|
|
|
error=validation.error,
|
|
error=validation.error,
|
|
|
|
|
+ failure=validation.failure,
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
async def _run_validation_preflight(
|
|
async def _run_validation_preflight(
|
|
@@ -2567,6 +2600,9 @@ class TaskCoordinator:
|
|
|
stats: ExecutionStats,
|
|
stats: ExecutionStats,
|
|
|
operation_id: Optional[str],
|
|
operation_id: Optional[str],
|
|
|
execution_epoch: int,
|
|
execution_epoch: int,
|
|
|
|
|
+ *,
|
|
|
|
|
+ failure: Optional[FailureDetail] = None,
|
|
|
|
|
+ worker_summary: str = "",
|
|
|
) -> None:
|
|
) -> None:
|
|
|
def mutate(ledger: TaskLedger) -> Dict[str, Any]:
|
|
def mutate(ledger: TaskLedger) -> Dict[str, Any]:
|
|
|
task = _task(ledger, task_id)
|
|
task = _task(ledger, task_id)
|
|
@@ -2581,6 +2617,8 @@ class TaskCoordinator:
|
|
|
status_map = {"stopped": AttemptStatus.STOPPED, "expired": AttemptStatus.EXPIRED}
|
|
status_map = {"stopped": AttemptStatus.STOPPED, "expired": AttemptStatus.EXPIRED}
|
|
|
attempt.status = status_map.get(run_status, AttemptStatus.FAILED)
|
|
attempt.status = status_map.get(run_status, AttemptStatus.FAILED)
|
|
|
attempt.error = error
|
|
attempt.error = error
|
|
|
|
|
+ attempt.failure = failure
|
|
|
|
|
+ attempt.worker_summary = worker_summary
|
|
|
attempt.execution_stats = stats
|
|
attempt.execution_stats = stats
|
|
|
attempt.completed_at = utc_now()
|
|
attempt.completed_at = utc_now()
|
|
|
attempt.updated_at = utc_now()
|
|
attempt.updated_at = utc_now()
|
|
@@ -2601,6 +2639,8 @@ class TaskCoordinator:
|
|
|
stats: ExecutionStats,
|
|
stats: ExecutionStats,
|
|
|
operation_id: Optional[str],
|
|
operation_id: Optional[str],
|
|
|
execution_epoch: int,
|
|
execution_epoch: int,
|
|
|
|
|
+ *,
|
|
|
|
|
+ failure: Optional[FailureDetail] = None,
|
|
|
) -> None:
|
|
) -> None:
|
|
|
def mutate(ledger: TaskLedger) -> Dict[str, Any]:
|
|
def mutate(ledger: TaskLedger) -> Dict[str, Any]:
|
|
|
task = _task(ledger, task_id)
|
|
task = _task(ledger, task_id)
|
|
@@ -2622,6 +2662,7 @@ class TaskCoordinator:
|
|
|
}
|
|
}
|
|
|
report.status = status_map.get(run_status, ValidationRunStatus.ERROR)
|
|
report.status = status_map.get(run_status, ValidationRunStatus.ERROR)
|
|
|
report.error = error
|
|
report.error = error
|
|
|
|
|
+ report.failure = failure
|
|
|
report.execution_stats = stats
|
|
report.execution_stats = stats
|
|
|
report.completed_at = utc_now()
|
|
report.completed_at = utc_now()
|
|
|
report.updated_at = utc_now()
|
|
report.updated_at = utc_now()
|
|
@@ -2633,17 +2674,54 @@ class TaskCoordinator:
|
|
|
await self._mutate(root_trace_id, "validation_error", mutate)
|
|
await self._mutate(root_trace_id, "validation_error", mutate)
|
|
|
|
|
|
|
|
@staticmethod
|
|
@staticmethod
|
|
|
- def _repair_feedback(ledger: TaskLedger, task: TaskRecord) -> Optional[Dict[str, Any]]:
|
|
|
|
|
- if not task.validation_ids:
|
|
|
|
|
|
|
+ def _prior_feedback(
|
|
|
|
|
+ ledger: TaskLedger,
|
|
|
|
|
+ task: TaskRecord,
|
|
|
|
|
+ current_attempt_id: str,
|
|
|
|
|
+ ) -> Optional[Dict[str, Any]]:
|
|
|
|
|
+ previous_ids = [
|
|
|
|
|
+ attempt_id
|
|
|
|
|
+ for attempt_id in task.attempt_ids
|
|
|
|
|
+ if attempt_id != current_attempt_id
|
|
|
|
|
+ ]
|
|
|
|
|
+ if not previous_ids:
|
|
|
return None
|
|
return None
|
|
|
- report = ledger.validations[task.validation_ids[-1]]
|
|
|
|
|
- return {
|
|
|
|
|
- "verdict": report.verdict.value if report.verdict else None,
|
|
|
|
|
- "summary": report.summary,
|
|
|
|
|
- "criterion_results": [_plain(asdict(x)) for x in report.criterion_results],
|
|
|
|
|
- "risks": report.risks,
|
|
|
|
|
- "recommendation": report.recommendation,
|
|
|
|
|
|
|
+ attempt = ledger.attempts[previous_ids[-1]]
|
|
|
|
|
+ report = next(
|
|
|
|
|
+ (
|
|
|
|
|
+ ledger.validations[validation_id]
|
|
|
|
|
+ for validation_id in reversed(task.validation_ids)
|
|
|
|
|
+ if ledger.validations[validation_id].attempt_id == attempt.attempt_id
|
|
|
|
|
+ ),
|
|
|
|
|
+ None,
|
|
|
|
|
+ )
|
|
|
|
|
+ feedback: Dict[str, Any] = {
|
|
|
|
|
+ "attempt": {
|
|
|
|
|
+ "attempt_id": attempt.attempt_id,
|
|
|
|
|
+ "status": attempt.status.value,
|
|
|
|
|
+ "error": attempt.error,
|
|
|
|
|
+ "failure": attempt.failure.to_dict() if attempt.failure else None,
|
|
|
|
|
+ "worker_trace_id": attempt.worker_trace_id,
|
|
|
|
|
+ "worker_summary": attempt.worker_summary,
|
|
|
|
|
+ "artifact_submitted": attempt.submission is not None,
|
|
|
|
|
+ "snapshot_id": attempt.snapshot_id,
|
|
|
|
|
+ },
|
|
|
}
|
|
}
|
|
|
|
|
+ if report is not None:
|
|
|
|
|
+ feedback["validation"] = {
|
|
|
|
|
+ "validation_id": report.validation_id,
|
|
|
|
|
+ "status": report.status.value,
|
|
|
|
|
+ "verdict": report.verdict.value if report.verdict else None,
|
|
|
|
|
+ "summary": report.summary,
|
|
|
|
|
+ "criterion_results": [
|
|
|
|
|
+ _plain(asdict(item)) for item in report.criterion_results
|
|
|
|
|
+ ],
|
|
|
|
|
+ "risks": report.risks,
|
|
|
|
|
+ "recommendation": report.recommendation,
|
|
|
|
|
+ "error": report.error,
|
|
|
|
|
+ "failure": report.failure.to_dict() if report.failure else None,
|
|
|
|
|
+ }
|
|
|
|
|
+ return feedback
|
|
|
|
|
|
|
|
@staticmethod
|
|
@staticmethod
|
|
|
def _accepted_child_results(
|
|
def _accepted_child_results(
|
|
@@ -2866,7 +2944,18 @@ def _cas_backoff(attempt_number: int) -> float:
|
|
|
return min(0.005 * (2**attempt_number), 0.1)
|
|
return min(0.005 * (2**attempt_number), 0.1)
|
|
|
|
|
|
|
|
|
|
|
|
|
-def _failure_code(run_status: str, *, protocol_failure: bool) -> FailureCode:
|
|
|
|
|
|
|
+def _failure_code(
|
|
|
|
|
+ run_status: str,
|
|
|
|
|
+ *,
|
|
|
|
|
+ protocol_failure: bool,
|
|
|
|
|
+ failure: Optional[FailureDetail] = None,
|
|
|
|
|
+) -> FailureCode:
|
|
|
|
|
+ if failure is not None:
|
|
|
|
|
+ if failure.code == "NO_PROGRESS":
|
|
|
|
|
+ return FailureCode.NO_PROGRESS
|
|
|
|
|
+ if failure.code == "PROTOCOL_VIOLATION":
|
|
|
|
|
+ return FailureCode.PROTOCOL_VIOLATION
|
|
|
|
|
+ return FailureCode.TOOL_FAILURE
|
|
|
if protocol_failure and run_status == "completed":
|
|
if protocol_failure and run_status == "completed":
|
|
|
return FailureCode.PROTOCOL_VIOLATION
|
|
return FailureCode.PROTOCOL_VIOLATION
|
|
|
return {
|
|
return {
|