|
|
@@ -187,25 +187,13 @@ class TaskCoordinator:
|
|
|
parent_path = ledger.tasks[effective_parent_id].display_path if effective_parent_id else ""
|
|
|
created: List[str] = []
|
|
|
for offset, draft in enumerate(drafts):
|
|
|
- objective = str(draft.get("objective", "")).strip()
|
|
|
- if not objective:
|
|
|
- raise ValueError("Task objective is required")
|
|
|
- task_id = new_id()
|
|
|
- criteria = [AcceptanceCriterion.from_dict(x) for x in draft.get("acceptance_criteria", [])]
|
|
|
- spec = TaskSpec(
|
|
|
- version=1,
|
|
|
- objective=objective,
|
|
|
- acceptance_criteria=criteria,
|
|
|
- context_refs=list(draft.get("context_refs", [])),
|
|
|
- )
|
|
|
index = start + offset
|
|
|
display_path = f"{parent_path}.{index}" if parent_path else str(index)
|
|
|
- ledger.tasks[task_id] = TaskRecord(
|
|
|
- task_id=task_id,
|
|
|
- goal_id=None,
|
|
|
- parent_task_id=effective_parent_id,
|
|
|
- display_path=display_path,
|
|
|
- specs=[spec],
|
|
|
+ task_id = self._add_task_record(
|
|
|
+ ledger,
|
|
|
+ draft,
|
|
|
+ effective_parent_id,
|
|
|
+ display_path,
|
|
|
)
|
|
|
if effective_parent_id:
|
|
|
children = ledger.tasks[effective_parent_id].child_task_ids
|
|
|
@@ -226,7 +214,12 @@ class TaskCoordinator:
|
|
|
result = await self._mutate(root_trace_id, "tasks_created", mutate, idempotency_key)
|
|
|
after_task_id = result.get("after_task_id")
|
|
|
for task_id in result["task_ids"]:
|
|
|
- await self._ensure_goal_projection(root_trace_id, task_id, after_task_id=after_task_id)
|
|
|
+ await self._project_goal_compatibility(
|
|
|
+ root_trace_id,
|
|
|
+ task_id,
|
|
|
+ ensure=True,
|
|
|
+ after_task_id=after_task_id,
|
|
|
+ )
|
|
|
after_task_id = task_id
|
|
|
return await self._tasks_result(root_trace_id, result["task_ids"])
|
|
|
|
|
|
@@ -237,13 +230,17 @@ class TaskCoordinator:
|
|
|
return {"task_id": task_id, "status": task.status.value}
|
|
|
|
|
|
result = await self._mutate(root_trace_id, "task_focused", mutate)
|
|
|
- ledger = await self.task_store.load(root_trace_id)
|
|
|
- task = ledger.tasks[task_id]
|
|
|
- if self.trace_store and task.goal_id:
|
|
|
- tree = await self.trace_store.get_goal_tree(root_trace_id)
|
|
|
- if tree and tree.find(task.goal_id):
|
|
|
- tree.focus(task.goal_id)
|
|
|
- await self.trace_store.update_goal_tree(root_trace_id, tree)
|
|
|
+ if self.trace_store:
|
|
|
+ try:
|
|
|
+ ledger = await self.task_store.load(root_trace_id)
|
|
|
+ task = ledger.tasks[task_id]
|
|
|
+ if task.goal_id:
|
|
|
+ tree = await self.trace_store.get_goal_tree(root_trace_id)
|
|
|
+ if tree and tree.find(task.goal_id):
|
|
|
+ tree.focus(task.goal_id)
|
|
|
+ await self.trace_store.update_goal_tree(root_trace_id, tree)
|
|
|
+ except Exception as exc:
|
|
|
+ self._log_projection_failure(root_trace_id, task_id, exc)
|
|
|
return result
|
|
|
|
|
|
async def dispatch_tasks(
|
|
|
@@ -271,16 +268,47 @@ class TaskCoordinator:
|
|
|
if full_idem:
|
|
|
existing = (await self.task_store.load(root_trace_id)).idempotency_results.get(full_idem)
|
|
|
if existing and "results" in existing:
|
|
|
+ recorded_task_ids = existing.get("task_ids")
|
|
|
+ recorded_presets = existing.get("worker_presets")
|
|
|
+ if recorded_task_ids is not None and list(recorded_task_ids) != list(task_ids):
|
|
|
+ raise TaskConflict("Idempotency key is bound to different task_ids")
|
|
|
+ if recorded_presets is not None and list(recorded_presets) != presets:
|
|
|
+ raise TaskConflict("Idempotency key is bound to different worker_presets")
|
|
|
return [TaskCycleResult.from_dict(x) for x in existing["results"]]
|
|
|
|
|
|
# Reserve every task before starting any worker so concurrent duplicate
|
|
|
# dispatches fail deterministically.
|
|
|
+ dispatch_owner = str(uuid4())
|
|
|
reservations: List[Tuple[int, str, str]] = []
|
|
|
early_results: Dict[int, TaskCycleResult] = {}
|
|
|
+ replayed_indices = set()
|
|
|
for index, (task_id, preset) in enumerate(zip(task_ids, presets)):
|
|
|
key = f"{idempotency_key}:{index}" if idempotency_key else None
|
|
|
try:
|
|
|
- attempt = await self._create_attempt(root_trace_id, task_id, preset, key)
|
|
|
+ attempt = await self._create_attempt(
|
|
|
+ root_trace_id,
|
|
|
+ task_id,
|
|
|
+ preset,
|
|
|
+ key,
|
|
|
+ reservation_owner=dispatch_owner,
|
|
|
+ )
|
|
|
+ if attempt.get("task_id") != task_id:
|
|
|
+ raise TaskConflict(
|
|
|
+ "Dispatch item idempotency key is bound to a different task"
|
|
|
+ )
|
|
|
+ recorded_preset = attempt.get("worker_preset")
|
|
|
+ if recorded_preset is not None and recorded_preset != preset:
|
|
|
+ raise TaskConflict(
|
|
|
+ "Dispatch item idempotency key is bound to a different worker preset"
|
|
|
+ )
|
|
|
+ if key and attempt.get("reservation_owner") != dispatch_owner:
|
|
|
+ replayed_indices.add(index)
|
|
|
+ early_results[index] = await self._cycle_result_from_ledger(
|
|
|
+ root_trace_id,
|
|
|
+ task_id,
|
|
|
+ attempt.get("attempt_id"),
|
|
|
+ )
|
|
|
+ continue
|
|
|
reservations.append((index, task_id, attempt["attempt_id"]))
|
|
|
except TaskConflict as exc:
|
|
|
# A conflict belongs to this item only; previously reserved
|
|
|
@@ -321,12 +349,79 @@ class TaskCoordinator:
|
|
|
indexed_results = dict(early_results)
|
|
|
indexed_results.update(completed)
|
|
|
results = [indexed_results[index] for index in range(len(task_ids))]
|
|
|
- if idempotency_key:
|
|
|
+ replay_in_progress = any(
|
|
|
+ early_results[index].task_status in {
|
|
|
+ TaskStatus.RUNNING,
|
|
|
+ TaskStatus.AWAITING_VALIDATION,
|
|
|
+ TaskStatus.VALIDATING,
|
|
|
+ }
|
|
|
+ for index in replayed_indices
|
|
|
+ )
|
|
|
+ if idempotency_key and not replay_in_progress:
|
|
|
def remember(ledger: TaskLedger) -> Dict[str, Any]:
|
|
|
- return {"results": [x.to_dict() for x in results]}
|
|
|
- await self._mutate(root_trace_id, "dispatch_completed", remember, idempotency_key)
|
|
|
+ return {
|
|
|
+ "results": [x.to_dict() for x in results],
|
|
|
+ "task_ids": list(task_ids),
|
|
|
+ "worker_presets": presets,
|
|
|
+ }
|
|
|
+ try:
|
|
|
+ await self._mutate(
|
|
|
+ root_trace_id,
|
|
|
+ "dispatch_completed",
|
|
|
+ remember,
|
|
|
+ idempotency_key,
|
|
|
+ )
|
|
|
+ except Exception as exc:
|
|
|
+ # Every Task cycle has already reached its own durable state.
|
|
|
+ # Returning those branch results is safer than making callers
|
|
|
+ # retry completed Workers because only the aggregate cache
|
|
|
+ # failed. A retry replays per-item reservations and can persist
|
|
|
+ # the aggregate result without executing the Agents again.
|
|
|
+ logger.warning(
|
|
|
+ "Dispatch result persistence failed after Task cycles "
|
|
|
+ "completed (%s, %s): %s",
|
|
|
+ root_trace_id,
|
|
|
+ idempotency_key,
|
|
|
+ exc,
|
|
|
+ )
|
|
|
return results
|
|
|
|
|
|
+ async def _cycle_result_from_ledger(
|
|
|
+ self,
|
|
|
+ root_trace_id: str,
|
|
|
+ task_id: str,
|
|
|
+ attempt_id: Optional[str],
|
|
|
+ ) -> TaskCycleResult:
|
|
|
+ """Rebuild an idempotent dispatch result without rerunning Agents."""
|
|
|
+
|
|
|
+ ledger = await self.task_store.load(root_trace_id)
|
|
|
+ task = _task(ledger, task_id)
|
|
|
+ attempt = ledger.attempts.get(attempt_id) if attempt_id else None
|
|
|
+ validation = None
|
|
|
+ validation_id = None
|
|
|
+ if attempt_id:
|
|
|
+ for candidate_id in reversed(task.validation_ids):
|
|
|
+ candidate = ledger.validations[candidate_id]
|
|
|
+ if candidate.attempt_id == attempt_id:
|
|
|
+ validation = candidate
|
|
|
+ validation_id = candidate_id
|
|
|
+ break
|
|
|
+ error = validation.error if validation else (attempt.error if attempt else None)
|
|
|
+ if task.status in {
|
|
|
+ TaskStatus.RUNNING,
|
|
|
+ TaskStatus.AWAITING_VALIDATION,
|
|
|
+ TaskStatus.VALIDATING,
|
|
|
+ }:
|
|
|
+ error = error or "Dispatch with this idempotency key is still in progress"
|
|
|
+ return TaskCycleResult(
|
|
|
+ task_id=task_id,
|
|
|
+ task_status=task.status,
|
|
|
+ attempt_id=attempt_id,
|
|
|
+ validation_id=validation_id,
|
|
|
+ validation=validation,
|
|
|
+ error=error,
|
|
|
+ )
|
|
|
+
|
|
|
async def _recover_cycle_failure(
|
|
|
self,
|
|
|
root_trace_id: str,
|
|
|
@@ -375,12 +470,12 @@ class TaskCoordinator:
|
|
|
|
|
|
try:
|
|
|
await self._mutate(root_trace_id, "task_cycle_failed", mutate)
|
|
|
- await self.project_goal_state(root_trace_id, task_id)
|
|
|
except Exception:
|
|
|
# The original adapter failure is the useful branch result. A
|
|
|
# persistent Store outage cannot be repaired in-process (V2), but
|
|
|
# must not cancel sibling Tasks in this batch.
|
|
|
return
|
|
|
+ await self._project_goal_compatibility(root_trace_id, task_id)
|
|
|
|
|
|
async def _cycle_error_result(
|
|
|
self,
|
|
|
@@ -421,6 +516,7 @@ class TaskCoordinator:
|
|
|
task_id: str,
|
|
|
worker_preset: str,
|
|
|
idempotency_key: Optional[str],
|
|
|
+ reservation_owner: Optional[str] = None,
|
|
|
) -> Dict[str, Any]:
|
|
|
def mutate(ledger: TaskLedger) -> Dict[str, Any]:
|
|
|
task = _task(ledger, task_id)
|
|
|
@@ -461,21 +557,13 @@ class TaskCoordinator:
|
|
|
"task_id": task_id,
|
|
|
"attempt_id": attempt.attempt_id,
|
|
|
"worker_trace_id": worker_trace_id,
|
|
|
+ "worker_preset": worker_preset,
|
|
|
"execution_mode": execution_mode,
|
|
|
+ "reservation_owner": reservation_owner,
|
|
|
}
|
|
|
|
|
|
result = await self._mutate(root_trace_id, "attempt_created", mutate, idempotency_key)
|
|
|
- try:
|
|
|
- await self.project_goal_state(root_trace_id, task_id)
|
|
|
- except Exception as exc:
|
|
|
- # GoalTree is a compatibility projection. The committed Attempt
|
|
|
- # remains authoritative and can execute; reconcile_goal_tree can
|
|
|
- # repair the projection later.
|
|
|
- logger.warning(
|
|
|
- "Goal projection failed after Attempt %s was committed: %s",
|
|
|
- result["attempt_id"],
|
|
|
- exc,
|
|
|
- )
|
|
|
+ await self._project_goal_compatibility(root_trace_id, task_id)
|
|
|
return result
|
|
|
|
|
|
async def _run_task_cycle(
|
|
|
@@ -624,7 +712,7 @@ class TaskCoordinator:
|
|
|
}
|
|
|
|
|
|
result = await self._mutate(root_trace_id, "attempt_submitted", mutate, tool_call_id)
|
|
|
- await self.project_goal_state(root_trace_id, task_id)
|
|
|
+ await self._project_goal_compatibility(root_trace_id, task_id)
|
|
|
return result
|
|
|
|
|
|
async def _start_validation(self, root_trace_id: str, task_id: str, attempt_id: str) -> str:
|
|
|
@@ -651,7 +739,7 @@ class TaskCoordinator:
|
|
|
return {"validation_id": validation.validation_id, "task_id": task_id}
|
|
|
|
|
|
result = await self._mutate(root_trace_id, "validation_started", mutate)
|
|
|
- await self.project_goal_state(root_trace_id, task_id)
|
|
|
+ await self._project_goal_compatibility(root_trace_id, task_id)
|
|
|
return result["validation_id"]
|
|
|
|
|
|
async def submit_validation(
|
|
|
@@ -714,7 +802,7 @@ class TaskCoordinator:
|
|
|
}
|
|
|
|
|
|
result = await self._mutate(root_trace_id, "validation_submitted", mutate, tool_call_id)
|
|
|
- await self.project_goal_state(root_trace_id, task_id)
|
|
|
+ await self._project_goal_compatibility(root_trace_id, task_id)
|
|
|
return result
|
|
|
|
|
|
@staticmethod
|
|
|
@@ -851,18 +939,24 @@ class TaskCoordinator:
|
|
|
}
|
|
|
|
|
|
result = await self._mutate(root_trace_id, "planner_decision", mutate, idempotency_key)
|
|
|
- ledger = await self.task_store.load(root_trace_id)
|
|
|
affected = [task_id]
|
|
|
affected.extend(result.get("payload", {}).get("child_task_ids", []))
|
|
|
replacement = result.get("payload", {}).get("replacement_task_id")
|
|
|
if replacement:
|
|
|
affected.append(replacement)
|
|
|
- parent_id = ledger.tasks[task_id].parent_task_id
|
|
|
- if parent_id:
|
|
|
- affected.append(parent_id)
|
|
|
+ try:
|
|
|
+ ledger = await self.task_store.load(root_trace_id)
|
|
|
+ parent_id = ledger.tasks[task_id].parent_task_id
|
|
|
+ if parent_id:
|
|
|
+ affected.append(parent_id)
|
|
|
+ except Exception as exc:
|
|
|
+ self._log_projection_failure(root_trace_id, task_id, exc)
|
|
|
for affected_task in dict.fromkeys(affected):
|
|
|
- await self._ensure_goal_projection(root_trace_id, affected_task)
|
|
|
- await self.project_goal_state(root_trace_id, affected_task)
|
|
|
+ await self._project_goal_compatibility(
|
|
|
+ root_trace_id,
|
|
|
+ affected_task,
|
|
|
+ ensure=True,
|
|
|
+ )
|
|
|
return result
|
|
|
|
|
|
@staticmethod
|
|
|
@@ -940,6 +1034,7 @@ class TaskCoordinator:
|
|
|
|
|
|
result = await self._mutate(root_trace_id, "validation_restarted", mutate, idempotency_key)
|
|
|
validation_id = result["validation_id"]
|
|
|
+ await self._project_goal_compatibility(root_trace_id, task_id)
|
|
|
ledger = await self.task_store.load(root_trace_id)
|
|
|
task = ledger.tasks[task_id]
|
|
|
attempt = ledger.attempts[attempt_id]
|
|
|
@@ -1039,7 +1134,7 @@ class TaskCoordinator:
|
|
|
return {"task_id": task_id, "attempt_id": attempt_id, "error": error}
|
|
|
|
|
|
await self._mutate(root_trace_id, "worker_failed", mutate)
|
|
|
- await self.project_goal_state(root_trace_id, task_id)
|
|
|
+ await self._project_goal_compatibility(root_trace_id, task_id)
|
|
|
|
|
|
async def _mark_validation_error(
|
|
|
self,
|
|
|
@@ -1065,7 +1160,7 @@ class TaskCoordinator:
|
|
|
return {"task_id": task_id, "validation_id": validation_id, "error": error}
|
|
|
|
|
|
await self._mutate(root_trace_id, "validation_error", mutate)
|
|
|
- await self.project_goal_state(root_trace_id, task_id)
|
|
|
+ await self._project_goal_compatibility(root_trace_id, task_id)
|
|
|
|
|
|
@staticmethod
|
|
|
def _repair_feedback(ledger: TaskLedger, task: TaskRecord) -> Optional[Dict[str, Any]]:
|
|
|
@@ -1098,7 +1193,36 @@ class TaskCoordinator:
|
|
|
return self._create_records(ledger, drafts, task.parent_task_id, parent_path)
|
|
|
|
|
|
@staticmethod
|
|
|
+ def _add_task_record(
|
|
|
+ ledger: TaskLedger,
|
|
|
+ draft: Dict[str, Any],
|
|
|
+ parent_task_id: Optional[str],
|
|
|
+ display_path: str,
|
|
|
+ ) -> str:
|
|
|
+ objective = str(draft.get("objective", "")).strip()
|
|
|
+ if not objective:
|
|
|
+ raise ValueError("Task objective is required")
|
|
|
+ task_id = new_id()
|
|
|
+ spec = TaskSpec(
|
|
|
+ version=1,
|
|
|
+ objective=objective,
|
|
|
+ acceptance_criteria=[
|
|
|
+ AcceptanceCriterion.from_dict(item)
|
|
|
+ for item in draft.get("acceptance_criteria", [])
|
|
|
+ ],
|
|
|
+ context_refs=list(draft.get("context_refs", [])),
|
|
|
+ )
|
|
|
+ ledger.tasks[task_id] = TaskRecord(
|
|
|
+ task_id=task_id,
|
|
|
+ goal_id=None,
|
|
|
+ parent_task_id=parent_task_id,
|
|
|
+ display_path=display_path,
|
|
|
+ specs=[spec],
|
|
|
+ )
|
|
|
+ return task_id
|
|
|
+
|
|
|
def _create_records(
|
|
|
+ self,
|
|
|
ledger: TaskLedger,
|
|
|
drafts: Sequence[Dict[str, Any]],
|
|
|
parent_task_id: Optional[str],
|
|
|
@@ -1110,21 +1234,13 @@ class TaskCoordinator:
|
|
|
)
|
|
|
created: List[str] = []
|
|
|
for offset, draft in enumerate(drafts, start=1):
|
|
|
- objective = str(draft.get("objective", "")).strip()
|
|
|
- if not objective:
|
|
|
- raise ValueError("New task objective is required")
|
|
|
- task_id = new_id()
|
|
|
index = len(siblings) + offset
|
|
|
display_path = f"{parent_path}.{index}" if parent_path else str(index)
|
|
|
- spec = TaskSpec(
|
|
|
- version=1,
|
|
|
- objective=objective,
|
|
|
- acceptance_criteria=[AcceptanceCriterion.from_dict(x) for x in draft.get("acceptance_criteria", [])],
|
|
|
- context_refs=list(draft.get("context_refs", [])),
|
|
|
- )
|
|
|
- ledger.tasks[task_id] = TaskRecord(
|
|
|
- task_id=task_id, goal_id=None, parent_task_id=parent_task_id,
|
|
|
- display_path=display_path, specs=[spec],
|
|
|
+ task_id = self._add_task_record(
|
|
|
+ ledger,
|
|
|
+ draft,
|
|
|
+ parent_task_id,
|
|
|
+ display_path,
|
|
|
)
|
|
|
if parent_task_id:
|
|
|
ledger.tasks[parent_task_id].child_task_ids.append(task_id)
|
|
|
@@ -1180,6 +1296,48 @@ class TaskCoordinator:
|
|
|
summary=summary,
|
|
|
)
|
|
|
|
|
|
+ async def _project_goal_compatibility(
|
|
|
+ self,
|
|
|
+ root_trace_id: str,
|
|
|
+ task_id: str,
|
|
|
+ *,
|
|
|
+ ensure: bool = False,
|
|
|
+ after_task_id: Optional[str] = None,
|
|
|
+ ) -> None:
|
|
|
+ """Best-effort GoalTree projection after an authoritative commit.
|
|
|
+
|
|
|
+ Automatic projection must never turn a successful Ledger mutation into
|
|
|
+ an apparent command failure. Explicit reconcile_goal_tree() deliberately
|
|
|
+ keeps using the strict helpers so operators can observe repair failures.
|
|
|
+ """
|
|
|
+
|
|
|
+ if not self.trace_store:
|
|
|
+ return
|
|
|
+ try:
|
|
|
+ if ensure:
|
|
|
+ await self._ensure_goal_projection(
|
|
|
+ root_trace_id,
|
|
|
+ task_id,
|
|
|
+ after_task_id=after_task_id,
|
|
|
+ )
|
|
|
+ await self.project_goal_state(root_trace_id, task_id)
|
|
|
+ except Exception as exc:
|
|
|
+ self._log_projection_failure(root_trace_id, task_id, exc)
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _log_projection_failure(
|
|
|
+ root_trace_id: str,
|
|
|
+ task_id: str,
|
|
|
+ error: Exception,
|
|
|
+ ) -> None:
|
|
|
+ logger.warning(
|
|
|
+ "Goal compatibility projection failed after Ledger commit "
|
|
|
+ "(%s, %s): %s",
|
|
|
+ root_trace_id,
|
|
|
+ task_id,
|
|
|
+ error,
|
|
|
+ )
|
|
|
+
|
|
|
async def _ensure_goal_projection(
|
|
|
self,
|
|
|
root_trace_id: str,
|