|
|
@@ -8,6 +8,10 @@ from typing import Any
|
|
|
from agent.orchestration import OperationStatus, TaskStatus
|
|
|
|
|
|
from script_build_host.application.phase_three import PhaseThreeContinuationService
|
|
|
+from script_build_host.application.phase_two_planning import (
|
|
|
+ PHASE_ONE_CAPABILITY_BOUNDARY,
|
|
|
+ PHASE_TWO_CANDIDATE_PORTFOLIO_READY,
|
|
|
+)
|
|
|
from script_build_host.application.publisher import ScriptPublisher
|
|
|
from script_build_host.domain.artifacts import ArtifactKind, ArtifactState
|
|
|
from script_build_host.domain.errors import ProtocolViolation, PublicationStateInconsistent
|
|
|
@@ -98,6 +102,7 @@ class PhaseThreeRecoveryService:
|
|
|
finalization: PhaseThreeFinalizationService,
|
|
|
artifacts: ScriptBusinessArtifactRepository | None = None,
|
|
|
legacy_projection: Any | None = None,
|
|
|
+ mission_service: Any | None = None,
|
|
|
) -> None:
|
|
|
self.coordinator = coordinator
|
|
|
self.bindings = bindings
|
|
|
@@ -108,6 +113,7 @@ class PhaseThreeRecoveryService:
|
|
|
self.finalization = finalization
|
|
|
self.artifacts = artifacts
|
|
|
self.legacy_projection = legacy_projection
|
|
|
+ self.mission_service = mission_service
|
|
|
|
|
|
async def resume(self, script_build_id: int, principal: Principal) -> ResumeResult:
|
|
|
await self.authorizer.require_access(principal, script_build_id)
|
|
|
@@ -116,6 +122,23 @@ class PhaseThreeRecoveryService:
|
|
|
publication = await self.publications.get_by_build(
|
|
|
script_build_id, publication_type=PublicationType.FINAL
|
|
|
)
|
|
|
+ if status in {BuildStatus.STOPPING, BuildStatus.STOPPED}:
|
|
|
+ if status is BuildStatus.STOPPING and self.mission_service is not None:
|
|
|
+ stopped = await self.mission_service.stop(script_build_id, principal)
|
|
|
+ status = stopped.status
|
|
|
+ return ResumeResult(
|
|
|
+ script_build_id,
|
|
|
+ RecoveryClassification.CONVERGE_STOP,
|
|
|
+ status,
|
|
|
+ "stop intent is terminal and cannot be resumed",
|
|
|
+ )
|
|
|
+ if publication is not None and publication.state is PublicationState.PUBLISHING:
|
|
|
+ return ResumeResult(
|
|
|
+ script_build_id,
|
|
|
+ RecoveryClassification.MANUAL_RECONCILIATION,
|
|
|
+ status,
|
|
|
+ "a publishing transaction requires explicit reconciliation",
|
|
|
+ )
|
|
|
if status is BuildStatus.SUCCESS:
|
|
|
if publication is None or publication.state is not PublicationState.PUBLISHED:
|
|
|
raise PublicationStateInconsistent()
|
|
|
@@ -165,33 +188,70 @@ class PhaseThreeRecoveryService:
|
|
|
RecoveryClassification.FINALIZE_ONLY,
|
|
|
BuildStatus.SUCCESS,
|
|
|
)
|
|
|
+ if root.status is TaskStatus.BLOCKED:
|
|
|
+ if root.blocked_reason == PHASE_ONE_CAPABILITY_BOUNDARY:
|
|
|
+ if self.mission_service is None:
|
|
|
+ return ResumeResult(
|
|
|
+ script_build_id,
|
|
|
+ RecoveryClassification.MANUAL_RECONCILIATION,
|
|
|
+ status,
|
|
|
+ "phase-two continuation is not configured",
|
|
|
+ )
|
|
|
+ await self.mission_service.advance_to_phase_two(script_build_id, principal)
|
|
|
+ return ResumeResult(
|
|
|
+ script_build_id,
|
|
|
+ RecoveryClassification.ADVANCE_PHASE_TWO,
|
|
|
+ BuildStatus.RUNNING,
|
|
|
+ "phase-two continuation started",
|
|
|
+ )
|
|
|
+ if root.blocked_reason == PHASE_TWO_CANDIDATE_PORTFOLIO_READY:
|
|
|
+ await self.continuation.advance(script_build_id, principal)
|
|
|
+ return ResumeResult(
|
|
|
+ script_build_id,
|
|
|
+ RecoveryClassification.ADVANCE_PHASE_THREE,
|
|
|
+ BuildStatus.RUNNING,
|
|
|
+ "phase-three continuation started",
|
|
|
+ )
|
|
|
+ return ResumeResult(
|
|
|
+ script_build_id,
|
|
|
+ RecoveryClassification.MANUAL_RECONCILIATION,
|
|
|
+ status,
|
|
|
+ "Root is blocked outside a recognized phase boundary",
|
|
|
+ )
|
|
|
active = [
|
|
|
item
|
|
|
for item in ledger.operations.values()
|
|
|
if item.status
|
|
|
in {OperationStatus.PENDING, OperationStatus.RUNNING, OperationStatus.STOP_REQUESTED}
|
|
|
]
|
|
|
- if any(item.status is OperationStatus.PENDING for item in active):
|
|
|
- return ResumeResult(
|
|
|
- script_build_id,
|
|
|
- RecoveryClassification.RESUME_PENDING_OPERATION,
|
|
|
- status,
|
|
|
- "pending durable Operation requires the framework recovery hook",
|
|
|
- )
|
|
|
if active:
|
|
|
+ await self.coordinator.recover_operations(binding.root_trace_id)
|
|
|
+ ledger = await self.coordinator.task_store.load(binding.root_trace_id)
|
|
|
+ if any(
|
|
|
+ item.status
|
|
|
+ in {
|
|
|
+ OperationStatus.PENDING,
|
|
|
+ OperationStatus.RUNNING,
|
|
|
+ OperationStatus.STOP_REQUESTED,
|
|
|
+ }
|
|
|
+ for item in ledger.operations.values()
|
|
|
+ ):
|
|
|
+ return ResumeResult(
|
|
|
+ script_build_id,
|
|
|
+ RecoveryClassification.MANUAL_RECONCILIATION,
|
|
|
+ status,
|
|
|
+ "durable Operations did not normalize",
|
|
|
+ )
|
|
|
+ if self.mission_service is not None and status in {
|
|
|
+ BuildStatus.RUNNING,
|
|
|
+ BuildStatus.PARTIAL,
|
|
|
+ }:
|
|
|
+ phase = await self.mission_service.resume_planner(script_build_id)
|
|
|
return ResumeResult(
|
|
|
script_build_id,
|
|
|
- RecoveryClassification.REPLAN_REQUIRED,
|
|
|
- status,
|
|
|
- "started execution must normalize before a new Attempt",
|
|
|
- )
|
|
|
- if root.status is TaskStatus.BLOCKED:
|
|
|
- await self.continuation.advance(script_build_id, principal)
|
|
|
- return ResumeResult(
|
|
|
- script_build_id,
|
|
|
- RecoveryClassification.REPLAN_REQUIRED,
|
|
|
- BuildStatus.RUNNING,
|
|
|
- "phase-three continuation started",
|
|
|
+ RecoveryClassification.RESUME_PLANNER,
|
|
|
+ BuildStatus.PARTIAL,
|
|
|
+ f"phase-{phase} Planner resumed on the durable Trace",
|
|
|
)
|
|
|
return ResumeResult(
|
|
|
script_build_id,
|
|
|
@@ -221,4 +281,10 @@ class PhaseThreeRecoveryService:
|
|
|
)
|
|
|
|
|
|
|
|
|
-__all__ = ["PhaseThreeFinalizationService", "PhaseThreeRecoveryService"]
|
|
|
+MissionRecoveryService = PhaseThreeRecoveryService
|
|
|
+
|
|
|
+__all__ = [
|
|
|
+ "MissionRecoveryService",
|
|
|
+ "PhaseThreeFinalizationService",
|
|
|
+ "PhaseThreeRecoveryService",
|
|
|
+]
|