|
|
@@ -36,7 +36,7 @@ class CandidateStateError(ValueError):
|
|
|
class CandidateService:
|
|
|
"""Validate, persist, and recover candidate version operations."""
|
|
|
|
|
|
- _locks: "WeakValueDictionary[str, asyncio.Lock]" = WeakValueDictionary()
|
|
|
+ _locks: "WeakValueDictionary[tuple[asyncio.AbstractEventLoop, int, str], asyncio.Lock]" = WeakValueDictionary()
|
|
|
|
|
|
def __init__(
|
|
|
self,
|
|
|
@@ -53,12 +53,12 @@ class CandidateService:
|
|
|
self.artifact_resolver = artifact_resolver
|
|
|
self.event_service = event_service
|
|
|
|
|
|
- @classmethod
|
|
|
- def _lock_for(cls, root_trace_id: str) -> asyncio.Lock:
|
|
|
- lock = cls._locks.get(root_trace_id)
|
|
|
+ def _lock_for(self, root_trace_id: str) -> asyncio.Lock:
|
|
|
+ key = (asyncio.get_running_loop(), id(self.store), root_trace_id)
|
|
|
+ lock = self._locks.get(key)
|
|
|
if lock is None:
|
|
|
lock = asyncio.Lock()
|
|
|
- cls._locks[root_trace_id] = lock
|
|
|
+ self._locks[key] = lock
|
|
|
return lock
|
|
|
|
|
|
async def manage(
|
|
|
@@ -69,6 +69,7 @@ class CandidateService:
|
|
|
content: Any,
|
|
|
parent_refs: list[CandidatePointer],
|
|
|
effective_at_sequence: int,
|
|
|
+ command_id: str | None = None,
|
|
|
) -> CandidateRef:
|
|
|
trace, root = await self._require_owner_trace(trace_id)
|
|
|
root_trace_id = root.trace_id
|
|
|
@@ -80,12 +81,18 @@ class CandidateService:
|
|
|
input_hash = sha256(
|
|
|
canonical_json(input_payload).encode("utf-8")
|
|
|
).hexdigest()
|
|
|
- # The command identity must survive a process crash and a subsequent
|
|
|
- # model retry. Sequence numbers identify attempts, not durable intent.
|
|
|
+ if command_id is not None and (not command_id or len(command_id) > 300):
|
|
|
+ raise CandidateStateError(
|
|
|
+ "Candidate command_id must be a non-empty string up to 300 chars"
|
|
|
+ )
|
|
|
+ # A persisted tool_call_id identifies one durable command. Direct
|
|
|
+ # internal callers that do not have an assistant call identity retain
|
|
|
+ # a sequence-scoped fallback, but the public tool always supplies it.
|
|
|
+ command_identity = command_id or f"sequence:{effective_at_sequence}"
|
|
|
command_hash = sha256(
|
|
|
canonical_json({
|
|
|
"trace_id": trace_id,
|
|
|
- "input_hash": input_hash,
|
|
|
+ "command_id": command_identity,
|
|
|
}).encode("utf-8")
|
|
|
).hexdigest()
|
|
|
operation_id = f"candidate:{command_hash}"
|
|
|
@@ -127,6 +134,7 @@ class CandidateService:
|
|
|
raise CandidateStateError(str(exc)) from exc
|
|
|
operation_record = CandidateOperationRecord(
|
|
|
operation_id=operation_id,
|
|
|
+ command_id=command_id,
|
|
|
operation=operation,
|
|
|
input_hash=input_hash,
|
|
|
candidate=pointer,
|
|
|
@@ -145,6 +153,8 @@ class CandidateService:
|
|
|
try:
|
|
|
parents = [ledger.candidate(item) for item in parent_refs]
|
|
|
self._validate_parents(trace, root, parents)
|
|
|
+ for parent in parents:
|
|
|
+ await self._assert_candidate_active(ledger, parent)
|
|
|
state = ensure_task_protocol(trace.context)
|
|
|
request = CandidateVersionRequest(
|
|
|
operation_id=operation_id,
|
|
|
@@ -215,6 +225,8 @@ class CandidateService:
|
|
|
self,
|
|
|
trace_id: str,
|
|
|
refs: list[CandidateRef],
|
|
|
+ *,
|
|
|
+ active_head_sequence: int | None = None,
|
|
|
) -> None:
|
|
|
trace, root = await self._require_owner_trace(trace_id)
|
|
|
async with self._lock_for(root.trace_id):
|
|
|
@@ -226,7 +238,17 @@ class CandidateService:
|
|
|
"TaskReport CandidateRef does not match the root ledger"
|
|
|
)
|
|
|
self._validate_candidate_owner(persisted, trace, root)
|
|
|
- if ledger.current_state(ref) != "proposed":
|
|
|
+ await self._assert_candidate_active(
|
|
|
+ ledger,
|
|
|
+ persisted,
|
|
|
+ active_head_sequence=active_head_sequence,
|
|
|
+ )
|
|
|
+ if await self._current_state(
|
|
|
+ ledger,
|
|
|
+ ref,
|
|
|
+ active_trace_id=trace_id,
|
|
|
+ active_head_sequence=active_head_sequence,
|
|
|
+ ) != "proposed":
|
|
|
raise CandidateStateError(
|
|
|
"TaskReport may only reference proposed candidates"
|
|
|
)
|
|
|
@@ -246,6 +268,7 @@ class CandidateService:
|
|
|
"CandidateRef does not match the root ledger"
|
|
|
)
|
|
|
self._validate_candidate_owner(persisted, trace, root)
|
|
|
+ await self._assert_candidate_active(ledger, persisted)
|
|
|
materials, issues = await resolve_artifact_refs(
|
|
|
[candidate_ref.artifact_ref],
|
|
|
resolver=self.artifact_resolver,
|
|
|
@@ -270,6 +293,7 @@ class CandidateService:
|
|
|
ledger = await self._load_ledger(root.trace_id)
|
|
|
persisted = ledger.candidate(candidate_ref)
|
|
|
self._validate_candidate_owner(persisted, trace, root)
|
|
|
+ await self._assert_candidate_active(ledger, persisted)
|
|
|
for raw in ledger.validations:
|
|
|
record = CandidateValidationRecord.model_validate(raw)
|
|
|
if (
|
|
|
@@ -298,6 +322,7 @@ class CandidateService:
|
|
|
"Candidate validation references an altered CandidateRef"
|
|
|
)
|
|
|
self._validate_candidate_owner(persisted, trace, root)
|
|
|
+ await self._assert_candidate_active(ledger, persisted)
|
|
|
for raw in ledger.validation_checkpoints:
|
|
|
checkpoint = CandidateValidationCheckpoint.model_validate(raw)
|
|
|
if (
|
|
|
@@ -331,6 +356,20 @@ class CandidateService:
|
|
|
plan_hash: str,
|
|
|
**updates: Any,
|
|
|
) -> CandidateValidationCheckpoint:
|
|
|
+ mutable_fields = {
|
|
|
+ "scope_results",
|
|
|
+ "fixed_checks",
|
|
|
+ "fixed_scope_errors",
|
|
|
+ "quality_completed",
|
|
|
+ "material_usage_recorded",
|
|
|
+ "aggregate_result",
|
|
|
+ }
|
|
|
+ unknown_fields = set(updates) - mutable_fields
|
|
|
+ if unknown_fields:
|
|
|
+ raise CandidateStateError(
|
|
|
+ "Candidate validation checkpoint fields are immutable: "
|
|
|
+ f"{', '.join(sorted(unknown_fields))}"
|
|
|
+ )
|
|
|
trace, root = await self._require_bound_trace(trace_id)
|
|
|
async with self._lock_for(root.trace_id):
|
|
|
ledger = await self._load_ledger(root.trace_id)
|
|
|
@@ -348,7 +387,10 @@ class CandidateService:
|
|
|
checkpoint.candidate_ref == candidate_ref
|
|
|
and checkpoint.plan_hash == plan_hash
|
|
|
):
|
|
|
- checkpoint = checkpoint.model_copy(update=updates)
|
|
|
+ checkpoint = CandidateValidationCheckpoint.model_validate({
|
|
|
+ **checkpoint.model_dump(mode="json"),
|
|
|
+ **updates,
|
|
|
+ })
|
|
|
result = checkpoint
|
|
|
checkpoints.append(checkpoint.model_dump(mode="json"))
|
|
|
if result is None:
|
|
|
@@ -376,6 +418,7 @@ class CandidateService:
|
|
|
"Candidate validation references an altered CandidateRef"
|
|
|
)
|
|
|
self._validate_candidate_owner(persisted, trace, root)
|
|
|
+ await self._assert_candidate_active(ledger, persisted)
|
|
|
retained = []
|
|
|
for raw in ledger.validations:
|
|
|
item = CandidateValidationRecord.model_validate(raw)
|
|
|
@@ -438,6 +481,7 @@ class CandidateService:
|
|
|
report_refs: list[CandidateRef],
|
|
|
actions: list[CandidateReviewAction],
|
|
|
effective_at_sequence: int,
|
|
|
+ command_id: str | None = None,
|
|
|
) -> list[CandidateLifecycleRecord]:
|
|
|
"""Apply parent-owned candidate decisions behind one root lock."""
|
|
|
if not actions:
|
|
|
@@ -485,6 +529,7 @@ class CandidateService:
|
|
|
raise CandidateStateError(
|
|
|
"Candidate action references another Task or altered revision"
|
|
|
)
|
|
|
+ await self._assert_candidate_active(ledger, persisted)
|
|
|
record = validation_by_key.get(key)
|
|
|
if record is None:
|
|
|
raise CandidateStateError(
|
|
|
@@ -503,11 +548,13 @@ class CandidateService:
|
|
|
raise CandidateStateError(
|
|
|
"Candidate revise requires a non-passed validation"
|
|
|
)
|
|
|
- state = ledger.current_state(action.candidate_ref)
|
|
|
+ state = await self._current_state(ledger, action.candidate_ref)
|
|
|
expected_operation_id = self._review_operation_id(
|
|
|
parent_trace_id,
|
|
|
child_trace_id,
|
|
|
action,
|
|
|
+ command_id=command_id,
|
|
|
+ effective_at_sequence=effective_at_sequence,
|
|
|
)
|
|
|
same_completed_operation = any(
|
|
|
item.operation_id == expected_operation_id
|
|
|
@@ -531,6 +578,7 @@ class CandidateService:
|
|
|
child_trace_id=child_trace_id,
|
|
|
root_trace_id=root.trace_id,
|
|
|
effective_at_sequence=effective_at_sequence,
|
|
|
+ command_id=command_id,
|
|
|
)
|
|
|
applied.append(record)
|
|
|
return applied
|
|
|
@@ -544,6 +592,7 @@ class CandidateService:
|
|
|
child_trace_id: str,
|
|
|
root_trace_id: str,
|
|
|
effective_at_sequence: int,
|
|
|
+ command_id: str | None,
|
|
|
) -> tuple[CandidateLedger, CandidateLifecycleRecord]:
|
|
|
pointer = CandidatePointer(
|
|
|
candidate_id=action.candidate_ref.candidate_id,
|
|
|
@@ -553,6 +602,8 @@ class CandidateService:
|
|
|
parent_trace_id,
|
|
|
child_trace_id,
|
|
|
action,
|
|
|
+ command_id=command_id,
|
|
|
+ effective_at_sequence=effective_at_sequence,
|
|
|
)
|
|
|
input_hash = sha256(
|
|
|
canonical_json(action).encode("utf-8")
|
|
|
@@ -575,6 +626,7 @@ class CandidateService:
|
|
|
else:
|
|
|
operation = CandidateOperationRecord(
|
|
|
operation_id=operation_id,
|
|
|
+ command_id=command_id,
|
|
|
operation=action.action,
|
|
|
input_hash=input_hash,
|
|
|
candidate=pointer,
|
|
|
@@ -674,6 +726,9 @@ class CandidateService:
|
|
|
parent_trace_id: str,
|
|
|
child_trace_id: str,
|
|
|
action: CandidateReviewAction,
|
|
|
+ *,
|
|
|
+ command_id: str | None = None,
|
|
|
+ effective_at_sequence: int = 0,
|
|
|
) -> str:
|
|
|
action_hash = sha256(
|
|
|
canonical_json(action).encode("utf-8")
|
|
|
@@ -682,6 +737,7 @@ class CandidateService:
|
|
|
canonical_json({
|
|
|
"parent_trace_id": parent_trace_id,
|
|
|
"child_trace_id": child_trace_id,
|
|
|
+ "command_id": command_id or f"sequence:{effective_at_sequence}",
|
|
|
"action_hash": action_hash,
|
|
|
}).encode("utf-8")
|
|
|
).hexdigest()
|
|
|
@@ -807,6 +863,106 @@ class CandidateService:
|
|
|
"Cannot rewind across a committed candidate adoption"
|
|
|
)
|
|
|
|
|
|
+ @staticmethod
|
|
|
+ def _candidate_operation(
|
|
|
+ ledger: CandidateLedger,
|
|
|
+ candidate: CandidateRef,
|
|
|
+ ) -> CandidateOperationRecord:
|
|
|
+ for operation in ledger.operations:
|
|
|
+ if (
|
|
|
+ operation.operation in {"create", "fork", "merge"}
|
|
|
+ and operation.candidate is not None
|
|
|
+ and operation.candidate.candidate_id == candidate.candidate_id
|
|
|
+ and operation.candidate.revision == candidate.revision
|
|
|
+ ):
|
|
|
+ return operation
|
|
|
+ raise CandidateStateError(
|
|
|
+ "Candidate has no persisted creation command"
|
|
|
+ )
|
|
|
+
|
|
|
+ async def _sequence_is_active(
|
|
|
+ self,
|
|
|
+ trace_id: str,
|
|
|
+ sequence: int,
|
|
|
+ *,
|
|
|
+ command_id: str | None,
|
|
|
+ allow_untracked: bool,
|
|
|
+ head_sequence: int | None = None,
|
|
|
+ ) -> bool:
|
|
|
+ trace = await self.store.get_trace(trace_id)
|
|
|
+ if trace is None:
|
|
|
+ return False
|
|
|
+ all_messages = await self.store.get_trace_messages(trace_id)
|
|
|
+ if not all_messages:
|
|
|
+ # CandidateService has a direct internal port for tests and
|
|
|
+ # controlled adapters. Production tool commands are always bound
|
|
|
+ # to a persisted assistant tool_call_id and never take this path.
|
|
|
+ return allow_untracked
|
|
|
+ path = await self.store.get_main_path_messages(
|
|
|
+ trace_id,
|
|
|
+ trace.head_sequence if head_sequence is None else head_sequence,
|
|
|
+ )
|
|
|
+ message = next((item for item in path if item.sequence == sequence), None)
|
|
|
+ if message is None:
|
|
|
+ return False
|
|
|
+ if command_id is not None:
|
|
|
+ return message.role == "tool" and message.tool_call_id == command_id
|
|
|
+ return True
|
|
|
+
|
|
|
+ async def _assert_candidate_active(
|
|
|
+ self,
|
|
|
+ ledger: CandidateLedger,
|
|
|
+ candidate: CandidateRef,
|
|
|
+ *,
|
|
|
+ active_head_sequence: int | None = None,
|
|
|
+ ) -> None:
|
|
|
+ operation = self._candidate_operation(ledger, candidate)
|
|
|
+ if not await self._sequence_is_active(
|
|
|
+ candidate.owner_trace_id,
|
|
|
+ candidate.created_at_sequence,
|
|
|
+ command_id=operation.command_id,
|
|
|
+ allow_untracked=operation.command_id is None,
|
|
|
+ head_sequence=active_head_sequence,
|
|
|
+ ):
|
|
|
+ raise CandidateStateError(
|
|
|
+ "Candidate is not on the current owner Trace branch"
|
|
|
+ )
|
|
|
+
|
|
|
+ async def _current_state(
|
|
|
+ self,
|
|
|
+ ledger: CandidateLedger,
|
|
|
+ pointer: CandidatePointer,
|
|
|
+ *,
|
|
|
+ active_trace_id: str | None = None,
|
|
|
+ active_head_sequence: int | None = None,
|
|
|
+ ) -> str | None:
|
|
|
+ candidate = ledger.candidate(pointer)
|
|
|
+ creation = self._candidate_operation(ledger, candidate)
|
|
|
+ state = None
|
|
|
+ operations = {item.operation_id: item for item in ledger.operations}
|
|
|
+ for lifecycle in ledger.lifecycle:
|
|
|
+ if (
|
|
|
+ lifecycle.candidate.candidate_id != pointer.candidate_id
|
|
|
+ or lifecycle.candidate.revision != pointer.revision
|
|
|
+ or lifecycle.source_trace_id is None
|
|
|
+ ):
|
|
|
+ continue
|
|
|
+ operation = operations.get(lifecycle.operation_id)
|
|
|
+ command_id = operation.command_id if operation is not None else None
|
|
|
+ if await self._sequence_is_active(
|
|
|
+ lifecycle.source_trace_id,
|
|
|
+ lifecycle.effective_at_sequence,
|
|
|
+ command_id=command_id,
|
|
|
+ allow_untracked=creation.command_id is None,
|
|
|
+ head_sequence=(
|
|
|
+ active_head_sequence
|
|
|
+ if lifecycle.source_trace_id == active_trace_id
|
|
|
+ else None
|
|
|
+ ),
|
|
|
+ ):
|
|
|
+ state = lifecycle.state
|
|
|
+ return state
|
|
|
+
|
|
|
async def _require_owner_trace(self, trace_id: str):
|
|
|
trace, root = await self._require_bound_trace(trace_id)
|
|
|
state = ensure_task_protocol(trace.context)
|