|
|
@@ -153,6 +153,10 @@ from cyber_agent.core.prompts import (
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
+class PendingToolApprovalRestored(RuntimeError):
|
|
|
+ """Internal control flow: an orphaned call was restored to user approval."""
|
|
|
+
|
|
|
+
|
|
|
@dataclass
|
|
|
class ContextUsage:
|
|
|
"""Context 使用情况"""
|
|
|
@@ -1350,6 +1354,14 @@ class AgentRunner:
|
|
|
history, sequence, created_messages, head_seq = await self._build_history(
|
|
|
trace.trace_id, messages, goal_tree, config, sequence, side_branch_ctx_for_build
|
|
|
)
|
|
|
+ # History recovery may replay an idempotent protocol command and
|
|
|
+ # commit a newer context before the model loop starts. Keep the
|
|
|
+ # yielded Trace object aligned with that authoritative Store state.
|
|
|
+ if self.trace_store:
|
|
|
+ recovered_trace = await self.trace_store.get_trace(trace.trace_id)
|
|
|
+ if recovered_trace is not None:
|
|
|
+ trace.context = recovered_trace.context
|
|
|
+ trace.last_sequence = recovered_trace.last_sequence
|
|
|
# Update trace's head_sequence in memory
|
|
|
trace.head_sequence = head_seq
|
|
|
for msg in created_messages:
|
|
|
@@ -1363,6 +1375,12 @@ class AgentRunner:
|
|
|
):
|
|
|
yield event
|
|
|
|
|
|
+ except PendingToolApprovalRestored:
|
|
|
+ if trace and self.trace_store:
|
|
|
+ waiting_trace = await self.trace_store.get_trace(trace.trace_id)
|
|
|
+ if waiting_trace is not None:
|
|
|
+ yield waiting_trace
|
|
|
+ return
|
|
|
except asyncio.CancelledError:
|
|
|
if trace and config.approval_batch_id:
|
|
|
await asyncio.shield(
|
|
|
@@ -2188,7 +2206,11 @@ class AgentRunner:
|
|
|
# they are executed before the next model turn.
|
|
|
if not config.approval_batch_id:
|
|
|
main_path, sequence = await self._heal_orphaned_tool_calls(
|
|
|
- main_path, trace_id, goal_tree, sequence,
|
|
|
+ main_path,
|
|
|
+ trace_obj,
|
|
|
+ goal_tree,
|
|
|
+ config,
|
|
|
+ sequence,
|
|
|
)
|
|
|
|
|
|
history = [msg.to_llm_dict() for msg in main_path]
|
|
|
@@ -2683,6 +2705,135 @@ class AgentRunner:
|
|
|
|
|
|
return None
|
|
|
|
|
|
+ def _create_tool_approval_batch(
|
|
|
+ self,
|
|
|
+ *,
|
|
|
+ trace_id: str,
|
|
|
+ assistant_msg: Message,
|
|
|
+ tool_calls: List[Dict[str, Any]],
|
|
|
+ auto_execute_tools: bool,
|
|
|
+ ) -> ToolApprovalBatchV1:
|
|
|
+ """Compile the one persisted approval representation used by all paths."""
|
|
|
+ approval_calls: List[ToolApprovalCallV1] = []
|
|
|
+ for tool_call in tool_calls:
|
|
|
+ tool_name = tool_call.get("function", {}).get("name", "")
|
|
|
+ raw_arguments = tool_call.get("function", {}).get("arguments", {})
|
|
|
+ if isinstance(raw_arguments, str):
|
|
|
+ try:
|
|
|
+ parsed_arguments = (
|
|
|
+ json.loads(raw_arguments)
|
|
|
+ if raw_arguments.strip()
|
|
|
+ else {}
|
|
|
+ )
|
|
|
+ except json.JSONDecodeError:
|
|
|
+ parsed_arguments = {"_raw": raw_arguments}
|
|
|
+ elif isinstance(raw_arguments, dict):
|
|
|
+ parsed_arguments = dict(raw_arguments)
|
|
|
+ else:
|
|
|
+ parsed_arguments = {}
|
|
|
+ policy = self.tools.get_runtime_policy(tool_name)
|
|
|
+ requires_confirmation = (
|
|
|
+ not auto_execute_tools
|
|
|
+ or bool(policy.get("requires_confirmation"))
|
|
|
+ )
|
|
|
+ decision = "pending" if requires_confirmation else "auto_approved"
|
|
|
+ call_id = str(tool_call.get("id") or "")
|
|
|
+ approval_calls.append(ToolApprovalCallV1(
|
|
|
+ tool_call_id=call_id,
|
|
|
+ tool_name=tool_name,
|
|
|
+ original_arguments=parsed_arguments,
|
|
|
+ effective_arguments=parsed_arguments,
|
|
|
+ argument_hash=tool_argument_hash(
|
|
|
+ tool_call_id=call_id,
|
|
|
+ tool_name=tool_name,
|
|
|
+ arguments=parsed_arguments,
|
|
|
+ ),
|
|
|
+ editable_params=list(policy.get("editable_params", [])),
|
|
|
+ requires_confirmation=requires_confirmation,
|
|
|
+ decision=decision,
|
|
|
+ ))
|
|
|
+ return ToolApprovalBatchV1.create(
|
|
|
+ trace_id=trace_id,
|
|
|
+ assistant_message_id=assistant_msg.message_id,
|
|
|
+ assistant_sequence=assistant_msg.sequence,
|
|
|
+ calls=approval_calls,
|
|
|
+ )
|
|
|
+
|
|
|
+ async def _restore_orphaned_tool_approval(
|
|
|
+ self,
|
|
|
+ *,
|
|
|
+ trace: Trace,
|
|
|
+ assistant_msg: Message,
|
|
|
+ tool_calls: List[Dict[str, Any]],
|
|
|
+ config: RunConfig,
|
|
|
+ ) -> None:
|
|
|
+ """Restore the approval gate without executing an orphaned side effect."""
|
|
|
+ if not self.trace_store or not hasattr(
|
|
|
+ self.trace_store,
|
|
|
+ "get_tool_approval_batch",
|
|
|
+ ):
|
|
|
+ raise RuntimeError("TraceStore does not support tool approvals")
|
|
|
+ expected = self._create_tool_approval_batch(
|
|
|
+ trace_id=trace.trace_id,
|
|
|
+ assistant_msg=assistant_msg,
|
|
|
+ tool_calls=tool_calls,
|
|
|
+ auto_execute_tools=config.auto_execute_tools,
|
|
|
+ )
|
|
|
+ batch = await self.trace_store.get_tool_approval_batch(trace.trace_id)
|
|
|
+ if batch is None:
|
|
|
+ batch = expected
|
|
|
+ await self.trace_store.replace_tool_approval_batch(
|
|
|
+ trace.trace_id,
|
|
|
+ batch,
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ same_source = (
|
|
|
+ batch.trace_id == expected.trace_id
|
|
|
+ and batch.assistant_message_id == expected.assistant_message_id
|
|
|
+ and batch.assistant_sequence == expected.assistant_sequence
|
|
|
+ and len(batch.calls) == len(expected.calls)
|
|
|
+ and all(
|
|
|
+ current.tool_call_id == wanted.tool_call_id
|
|
|
+ and current.tool_name == wanted.tool_name
|
|
|
+ and current.original_arguments == wanted.original_arguments
|
|
|
+ and current.argument_hash == wanted.argument_hash
|
|
|
+ and current.editable_params == wanted.editable_params
|
|
|
+ and current.requires_confirmation == wanted.requires_confirmation
|
|
|
+ for current, wanted in zip(batch.calls, expected.calls)
|
|
|
+ )
|
|
|
+ )
|
|
|
+ if not same_source:
|
|
|
+ raise RuntimeError(
|
|
|
+ "Persisted tool approval batch does not match the orphaned call"
|
|
|
+ )
|
|
|
+ if batch.status == "executing" or any(
|
|
|
+ call.execution_status == "executing" for call in batch.calls
|
|
|
+ ):
|
|
|
+ batch.status = "execution_unknown"
|
|
|
+ for call in batch.calls:
|
|
|
+ if call.execution_status == "executing":
|
|
|
+ call.execution_status = "execution_unknown"
|
|
|
+ batch.updated_at = datetime.now().isoformat()
|
|
|
+ await self.trace_store.replace_tool_approval_batch(
|
|
|
+ trace.trace_id,
|
|
|
+ batch,
|
|
|
+ )
|
|
|
+ raise RuntimeError(
|
|
|
+ "Tool execution outcome is unknown; refusing automatic retry"
|
|
|
+ )
|
|
|
+ if batch.status in {"execution_unknown", "completed", "cancelled"}:
|
|
|
+ raise RuntimeError(
|
|
|
+ f"Orphaned tool approval is not recoverable: {batch.status}"
|
|
|
+ )
|
|
|
+ await self.trace_store.update_trace(
|
|
|
+ trace.trace_id,
|
|
|
+ status="waiting_confirmation",
|
|
|
+ head_sequence=assistant_msg.sequence,
|
|
|
+ )
|
|
|
+ raise PendingToolApprovalRestored(
|
|
|
+ "orphaned tool call restored to persistent approval"
|
|
|
+ )
|
|
|
+
|
|
|
async def _resume_approved_tool_batch(
|
|
|
self,
|
|
|
*,
|
|
|
@@ -3640,53 +3791,11 @@ class AgentRunner:
|
|
|
or self.tools.check_confirmation_required(tool_calls)
|
|
|
)
|
|
|
if needs_confirmation:
|
|
|
- approval_calls: List[ToolApprovalCallV1] = []
|
|
|
- for tool_call in tool_calls:
|
|
|
- tool_name = tool_call.get("function", {}).get("name", "")
|
|
|
- raw_arguments = tool_call.get("function", {}).get(
|
|
|
- "arguments", {}
|
|
|
- )
|
|
|
- if isinstance(raw_arguments, str):
|
|
|
- try:
|
|
|
- parsed_arguments = (
|
|
|
- json.loads(raw_arguments)
|
|
|
- if raw_arguments.strip()
|
|
|
- else {}
|
|
|
- )
|
|
|
- except json.JSONDecodeError:
|
|
|
- parsed_arguments = {"_raw": raw_arguments}
|
|
|
- elif isinstance(raw_arguments, dict):
|
|
|
- parsed_arguments = dict(raw_arguments)
|
|
|
- else:
|
|
|
- parsed_arguments = {}
|
|
|
- policy = self.tools.get_runtime_policy(tool_name)
|
|
|
- requires_confirmation = (
|
|
|
- not config.auto_execute_tools
|
|
|
- or bool(policy.get("requires_confirmation"))
|
|
|
- )
|
|
|
- decision = (
|
|
|
- "pending" if requires_confirmation else "auto_approved"
|
|
|
- )
|
|
|
- call_id = str(tool_call.get("id") or "")
|
|
|
- approval_calls.append(ToolApprovalCallV1(
|
|
|
- tool_call_id=call_id,
|
|
|
- tool_name=tool_name,
|
|
|
- original_arguments=parsed_arguments,
|
|
|
- effective_arguments=parsed_arguments,
|
|
|
- argument_hash=tool_argument_hash(
|
|
|
- tool_call_id=call_id,
|
|
|
- tool_name=tool_name,
|
|
|
- arguments=parsed_arguments,
|
|
|
- ),
|
|
|
- editable_params=list(policy.get("editable_params", [])),
|
|
|
- requires_confirmation=requires_confirmation,
|
|
|
- decision=decision,
|
|
|
- ))
|
|
|
- batch = ToolApprovalBatchV1.create(
|
|
|
+ batch = self._create_tool_approval_batch(
|
|
|
trace_id=trace_id,
|
|
|
- assistant_message_id=assistant_msg.message_id,
|
|
|
- assistant_sequence=assistant_msg.sequence,
|
|
|
- calls=approval_calls,
|
|
|
+ assistant_msg=assistant_msg,
|
|
|
+ tool_calls=tool_calls,
|
|
|
+ auto_execute_tools=config.auto_execute_tools,
|
|
|
)
|
|
|
if not self.trace_store or not hasattr(
|
|
|
self.trace_store, "replace_tool_approval_batch"
|
|
|
@@ -4700,11 +4809,98 @@ class AgentRunner:
|
|
|
|
|
|
return cutoff
|
|
|
|
|
|
+ async def _replay_orphaned_review_call(
|
|
|
+ self,
|
|
|
+ *,
|
|
|
+ trace: Trace,
|
|
|
+ goal_tree: Optional[GoalTree],
|
|
|
+ config: RunConfig,
|
|
|
+ assistant_msg: Message,
|
|
|
+ tool_call: Dict[str, Any],
|
|
|
+ sequence: int,
|
|
|
+ head_sequence: int,
|
|
|
+ ) -> Message:
|
|
|
+ """Replay one durable, idempotent review command before another LLM turn."""
|
|
|
+ function = tool_call.get("function") or {}
|
|
|
+ raw_arguments = function.get("arguments", {})
|
|
|
+ if isinstance(raw_arguments, str):
|
|
|
+ try:
|
|
|
+ arguments = json.loads(raw_arguments) if raw_arguments.strip() else {}
|
|
|
+ except json.JSONDecodeError as exc:
|
|
|
+ raise RuntimeError(
|
|
|
+ "Orphaned review command has invalid persisted arguments"
|
|
|
+ ) from exc
|
|
|
+ elif isinstance(raw_arguments, dict):
|
|
|
+ arguments = dict(raw_arguments)
|
|
|
+ else:
|
|
|
+ raise RuntimeError(
|
|
|
+ "Orphaned review command has invalid persisted arguments"
|
|
|
+ )
|
|
|
+ tool_call_id = str(tool_call.get("id") or "")
|
|
|
+ if not tool_call_id:
|
|
|
+ raise RuntimeError("Orphaned review command has no tool_call_id")
|
|
|
+ fresh_trace = (
|
|
|
+ await self.trace_store.get_trace(trace.trace_id)
|
|
|
+ if self.trace_store
|
|
|
+ else None
|
|
|
+ )
|
|
|
+ if fresh_trace is None:
|
|
|
+ raise RuntimeError("Orphaned review Trace no longer exists")
|
|
|
+ result = await self.tools.execute(
|
|
|
+ "review_task_result",
|
|
|
+ arguments,
|
|
|
+ uid=config.uid or "",
|
|
|
+ context=self._build_tool_context(
|
|
|
+ config=config,
|
|
|
+ trace=fresh_trace,
|
|
|
+ trace_id=fresh_trace.trace_id,
|
|
|
+ goal_id=assistant_msg.goal_id,
|
|
|
+ goal_tree=goal_tree,
|
|
|
+ sequence=sequence,
|
|
|
+ head_sequence=head_sequence,
|
|
|
+ tool_call_id=tool_call_id,
|
|
|
+ side_branch_ctx=None,
|
|
|
+ trigger_event=None,
|
|
|
+ ),
|
|
|
+ allowed_tool_names={"review_task_result"},
|
|
|
+ tool_call_id=tool_call_id,
|
|
|
+ )
|
|
|
+ result_text = (
|
|
|
+ result
|
|
|
+ if isinstance(result, str)
|
|
|
+ else json.dumps(result, ensure_ascii=False)
|
|
|
+ )
|
|
|
+ try:
|
|
|
+ payload = json.loads(result_text)
|
|
|
+ except (json.JSONDecodeError, TypeError) as exc:
|
|
|
+ raise RuntimeError(
|
|
|
+ "Orphaned review replay returned a non-JSON result"
|
|
|
+ ) from exc
|
|
|
+ if not isinstance(payload, dict) or payload.get("status") != "completed":
|
|
|
+ reason = payload.get("error") if isinstance(payload, dict) else None
|
|
|
+ raise RuntimeError(
|
|
|
+ "Orphaned review replay failed closed: "
|
|
|
+ + str(reason or "unexpected tool result")
|
|
|
+ )
|
|
|
+ return Message.create(
|
|
|
+ trace_id=fresh_trace.trace_id,
|
|
|
+ role="tool",
|
|
|
+ sequence=sequence,
|
|
|
+ goal_id=assistant_msg.goal_id,
|
|
|
+ parent_sequence=head_sequence,
|
|
|
+ tool_call_id=tool_call_id,
|
|
|
+ content={
|
|
|
+ "tool_name": "review_task_result",
|
|
|
+ "result": result_text,
|
|
|
+ },
|
|
|
+ )
|
|
|
+
|
|
|
async def _heal_orphaned_tool_calls(
|
|
|
self,
|
|
|
messages: List[Message],
|
|
|
- trace_id: str,
|
|
|
+ trace: Trace,
|
|
|
goal_tree: Optional[GoalTree],
|
|
|
+ config: RunConfig,
|
|
|
sequence: int,
|
|
|
) -> tuple:
|
|
|
"""
|
|
|
@@ -4713,7 +4909,8 @@ class AgentRunner:
|
|
|
当 agent 被 stop/crash 中断时,可能有 assistant 的 tool_calls 没有对应的
|
|
|
tool results(包括多 tool_call 部分完成的情况)。直接发给 LLM 会导致 400。
|
|
|
|
|
|
- 修复策略:为每个缺失的 tool_result 插入合成的"中断通知"消息,而非裁剪。
|
|
|
+ 修复策略:幂等的 ``review_task_result`` 使用原 tool_call_id 在 LLM 前
|
|
|
+ 受控重放;其他缺失调用插入合成的"中断通知"消息,而非裁剪。
|
|
|
- 普通工具:简短中断提示
|
|
|
- agent/evaluate:包含 sub_trace_id、执行统计、continue_from 指引
|
|
|
|
|
|
@@ -4756,6 +4953,49 @@ class AgentRunner:
|
|
|
assistant_msg, tc = tc_map[tc_id]
|
|
|
tool_name = tc.get("function", {}).get("name", "unknown")
|
|
|
|
|
|
+ if tool_name == "review_task_result":
|
|
|
+ assistant_calls = (
|
|
|
+ assistant_msg.content.get("tool_calls", [])
|
|
|
+ if isinstance(assistant_msg.content, dict)
|
|
|
+ else []
|
|
|
+ )
|
|
|
+ if len(assistant_calls) != 1:
|
|
|
+ raise RuntimeError(
|
|
|
+ "Orphaned lifecycle-exclusive review was persisted in a mixed batch"
|
|
|
+ )
|
|
|
+ needs_confirmation = (
|
|
|
+ not config.auto_execute_tools
|
|
|
+ or self.tools.check_confirmation_required([tc])
|
|
|
+ )
|
|
|
+ persisted_approval = (
|
|
|
+ await self.trace_store.get_tool_approval_batch(trace.trace_id)
|
|
|
+ if self.trace_store
|
|
|
+ and hasattr(self.trace_store, "get_tool_approval_batch")
|
|
|
+ else None
|
|
|
+ )
|
|
|
+ if needs_confirmation or persisted_approval is not None:
|
|
|
+ await self._restore_orphaned_tool_approval(
|
|
|
+ trace=trace,
|
|
|
+ assistant_msg=assistant_msg,
|
|
|
+ tool_calls=[tc],
|
|
|
+ config=config,
|
|
|
+ )
|
|
|
+ replayed = await self._replay_orphaned_review_call(
|
|
|
+ trace=trace,
|
|
|
+ goal_tree=goal_tree,
|
|
|
+ config=config,
|
|
|
+ assistant_msg=assistant_msg,
|
|
|
+ tool_call=tc,
|
|
|
+ sequence=sequence,
|
|
|
+ head_sequence=head_seq,
|
|
|
+ )
|
|
|
+ if self.trace_store:
|
|
|
+ await self.trace_store.add_message(replayed)
|
|
|
+ healed.append(replayed)
|
|
|
+ head_seq = sequence
|
|
|
+ sequence += 1
|
|
|
+ continue
|
|
|
+
|
|
|
if tool_name in ("agent", "evaluate"):
|
|
|
result_text = self._build_agent_interrupted_result(
|
|
|
tc, goal_tree, assistant_msg,
|
|
|
@@ -4764,7 +5004,7 @@ class AgentRunner:
|
|
|
result_text = build_tool_interrupted_message(tool_name)
|
|
|
|
|
|
synthetic_msg = Message.create(
|
|
|
- trace_id=trace_id,
|
|
|
+ trace_id=trace.trace_id,
|
|
|
role="tool",
|
|
|
sequence=sequence,
|
|
|
goal_id=assistant_msg.goal_id,
|
|
|
@@ -4783,7 +5023,7 @@ class AgentRunner:
|
|
|
# 更新 trace head/last sequence
|
|
|
if self.trace_store:
|
|
|
await self.trace_store.update_trace(
|
|
|
- trace_id,
|
|
|
+ trace.trace_id,
|
|
|
head_sequence=head_seq,
|
|
|
last_sequence=max(head_seq, sequence - 1),
|
|
|
)
|