|
|
@@ -596,12 +596,14 @@ async def _collect_diagnostics(host: Any, script_build_id: int) -> dict[str, Any
|
|
|
task_seconds: dict[str, float] = {}
|
|
|
preset_usage: dict[str, dict[str, int | float]] = {}
|
|
|
domain_errors: list[dict[str, str]] = []
|
|
|
+ planner_tool_errors: list[dict[str, str]] = []
|
|
|
+ trace_store = host.composition.mission_service.runner.trace_store
|
|
|
trace_presets = {binding.root_trace_id: "script_planner"}
|
|
|
trace_presets.update({item.worker_trace_id: item.worker_preset for item in attempts})
|
|
|
trace_presets.update({item.validator_trace_id: item.validator_preset for item in validations})
|
|
|
traces_with_usage: set[str] = set()
|
|
|
usage_reader = getattr(
|
|
|
- host.composition.mission_service.runner.trace_store, "get_model_usage", None
|
|
|
+ trace_store, "get_model_usage", None
|
|
|
)
|
|
|
if callable(usage_reader):
|
|
|
for trace_id, preset in trace_presets.items():
|
|
|
@@ -620,6 +622,10 @@ async def _collect_diagnostics(host: Any, script_build_id: int) -> dict[str, Any
|
|
|
usage["completion_tokens"] += int(model.get("completion_tokens") or 0)
|
|
|
usage["cache_read_tokens"] += int(model.get("cache_read_tokens") or 0)
|
|
|
usage["tokens"] += int(model.get("total_tokens") or 0)
|
|
|
+ planner_trace = await trace_store.get_trace(binding.root_trace_id)
|
|
|
+ preset_usage.setdefault("script_planner", _empty_preset_usage())["seconds"] += (
|
|
|
+ _trace_elapsed_seconds(planner_trace)
|
|
|
+ )
|
|
|
for item in [*attempts, *validations]:
|
|
|
task = ledger.tasks[item.task_id]
|
|
|
kind = next(
|
|
|
@@ -670,21 +676,24 @@ async def _collect_diagnostics(host: Any, script_build_id: int) -> dict[str, Any
|
|
|
tool_calls: dict[str, int] = {}
|
|
|
broker_events = [
|
|
|
item
|
|
|
- for item in await host.composition.mission_service.runner.trace_store.get_events(
|
|
|
- binding.root_trace_id
|
|
|
- )
|
|
|
+ for item in await trace_store.get_events(binding.root_trace_id)
|
|
|
if item.get("event") == "context_broker_receipt"
|
|
|
]
|
|
|
for trace_id in trace_ids:
|
|
|
- for message in await host.composition.mission_service.runner.trace_store.get_trace_messages(
|
|
|
- trace_id
|
|
|
- ):
|
|
|
+ for message in await trace_store.get_trace_messages(trace_id):
|
|
|
content = message.content
|
|
|
if not isinstance(content, dict):
|
|
|
continue
|
|
|
+ if trace_id == binding.root_trace_id:
|
|
|
+ failure = _tool_failure_from_message(message)
|
|
|
+ if failure is not None:
|
|
|
+ if not failure["task_id"]:
|
|
|
+ failure["task_id"] = root.task_id
|
|
|
+ planner_tool_errors.append(failure)
|
|
|
for call in content.get("tool_calls") or []:
|
|
|
name = str(call.get("function", {}).get("name") or "unknown")
|
|
|
tool_calls[name] = tool_calls.get(name, 0) + 1
|
|
|
+ historical_errors = [*domain_errors, *planner_tool_errors]
|
|
|
last_task = max(ledger.tasks.values(), key=lambda item: item.updated_at)
|
|
|
last_attempt = max(attempts, key=lambda item: item.updated_at) if attempts else None
|
|
|
last_validation = max(validations, key=lambda item: item.updated_at) if validations else None
|
|
|
@@ -722,8 +731,8 @@ async def _collect_diagnostics(host: Any, script_build_id: int) -> dict[str, Any
|
|
|
None,
|
|
|
),
|
|
|
"error_history_by_code": {
|
|
|
- code: sum(item["code"] == code for item in domain_errors)
|
|
|
- for code in sorted({item["code"] for item in domain_errors})
|
|
|
+ code: sum(item["code"] == code for item in historical_errors)
|
|
|
+ for code in sorted({item["code"] for item in historical_errors})
|
|
|
},
|
|
|
"phase_timing_seconds": task_seconds,
|
|
|
"preset_usage": preset_usage,
|
|
|
@@ -739,6 +748,54 @@ async def _collect_diagnostics(host: Any, script_build_id: int) -> dict[str, Any
|
|
|
}
|
|
|
|
|
|
|
|
|
+def _tool_failure_from_message(message: Any) -> dict[str, str] | None:
|
|
|
+ if getattr(message, "role", None) != "tool" or not isinstance(message.content, dict):
|
|
|
+ return None
|
|
|
+ result = message.content.get("result")
|
|
|
+ if isinstance(result, str):
|
|
|
+ try:
|
|
|
+ payload = json.loads(result)
|
|
|
+ except (TypeError, ValueError):
|
|
|
+ return None
|
|
|
+ elif isinstance(result, dict):
|
|
|
+ payload = result
|
|
|
+ else:
|
|
|
+ return None
|
|
|
+ if not isinstance(payload, dict):
|
|
|
+ return None
|
|
|
+ failure = payload.get("failure")
|
|
|
+ if not isinstance(failure, dict):
|
|
|
+ control = payload.get("_control")
|
|
|
+ failure = control.get("failure") if isinstance(control, dict) else None
|
|
|
+ if not isinstance(failure, dict):
|
|
|
+ return None
|
|
|
+ code = str(failure.get("code") or "").strip()
|
|
|
+ if not code:
|
|
|
+ return None
|
|
|
+ details = failure.get("details")
|
|
|
+ task_id = str(details.get("task_id") or "") if isinstance(details, dict) else ""
|
|
|
+ return {
|
|
|
+ "code": code,
|
|
|
+ "message": str(failure.get("message") or ""),
|
|
|
+ "task_id": task_id,
|
|
|
+ "source_tool": str(
|
|
|
+ failure.get("source_tool") or message.content.get("tool_name") or ""
|
|
|
+ ),
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def _trace_elapsed_seconds(trace: Any) -> float:
|
|
|
+ if trace is None:
|
|
|
+ return 0.0
|
|
|
+ started_at = getattr(trace, "created_at", None)
|
|
|
+ completed_at = getattr(trace, "completed_at", None) or getattr(
|
|
|
+ trace, "last_activity_at", None
|
|
|
+ )
|
|
|
+ if isinstance(started_at, datetime) and isinstance(completed_at, datetime):
|
|
|
+ return max((completed_at - started_at).total_seconds(), 0.0)
|
|
|
+ return max(float(getattr(trace, "total_duration_ms", 0) or 0) / 1000, 0.0)
|
|
|
+
|
|
|
+
|
|
|
def _context_broker_diagnostics(events: list[dict[str, Any]]) -> dict[str, Any]:
|
|
|
sizes = sorted(int(item.get("estimated_tokens") or 0) for item in events)
|
|
|
detail_pages = [
|