|
|
@@ -668,6 +668,13 @@ async def _collect_diagnostics(host: Any, script_build_id: int) -> dict[str, Any
|
|
|
trace_ids.update(item.worker_trace_id for item in attempts)
|
|
|
trace_ids.update(item.validator_trace_id for item in validations)
|
|
|
tool_calls: dict[str, int] = {}
|
|
|
+ broker_events = [
|
|
|
+ item
|
|
|
+ for item in await host.composition.mission_service.runner.trace_store.get_events(
|
|
|
+ binding.root_trace_id
|
|
|
+ )
|
|
|
+ if item.get("event") == "context_broker_receipt"
|
|
|
+ ]
|
|
|
for trace_id in trace_ids:
|
|
|
for message in await host.composition.mission_service.runner.trace_store.get_trace_messages(
|
|
|
trace_id
|
|
|
@@ -722,15 +729,45 @@ async def _collect_diagnostics(host: Any, script_build_id: int) -> dict[str, Any
|
|
|
"preset_usage": preset_usage,
|
|
|
"tool_calls": tool_calls,
|
|
|
"paragraph_tool_calls": {
|
|
|
- "single": tool_calls.get("create_script_paragraph", 0),
|
|
|
- "batch": tool_calls.get("create_script_paragraphs", 0),
|
|
|
+ "paragraph_upsert": tool_calls.get("save_script_paragraphs", 0),
|
|
|
+ "element_upsert": tool_calls.get("save_script_elements", 0),
|
|
|
},
|
|
|
+ "context_broker": _context_broker_diagnostics(broker_events),
|
|
|
"deterministic_preflight_rejection": any(
|
|
|
"Deterministic preflight rejected" in item.summary for item in validations
|
|
|
),
|
|
|
}
|
|
|
|
|
|
|
|
|
+def _context_broker_diagnostics(events: list[dict[str, Any]]) -> dict[str, Any]:
|
|
|
+ sizes = sorted(int(item.get("estimated_tokens") or 0) for item in events)
|
|
|
+ detail_pages = [
|
|
|
+ str(item.get("detail_page_key"))
|
|
|
+ for item in events
|
|
|
+ if int(item.get("detail_reads") or 0) > 0 and item.get("detail_page_key")
|
|
|
+ ]
|
|
|
+ return {
|
|
|
+ "requests": len(events),
|
|
|
+ "bundle_tokens": {
|
|
|
+ "p50": _percentile(sizes, 0.50),
|
|
|
+ "p95": _percentile(sizes, 0.95),
|
|
|
+ "max": max(sizes, default=0),
|
|
|
+ },
|
|
|
+ "omitted_count": sum(int(item.get("omitted_count") or 0) for item in events),
|
|
|
+ "detail_reads": sum(int(item.get("detail_reads") or 0) for item in events),
|
|
|
+ "duplicate_detail_reads": len(detail_pages) - len(set(detail_pages)),
|
|
|
+ "semantic_calls": sum(int(item.get("semantic_calls") or 0) for item in events),
|
|
|
+ "semantic_fallbacks": sum(bool(item.get("fallback")) for item in events),
|
|
|
+ "semantic_cache_hits": sum(bool(item.get("cache_hit")) for item in events),
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def _percentile(values: list[int], ratio: float) -> int:
|
|
|
+ if not values:
|
|
|
+ return 0
|
|
|
+ return values[min(len(values) - 1, round((len(values) - 1) * ratio))]
|
|
|
+
|
|
|
+
|
|
|
def _empty_preset_usage() -> dict[str, int | float]:
|
|
|
return {
|
|
|
"agent_runs": 0,
|