runtime_event_index.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362
  1. from __future__ import annotations
  2. import json
  3. from collections import defaultdict
  4. from dataclasses import dataclass
  5. from typing import Any, Iterable, Iterator
  6. from .runtime_tool_catalog import MAIN_DECISION_READ_TOOLS
  7. _SUCCESS_STATUSES = {"ok", "success", "succeeded", "completed", "complete"}
  8. _FAILURE_STATUSES = {"error", "failed", "failure"}
  9. @dataclass(frozen=True)
  10. class EventBounds:
  11. """Sequence bounds used by projectors; both ends are exclusive by default."""
  12. after_seq: int | None = None
  13. before_seq: int | None = None
  14. class RuntimeEventIndex:
  15. """Read-only structural index over ``script_build_event`` projections.
  16. This class intentionally knows nothing about card wording. It establishes
  17. event identity, order, hierarchy, scope and actor once, then exposes strict
  18. queries used by the three main-Agent decision projectors.
  19. """
  20. def __init__(self, events: Iterable[dict[str, Any]]):
  21. ordered = sorted((dict(item) for item in events), key=event_order)
  22. self._ordered: tuple[dict[str, Any], ...] = tuple(ordered)
  23. self._by_id: dict[int, dict[str, Any]] = {}
  24. self._by_name: dict[str, list[dict[str, Any]]] = defaultdict(list)
  25. self._by_scope: dict[int, list[dict[str, Any]]] = defaultdict(list)
  26. self._children: dict[int, list[dict[str, Any]]] = defaultdict(list)
  27. self._by_actor: dict[str, list[dict[str, Any]]] = defaultdict(list)
  28. self._by_round: dict[int, list[dict[str, Any]]] = defaultdict(list)
  29. for event in ordered:
  30. event_id = integer(event.get("id"))
  31. if event_id is not None:
  32. self._by_id[event_id] = event
  33. self._by_name[str(event.get("event_name") or "")].append(event)
  34. scope_id = integer(event.get("scope_event_id"))
  35. if scope_id is not None:
  36. self._by_scope[scope_id].append(event)
  37. parent_id = integer(event.get("parent_event_id"))
  38. if parent_id is not None:
  39. self._children[parent_id].append(event)
  40. self._by_actor[event_actor(event)].append(event)
  41. round_index = integer(event.get("round_index"))
  42. if round_index is not None:
  43. self._by_round[round_index].append(event)
  44. self._main_scope_ids = frozenset(
  45. event_id
  46. for event in ordered
  47. if is_main_scope(event)
  48. and (event_id := integer(event.get("scope_event_id")) or integer(event.get("id")))
  49. is not None
  50. )
  51. @property
  52. def ordered(self) -> tuple[dict[str, Any], ...]:
  53. return self._ordered
  54. @property
  55. def main_scope_ids(self) -> frozenset[int]:
  56. return self._main_scope_ids
  57. def by_id(self, event_id: int | None) -> dict[str, Any] | None:
  58. return self._by_id.get(int(event_id)) if event_id is not None else None
  59. def children_of(self, event_id: int) -> tuple[dict[str, Any], ...]:
  60. return tuple(self._children.get(int(event_id), ()))
  61. def events_in_scope(self, scope_id: int) -> tuple[dict[str, Any], ...]:
  62. return tuple(self._by_scope.get(int(scope_id), ()))
  63. def events_for_actor(self, actor: str) -> tuple[dict[str, Any], ...]:
  64. return tuple(self._by_actor.get(str(actor), ()))
  65. def events_for_round(self, round_index: int) -> tuple[dict[str, Any], ...]:
  66. return tuple(self._by_round.get(int(round_index), ()))
  67. def tools_in_scope(
  68. self, scope_id: int, *, names: set[str] | frozenset[str] | None = None
  69. ) -> list[dict[str, Any]]:
  70. return [
  71. event
  72. for event in self.events_in_scope(scope_id)
  73. if str(event.get("event_type") or "") == "tool_call"
  74. and (names is None or str(event.get("event_name") or "") in names)
  75. ]
  76. def named(
  77. self,
  78. name: str,
  79. *,
  80. event_type: str | None = None,
  81. successful: bool | None = None,
  82. bounds: EventBounds | None = None,
  83. ) -> list[dict[str, Any]]:
  84. events = self._by_name.get(str(name), ())
  85. return [
  86. event
  87. for event in events
  88. if (event_type is None or str(event.get("event_type") or "") == event_type)
  89. and (successful is None or event_succeeded(event) is successful)
  90. and _within_bounds(event, bounds)
  91. ]
  92. def anchors(
  93. self,
  94. name: str,
  95. *,
  96. successful: bool | None = None,
  97. bounds: EventBounds | None = None,
  98. ) -> list[dict[str, Any]]:
  99. """Return only anchor calls made directly by a real main scope."""
  100. return [
  101. event
  102. for event in self.named(
  103. name,
  104. event_type="tool_call",
  105. successful=successful,
  106. bounds=bounds,
  107. )
  108. if self.is_main_direct_event(event)
  109. ]
  110. def is_main_direct_event(self, event: dict[str, Any]) -> bool:
  111. return (
  112. str(event.get("agent_role") or "") == "main"
  113. and integer(event.get("agent_depth")) == 0
  114. and integer(event.get("scope_event_id")) in self._main_scope_ids
  115. )
  116. def main_direct_reads(
  117. self,
  118. *,
  119. scope_id: int | None = None,
  120. names: set[str] | frozenset[str] | None = None,
  121. bounds: EventBounds | None = None,
  122. successful: bool | None = True,
  123. ) -> list[dict[str, Any]]:
  124. allowed = names or MAIN_DECISION_READ_TOOLS
  125. return [
  126. event
  127. for event in self._ordered
  128. if str(event.get("event_type") or "") == "tool_call"
  129. and str(event.get("event_name") or "") in allowed
  130. and self.is_main_direct_event(event)
  131. and (scope_id is None or integer(event.get("scope_event_id")) == scope_id)
  132. and _within_bounds(event, bounds)
  133. and (successful is None or event_succeeded(event) is successful)
  134. ]
  135. def agent_returns(
  136. self,
  137. name: str,
  138. *,
  139. round_index: int | None = None,
  140. before_seq: int | None = None,
  141. ) -> list[tuple[dict[str, Any], str]]:
  142. """Return evaluator calls and their honest association strength.
  143. ``returned-report`` requires an explicit parent main scope. A matching
  144. structured round without that hierarchy is only ``runtime-associated``.
  145. Report text is never inspected to infer its round.
  146. """
  147. results: list[tuple[dict[str, Any], str]] = []
  148. for event in self.named(name, event_type="agent_invoke", successful=True):
  149. if before_seq is not None and event_sequence(event) >= before_seq:
  150. continue
  151. if round_index is not None and integer(event.get("round_index")) != round_index:
  152. continue
  153. parent_id = integer(event.get("parent_event_id"))
  154. relation = (
  155. "returned-report"
  156. if parent_id in self._main_scope_ids
  157. else "runtime-associated"
  158. )
  159. results.append((event, relation))
  160. return results
  161. def resolved_begin_round(self, event: dict[str, Any]) -> int | None:
  162. """Resolve the newly opened round, never trusting the pre-call context first."""
  163. output = event_output(event)
  164. if isinstance(output, dict):
  165. round_index = integer(output.get("round_index"))
  166. if output.get("success") is not False and round_index is not None:
  167. return round_index
  168. seq = event_sequence(event)
  169. scope_id = integer(event.get("scope_event_id"))
  170. for marker in self._ordered:
  171. marker_seq = event_sequence(marker)
  172. if marker_seq <= seq:
  173. continue
  174. if str(marker.get("event_type") or "") == "tool_call" and str(
  175. marker.get("event_name") or ""
  176. ) == "begin_round":
  177. break
  178. if str(marker.get("event_type") or "") != "round_begin":
  179. continue
  180. marker_scope = integer(marker.get("scope_event_id"))
  181. if scope_id is not None and marker_scope not in {None, scope_id}:
  182. continue
  183. payload = marker.get("payload")
  184. round_index = integer(payload.get("round_index")) if isinstance(payload, dict) else None
  185. return round_index or integer(marker.get("event_name"))
  186. return None
  187. def successful_begin_rounds(self) -> dict[int, list[dict[str, Any]]]:
  188. result: dict[int, list[dict[str, Any]]] = defaultdict(list)
  189. for event in self.anchors("begin_round", successful=True):
  190. round_index = self.resolved_begin_round(event)
  191. if round_index is not None:
  192. result[round_index].append(event)
  193. return dict(result)
  194. def is_main_scope(event: dict[str, Any]) -> bool:
  195. return (
  196. str(event.get("event_type") or "") == "agent_scope"
  197. and str(event.get("event_name") or "") == "main"
  198. and str(event.get("agent_role") or "") == "main"
  199. and integer(event.get("agent_depth")) == 0
  200. )
  201. def event_actor(event: dict[str, Any]) -> str:
  202. role = str(event.get("agent_role") or "")
  203. name = str(event.get("event_name") or "")
  204. if role == "main" and integer(event.get("agent_depth")) == 0:
  205. return "main"
  206. if role == "script_evaluator" or name == "script_evaluator":
  207. return "overall-evaluator"
  208. if role == "script_multipath_evaluator" or name == "script_multipath_evaluator":
  209. return "multipath-evaluator"
  210. return role or "unknown"
  211. def event_order(event: dict[str, Any]) -> tuple[int, int]:
  212. return (event_sequence(event), integer(event.get("id")) or 0)
  213. def event_sequence(event: dict[str, Any]) -> int:
  214. return integer(event.get("event_seq")) or 0
  215. def event_succeeded(event: dict[str, Any]) -> bool:
  216. status = str(event.get("status") or "").lower()
  217. if status in _FAILURE_STATUSES or status == "running":
  218. return False
  219. output = event_output(event)
  220. if isinstance(output, dict):
  221. if output.get("success") is False:
  222. return False
  223. if output.get("error") not in (None, "", False, [], {}):
  224. return False
  225. return status in _SUCCESS_STATUSES or bool(event.get("ended_at"))
  226. def event_input(event: dict[str, Any]) -> Any:
  227. if "inputData" in event:
  228. return event.get("inputData")
  229. return _json_value(event.get("input_preview"))
  230. def event_output(event: dict[str, Any]) -> Any:
  231. for key in ("agentOutputData", "outputData"):
  232. if key in event and event.get(key) is not None:
  233. return _unwrap_output(event.get(key))
  234. return _unwrap_output(_json_value(event.get("output_preview")))
  235. def event_text_output(event: dict[str, Any]) -> str | None:
  236. output = event_output(event)
  237. if isinstance(output, dict):
  238. for key in ("summary", "report", "content", "message"):
  239. value = output.get(key)
  240. if isinstance(value, str) and value.strip():
  241. return value.strip()
  242. return None
  243. return output.strip() if isinstance(output, str) and output.strip() else None
  244. def detail_ref(event: dict[str, Any]) -> str | None:
  245. event_id = integer(event.get("id"))
  246. return f"event:{event_id}" if event_id is not None else None
  247. def event_summary(
  248. event: dict[str, Any], association: str | None
  249. ) -> dict[str, Any]:
  250. """Return a lightweight shared event reference without full body data."""
  251. event_id = integer(event.get("id"))
  252. return {
  253. "id": f"event:{event_id}" if event_id is not None else "event:unknown",
  254. "eventId": event_id,
  255. "eventSeq": event.get("event_seq"),
  256. "type": event.get("event_type"),
  257. "name": event.get("event_name"),
  258. "status": event.get("status"),
  259. "title": event.get("title"),
  260. "inputPreview": event.get("input_preview"),
  261. "outputPreview": event.get("output_preview"),
  262. "startedAt": event.get("started_at"),
  263. "endedAt": event.get("ended_at"),
  264. "association": association,
  265. "detailRef": f"event:{event_id}" if event_id is not None else None,
  266. }
  267. def integer(value: Any) -> int | None:
  268. try:
  269. return int(value) if value is not None else None
  270. except (TypeError, ValueError):
  271. return None
  272. def _within_bounds(event: dict[str, Any], bounds: EventBounds | None) -> bool:
  273. if bounds is None:
  274. return True
  275. seq = event_sequence(event)
  276. if bounds.after_seq is not None and seq <= bounds.after_seq:
  277. return False
  278. if bounds.before_seq is not None and seq >= bounds.before_seq:
  279. return False
  280. return True
  281. def _json_value(value: Any) -> Any:
  282. if not isinstance(value, str):
  283. return value
  284. text = value.strip()
  285. if not text:
  286. return None
  287. try:
  288. return json.loads(text)
  289. except json.JSONDecodeError:
  290. return value
  291. def _unwrap_output(value: Any) -> Any:
  292. # Tool bodies currently store many JSON results as a JSON-looking string.
  293. # Unwrap at most twice so malformed or ordinary prose remains untouched.
  294. result = value
  295. for _ in range(2):
  296. parsed = _json_value(result)
  297. if parsed is result or parsed == result:
  298. break
  299. result = parsed
  300. return result