store.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868
  1. """
  2. FileSystem Trace Store - 文件系统存储实现
  3. 用于跨进程数据共享,数据持久化到 .trace/ 目录
  4. 目录结构:
  5. .trace/{trace_id}/
  6. ├── meta.json # Trace 元数据
  7. ├── goal.json # GoalTree(扁平 JSON,通过 parent_id 构建层级)
  8. ├── messages/ # Messages(每条独立文件)
  9. │ ├── {message_id}.json
  10. │ └── ...
  11. └── events.jsonl # 事件流(WebSocket 续传)
  12. Sub-Trace 是完全独立的 Trace,有自己的目录:
  13. .trace/{parent_id}@{mode}-{timestamp}-{seq}/
  14. ├── meta.json # parent_trace_id 指向父 Trace
  15. ├── goal.json
  16. ├── messages/
  17. └── events.jsonl
  18. """
  19. import json
  20. import os
  21. import logging
  22. from pathlib import Path
  23. from typing import Dict, List, Optional, Any
  24. from datetime import datetime
  25. from .models import Trace, Message
  26. from .goal_models import GoalTree, Goal, GoalStats
  27. logger = logging.getLogger(__name__)
  28. class FileSystemTraceStore:
  29. """文件系统 Trace 存储"""
  30. def __init__(self, base_path: str = ".trace"):
  31. self.base_path = Path(base_path)
  32. self.base_path.mkdir(exist_ok=True)
  33. def _get_trace_dir(self, trace_id: str) -> Path:
  34. """获取 trace 目录"""
  35. return self.base_path / trace_id
  36. def _get_meta_file(self, trace_id: str) -> Path:
  37. """获取 meta.json 文件路径"""
  38. return self._get_trace_dir(trace_id) / "meta.json"
  39. def _get_goal_file(self, trace_id: str) -> Path:
  40. """获取 goal.json 文件路径"""
  41. return self._get_trace_dir(trace_id) / "goal.json"
  42. def _get_messages_dir(self, trace_id: str) -> Path:
  43. """获取 messages 目录"""
  44. return self._get_trace_dir(trace_id) / "messages"
  45. def _get_message_file(self, trace_id: str, message_id: str) -> Path:
  46. """获取 message 文件路径"""
  47. return self._get_messages_dir(trace_id) / f"{message_id}.json"
  48. def _get_events_file(self, trace_id: str) -> Path:
  49. """获取 events.jsonl 文件路径"""
  50. return self._get_trace_dir(trace_id) / "events.jsonl"
  51. def _get_model_usage_file(self, trace_id: str) -> Path:
  52. """获取 model_usage.json 文件路径"""
  53. return self._get_trace_dir(trace_id) / "model_usage.json"
  54. # ===== Trace 操作 =====
  55. async def create_trace(self, trace: Trace) -> str:
  56. """创建新的 Trace"""
  57. trace_dir = self._get_trace_dir(trace.trace_id)
  58. trace_dir.mkdir(exist_ok=True)
  59. # 创建 messages 目录
  60. messages_dir = self._get_messages_dir(trace.trace_id)
  61. messages_dir.mkdir(exist_ok=True)
  62. # 写入 meta.json
  63. meta_file = self._get_meta_file(trace.trace_id)
  64. meta_file.write_text(json.dumps(trace.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
  65. # 创建空的 events.jsonl
  66. events_file = self._get_events_file(trace.trace_id)
  67. events_file.touch()
  68. return trace.trace_id
  69. async def get_trace(self, trace_id: str) -> Optional[Trace]:
  70. """获取 Trace"""
  71. meta_file = self._get_meta_file(trace_id)
  72. if not meta_file.exists():
  73. return None
  74. data = json.loads(meta_file.read_text(encoding="utf-8"))
  75. # 解析 datetime 字段
  76. if data.get("created_at"):
  77. data["created_at"] = datetime.fromisoformat(data["created_at"])
  78. if data.get("completed_at"):
  79. data["completed_at"] = datetime.fromisoformat(data["completed_at"])
  80. return Trace.from_dict(data)
  81. async def update_trace(self, trace_id: str, **updates) -> None:
  82. """更新 Trace"""
  83. trace = await self.get_trace(trace_id)
  84. if not trace:
  85. return
  86. # 更新字段
  87. for key, value in updates.items():
  88. if hasattr(trace, key):
  89. setattr(trace, key, value)
  90. # 写回文件
  91. meta_file = self._get_meta_file(trace_id)
  92. meta_file.write_text(json.dumps(trace.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
  93. async def list_traces(
  94. self,
  95. mode: Optional[str] = None,
  96. agent_type: Optional[str] = None,
  97. uid: Optional[str] = None,
  98. status: Optional[str] = None,
  99. limit: int = 50
  100. ) -> List[Trace]:
  101. """列出 Traces"""
  102. traces = []
  103. if not self.base_path.exists():
  104. return []
  105. for trace_dir in self.base_path.iterdir():
  106. if not trace_dir.is_dir():
  107. continue
  108. meta_file = trace_dir / "meta.json"
  109. if not meta_file.exists():
  110. continue
  111. try:
  112. data = json.loads(meta_file.read_text(encoding="utf-8"))
  113. # 过滤
  114. if mode and data.get("mode") != mode:
  115. continue
  116. if agent_type and data.get("agent_type") != agent_type:
  117. continue
  118. if uid and data.get("uid") != uid:
  119. continue
  120. if status and data.get("status") != status:
  121. continue
  122. # 解析 datetime
  123. if data.get("created_at"):
  124. data["created_at"] = datetime.fromisoformat(data["created_at"])
  125. if data.get("completed_at"):
  126. data["completed_at"] = datetime.fromisoformat(data["completed_at"])
  127. traces.append(Trace.from_dict(data))
  128. except Exception:
  129. continue
  130. # 排序(最新的在前)
  131. traces.sort(key=lambda t: t.created_at, reverse=True)
  132. return traces[:limit]
  133. # ===== GoalTree 操作 =====
  134. async def get_goal_tree(self, trace_id: str) -> Optional[GoalTree]:
  135. """获取 GoalTree"""
  136. goal_file = self._get_goal_file(trace_id)
  137. if not goal_file.exists():
  138. return None
  139. try:
  140. data = json.loads(goal_file.read_text(encoding="utf-8"))
  141. return GoalTree.from_dict(data)
  142. except Exception:
  143. return None
  144. async def update_goal_tree(self, trace_id: str, tree: GoalTree) -> None:
  145. """更新完整 GoalTree"""
  146. goal_file = self._get_goal_file(trace_id)
  147. goal_file.write_text(json.dumps(tree.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
  148. async def add_goal(self, trace_id: str, goal: Goal) -> None:
  149. """添加 Goal 到 GoalTree"""
  150. tree = await self.get_goal_tree(trace_id)
  151. if not tree:
  152. return
  153. tree.goals.append(goal)
  154. await self.update_goal_tree(trace_id, tree)
  155. # 推送 goal_added 事件
  156. event_data = {
  157. "goal": goal.to_dict(),
  158. "parent_id": goal.parent_id
  159. }
  160. await self.append_event(trace_id, "goal_added", event_data)
  161. # 打印详细的 goal 信息
  162. desc_preview = goal.description[:80] + "..." if len(goal.description) > 80 else goal.description
  163. print(f"[Goal Added] ID={goal.id}, Parent={goal.parent_id or 'root'}")
  164. print(f" 📝 {desc_preview}")
  165. if goal.reason:
  166. reason_preview = goal.reason[:60] + "..." if len(goal.reason) > 60 else goal.reason
  167. print(f" 💡 {reason_preview}")
  168. async def update_goal(self, trace_id: str, goal_id: str, **updates) -> None:
  169. """更新 Goal 字段"""
  170. tree = await self.get_goal_tree(trace_id)
  171. if not tree:
  172. return
  173. goal = tree.find(goal_id)
  174. if not goal:
  175. return
  176. # 更新字段
  177. for key, value in updates.items():
  178. if hasattr(goal, key):
  179. # 特殊处理 stats 字段(可能是 dict)
  180. if key in ["self_stats", "cumulative_stats"] and isinstance(value, dict):
  181. value = GoalStats.from_dict(value)
  182. setattr(goal, key, value)
  183. await self.update_goal_tree(trace_id, tree)
  184. # 推送 goal_updated 事件
  185. # 如果状态变为 completed,检查是否需要级联完成父 Goal
  186. affected_goals = [{"goal_id": goal_id, "updates": updates}]
  187. if updates.get("status") == "completed":
  188. # 检查级联完成:如果所有兄弟 Goal 都完成,父 Goal 也完成
  189. cascade_completed = await self._check_cascade_completion(trace_id, goal)
  190. affected_goals.extend(cascade_completed)
  191. await self.append_event(trace_id, "goal_updated", {
  192. "goal_id": goal_id,
  193. "updates": updates,
  194. "affected_goals": affected_goals
  195. })
  196. print(f"[DEBUG] Pushed goal_updated event: goal_id={goal_id}, updates={updates}, affected={len(affected_goals)}")
  197. # Goal 完成时触发知识评估
  198. if updates.get("status") in ["completed", "abandoned"]:
  199. pending = await self.get_pending_knowledge_entries(trace_id)
  200. if pending:
  201. # 在trace.context中设置标志,由runner主循环检查
  202. trace = await self.get_trace(trace_id)
  203. if trace:
  204. if not trace.context:
  205. trace.context = {}
  206. trace.context["pending_knowledge_eval"] = True
  207. trace.context["knowledge_eval_trigger"] = "goal_completion"
  208. await self.update_trace(trace_id, context=trace.context)
  209. logger.info(f"[Knowledge Eval] Goal {goal_id} 完成,设置评估标志,待评估知识: {len(pending)} 条")
  210. async def _check_cascade_completion(
  211. self,
  212. trace_id: str,
  213. completed_goal: Goal
  214. ) -> List[Dict[str, Any]]:
  215. """
  216. 检查级联完成:如果一个 Goal 的所有子 Goal 都完成,则自动完成父 Goal
  217. Args:
  218. trace_id: Trace ID
  219. completed_goal: 刚完成的 Goal
  220. Returns:
  221. 受影响的父 Goals 列表(自动完成的)
  222. """
  223. if not completed_goal.parent_id:
  224. return []
  225. tree = await self.get_goal_tree(trace_id)
  226. if not tree:
  227. return []
  228. affected = []
  229. parent = tree.find(completed_goal.parent_id)
  230. if not parent:
  231. return []
  232. # 获取父 Goal 的所有子 Goal
  233. children = tree.get_children(parent.id)
  234. # 检查是否所有子 Goal 都已完成(排除 abandoned)
  235. all_completed = all(
  236. child.status in ["completed", "abandoned"]
  237. for child in children
  238. )
  239. if all_completed and parent.status != "completed":
  240. # 自动完成父 Goal
  241. parent.status = "completed"
  242. if not parent.summary:
  243. # 生成自动摘要
  244. completed_count = sum(1 for c in children if c.status == "completed")
  245. parent.summary = f"所有子目标已完成 ({completed_count}/{len(children)})"
  246. await self.update_goal_tree(trace_id, tree)
  247. affected.append({
  248. "goal_id": parent.id,
  249. "status": "completed",
  250. "summary": parent.summary,
  251. "cumulative_stats": parent.cumulative_stats.to_dict()
  252. })
  253. # 递归检查祖父 Goal
  254. grandparent_affected = await self._check_cascade_completion(trace_id, parent)
  255. affected.extend(grandparent_affected)
  256. return affected
  257. # ===== Message 操作 =====
  258. async def add_message(self, message: Message) -> str:
  259. """
  260. 添加 Message
  261. 自动更新关联 Goal 的 stats(self_stats 和祖先的 cumulative_stats)
  262. """
  263. trace_id = message.trace_id
  264. # 1. 写入 message 文件
  265. messages_dir = self._get_messages_dir(trace_id)
  266. message_file = messages_dir / f"{message.message_id}.json"
  267. message_file.write_text(json.dumps(message.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
  268. # 2. 更新 trace 统计
  269. trace = await self.get_trace(trace_id)
  270. if trace:
  271. trace.total_messages += 1
  272. trace.last_sequence = max(trace.last_sequence, message.sequence)
  273. # 累计 tokens(完整版)
  274. if message.prompt_tokens:
  275. trace.total_prompt_tokens += message.prompt_tokens
  276. if message.completion_tokens:
  277. trace.total_completion_tokens += message.completion_tokens
  278. if message.reasoning_tokens:
  279. trace.total_reasoning_tokens += message.reasoning_tokens
  280. if message.cache_creation_tokens:
  281. trace.total_cache_creation_tokens += message.cache_creation_tokens
  282. if message.cache_read_tokens:
  283. trace.total_cache_read_tokens += message.cache_read_tokens
  284. # 向后兼容:也更新 total_tokens
  285. if message.tokens:
  286. trace.total_tokens += message.tokens
  287. elif message.prompt_tokens or message.completion_tokens:
  288. trace.total_tokens += (message.prompt_tokens or 0) + (message.completion_tokens or 0)
  289. if message.cost:
  290. trace.total_cost += message.cost
  291. if message.duration_ms:
  292. trace.total_duration_ms += message.duration_ms
  293. # 更新 Trace
  294. await self.update_trace(
  295. trace_id,
  296. total_messages=trace.total_messages,
  297. last_sequence=trace.last_sequence,
  298. total_tokens=trace.total_tokens,
  299. total_prompt_tokens=trace.total_prompt_tokens,
  300. total_completion_tokens=trace.total_completion_tokens,
  301. total_reasoning_tokens=trace.total_reasoning_tokens,
  302. total_cache_creation_tokens=trace.total_cache_creation_tokens,
  303. total_cache_read_tokens=trace.total_cache_read_tokens,
  304. total_cost=trace.total_cost,
  305. total_duration_ms=trace.total_duration_ms
  306. )
  307. # 3. 更新 Goal stats
  308. await self._update_goal_stats(trace_id, message)
  309. # 4. 追加 message_added 事件
  310. affected_goals = await self._get_affected_goals(trace_id, message)
  311. event_id = await self.append_event(trace_id, "message_added", {
  312. "message": message.to_dict(),
  313. "affected_goals": affected_goals
  314. })
  315. if event_id:
  316. try:
  317. from . import websocket as trace_ws
  318. await trace_ws.broadcast_message_added(
  319. trace_id=trace_id,
  320. event_id=event_id,
  321. message_dict=message.to_dict(),
  322. affected_goals=affected_goals,
  323. )
  324. except Exception:
  325. logger.exception("Failed to broadcast message_added (trace_id=%s, event_id=%s)", trace_id, event_id)
  326. return message.message_id
  327. async def _update_goal_stats(self, trace_id: str, message: Message) -> None:
  328. """更新 Goal 的 self_stats 和祖先的 cumulative_stats"""
  329. tree = await self.get_goal_tree(trace_id)
  330. if not tree:
  331. return
  332. # 找到关联的 Goal
  333. goal = tree.find(message.goal_id)
  334. if not goal:
  335. return
  336. # 更新自身 self_stats
  337. goal.self_stats.message_count += 1
  338. if message.tokens:
  339. goal.self_stats.total_tokens += message.tokens
  340. if message.cost:
  341. goal.self_stats.total_cost += message.cost
  342. # TODO: 更新 preview(工具调用摘要)
  343. # 更新自身 cumulative_stats
  344. goal.cumulative_stats.message_count += 1
  345. if message.tokens:
  346. goal.cumulative_stats.total_tokens += message.tokens
  347. if message.cost:
  348. goal.cumulative_stats.total_cost += message.cost
  349. # 沿祖先链向上更新 cumulative_stats
  350. current_goal = goal
  351. while current_goal.parent_id:
  352. parent = tree.find(current_goal.parent_id)
  353. if not parent:
  354. break
  355. parent.cumulative_stats.message_count += 1
  356. if message.tokens:
  357. parent.cumulative_stats.total_tokens += message.tokens
  358. if message.cost:
  359. parent.cumulative_stats.total_cost += message.cost
  360. current_goal = parent
  361. # 保存更新后的 tree
  362. await self.update_goal_tree(trace_id, tree)
  363. async def _get_affected_goals(self, trace_id: str, message: Message) -> List[Dict[str, Any]]:
  364. """获取受影响的 Goals(自身 + 所有祖先)"""
  365. tree = await self.get_goal_tree(trace_id)
  366. if not tree:
  367. return []
  368. goal = tree.find(message.goal_id)
  369. if not goal:
  370. return []
  371. affected = []
  372. # 添加自身(包含 self_stats 和 cumulative_stats)
  373. affected.append({
  374. "goal_id": goal.id,
  375. "self_stats": goal.self_stats.to_dict(),
  376. "cumulative_stats": goal.cumulative_stats.to_dict()
  377. })
  378. # 添加所有祖先(仅 cumulative_stats)
  379. current_goal = goal
  380. while current_goal.parent_id:
  381. parent = tree.find(current_goal.parent_id)
  382. if not parent:
  383. break
  384. affected.append({
  385. "goal_id": parent.id,
  386. "cumulative_stats": parent.cumulative_stats.to_dict()
  387. })
  388. current_goal = parent
  389. return affected
  390. return affected
  391. async def get_message(self, message_id: str) -> Optional[Message]:
  392. """获取 Message(扫描所有 trace)"""
  393. for trace_dir in self.base_path.iterdir():
  394. if not trace_dir.is_dir():
  395. continue
  396. # 检查 messages 目录
  397. message_file = trace_dir / "messages" / f"{message_id}.json"
  398. if message_file.exists():
  399. try:
  400. data = json.loads(message_file.read_text(encoding="utf-8"))
  401. return Message.from_dict(data)
  402. except Exception:
  403. pass
  404. return None
  405. async def get_trace_messages(
  406. self,
  407. trace_id: str,
  408. ) -> List[Message]:
  409. """获取 Trace 的所有 Messages(包含所有分支,按 sequence 排序)"""
  410. messages_dir = self._get_messages_dir(trace_id)
  411. if not messages_dir.exists():
  412. return []
  413. messages = []
  414. for message_file in messages_dir.glob("*.json"):
  415. try:
  416. data = json.loads(message_file.read_text(encoding="utf-8"))
  417. msg = Message.from_dict(data)
  418. messages.append(msg)
  419. except Exception:
  420. continue
  421. # 按 sequence 排序
  422. messages.sort(key=lambda m: m.sequence)
  423. return messages
  424. async def get_main_path_messages(
  425. self,
  426. trace_id: str,
  427. head_sequence: int
  428. ) -> List[Message]:
  429. """
  430. 获取从 head_sequence 沿 parent_sequence 链回溯到 root 的完整路径
  431. 此函数是通用的路径追溯函数,返回从指定 head 到 root 的完整消息链。
  432. 只要 trace.head_sequence 管理正确(指向主路径),此函数自然返回主路径消息。
  433. 侧分支消息通过 parent_sequence 链自然被跳过(因为主路径的 parent 不指向侧分支)。
  434. Returns:
  435. 按 sequence 正序排列的路径 Message 列表
  436. """
  437. # 加载所有消息,建立 sequence -> Message 索引
  438. all_messages = await self.get_trace_messages(trace_id)
  439. messages_by_seq = {m.sequence: m for m in all_messages}
  440. # 从 head 沿 parent chain 回溯
  441. path = []
  442. seq = head_sequence
  443. while seq is not None:
  444. msg = messages_by_seq.get(seq)
  445. if not msg:
  446. break
  447. path.append(msg)
  448. seq = msg.parent_sequence
  449. # 反转为正序(root → head)
  450. path.reverse()
  451. return path
  452. async def get_messages_by_goal(
  453. self,
  454. trace_id: str,
  455. goal_id: str
  456. ) -> List[Message]:
  457. """获取指定 Goal 关联的所有 Messages"""
  458. all_messages = await self.get_trace_messages(trace_id)
  459. return [m for m in all_messages if m.goal_id == goal_id]
  460. async def update_message(self, message_id: str, **updates) -> None:
  461. """更新 Message 字段"""
  462. message = await self.get_message(message_id)
  463. if not message:
  464. return
  465. # 更新字段
  466. for key, value in updates.items():
  467. if hasattr(message, key):
  468. setattr(message, key, value)
  469. # 确定文件路径
  470. messages_dir = self._get_messages_dir(message.trace_id)
  471. message_file = messages_dir / f"{message_id}.json"
  472. message_file.write_text(json.dumps(message.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
  473. async def abandon_messages_after(self, trace_id: str, cutoff_sequence: int) -> List[str]:
  474. """
  475. 将 sequence > cutoff_sequence 的 active messages 标记为 abandoned。
  476. 返回被 abandon 的 message_id 列表。
  477. """
  478. all_messages = await self.get_trace_messages(trace_id)
  479. abandoned_ids = []
  480. now = datetime.now()
  481. for msg in all_messages:
  482. if msg.sequence > cutoff_sequence and msg.status == "active":
  483. msg.status = "abandoned"
  484. msg.abandoned_at = now
  485. # 直接写回文件
  486. message_file = self._get_messages_dir(trace_id) / f"{msg.message_id}.json"
  487. message_file.write_text(
  488. json.dumps(msg.to_dict(), indent=2, ensure_ascii=False),
  489. encoding="utf-8"
  490. )
  491. abandoned_ids.append(msg.message_id)
  492. return abandoned_ids
  493. # ===== 模型使用追踪 =====
  494. async def record_model_usage(
  495. self,
  496. trace_id: str,
  497. sequence: int,
  498. role: str,
  499. model: str,
  500. prompt_tokens: int,
  501. completion_tokens: int,
  502. cache_read_tokens: int = 0,
  503. tool_name: Optional[str] = None,
  504. ) -> None:
  505. """
  506. 记录模型使用情况到 model_usage.json
  507. Args:
  508. trace_id: Trace ID
  509. sequence: 消息序号
  510. role: 角色(assistant/tool)
  511. model: 模型名称
  512. prompt_tokens: 输入tokens
  513. completion_tokens: 输出tokens
  514. cache_read_tokens: 缓存读取tokens
  515. tool_name: 工具名称(role=tool时)
  516. """
  517. usage_file = self._get_model_usage_file(trace_id)
  518. # 读取现有数据
  519. if usage_file.exists():
  520. data = json.loads(usage_file.read_text(encoding="utf-8"))
  521. else:
  522. data = {
  523. "summary": {
  524. "total_models": 0,
  525. "total_tokens": 0,
  526. "total_cache_read_tokens": 0,
  527. "agent_tokens": 0,
  528. "tool_tokens": 0,
  529. },
  530. "models": [],
  531. "timeline": [],
  532. }
  533. # 更新summary
  534. total_tokens = prompt_tokens + completion_tokens
  535. data["summary"]["total_tokens"] += total_tokens
  536. data["summary"]["total_cache_read_tokens"] += cache_read_tokens
  537. if role == "assistant":
  538. data["summary"]["agent_tokens"] += total_tokens
  539. source = "agent"
  540. else:
  541. data["summary"]["tool_tokens"] += total_tokens
  542. source = f"tool:{tool_name}" if tool_name else "tool"
  543. # 更新models列表
  544. model_entry = None
  545. for m in data["models"]:
  546. if m["model"] == model and m["source"] == source:
  547. model_entry = m
  548. break
  549. if model_entry:
  550. model_entry["prompt_tokens"] += prompt_tokens
  551. model_entry["completion_tokens"] += completion_tokens
  552. model_entry["total_tokens"] += total_tokens
  553. model_entry["cache_read_tokens"] += cache_read_tokens
  554. model_entry["call_count"] += 1
  555. else:
  556. data["models"].append({
  557. "model": model,
  558. "source": source,
  559. "prompt_tokens": prompt_tokens,
  560. "completion_tokens": completion_tokens,
  561. "total_tokens": total_tokens,
  562. "cache_read_tokens": cache_read_tokens,
  563. "call_count": 1,
  564. })
  565. data["summary"]["total_models"] = len(data["models"])
  566. # 添加到timeline
  567. timeline_entry = {
  568. "sequence": sequence,
  569. "role": role,
  570. "model": model,
  571. "prompt_tokens": prompt_tokens,
  572. "completion_tokens": completion_tokens,
  573. }
  574. if cache_read_tokens > 0:
  575. timeline_entry["cache_read_tokens"] = cache_read_tokens
  576. if tool_name:
  577. timeline_entry["tool_name"] = tool_name
  578. data["timeline"].append(timeline_entry)
  579. # 写回文件
  580. usage_file.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
  581. # ===== 事件流操作(用于 WebSocket 断线续传)=====
  582. async def get_events(
  583. self,
  584. trace_id: str,
  585. since_event_id: int = 0
  586. ) -> List[Dict[str, Any]]:
  587. """获取事件流"""
  588. events_file = self._get_events_file(trace_id)
  589. if not events_file.exists():
  590. return []
  591. events = []
  592. with events_file.open('r', encoding='utf-8') as f:
  593. for line in f:
  594. try:
  595. event = json.loads(line.strip())
  596. if event.get("event_id", 0) > since_event_id:
  597. events.append(event)
  598. except Exception:
  599. continue
  600. return events
  601. async def append_event(
  602. self,
  603. trace_id: str,
  604. event_type: str,
  605. payload: Dict[str, Any]
  606. ) -> int:
  607. """追加事件,返回 event_id"""
  608. # 获取 trace 并递增 event_id
  609. trace = await self.get_trace(trace_id)
  610. if not trace:
  611. return 0
  612. trace.last_event_id += 1
  613. event_id = trace.last_event_id
  614. # 更新 trace 的 last_event_id
  615. await self.update_trace(trace_id, last_event_id=event_id)
  616. # 创建事件
  617. event = {
  618. "event_id": event_id,
  619. "event": event_type,
  620. "ts": datetime.now().isoformat(),
  621. **payload
  622. }
  623. # 追加到 events.jsonl
  624. events_file = self._get_events_file(trace_id)
  625. with events_file.open('a', encoding='utf-8') as f:
  626. f.write(json.dumps(event, ensure_ascii=False) + '\n')
  627. return event_id
  628. # ===== Knowledge Log 管理 =====
  629. def _get_knowledge_log_file(self, trace_id: str) -> Path:
  630. """获取 knowledge_log.json 文件路径"""
  631. return self._get_trace_dir(trace_id) / "knowledge_log.json"
  632. async def get_knowledge_log(self, trace_id: str) -> Dict[str, Any]:
  633. """读取知识日志"""
  634. log_file = self._get_knowledge_log_file(trace_id)
  635. if not log_file.exists():
  636. return {"trace_id": trace_id, "entries": []}
  637. return json.loads(log_file.read_text(encoding="utf-8"))
  638. async def append_knowledge_entry(
  639. self,
  640. trace_id: str,
  641. knowledge_id: str,
  642. goal_id: str,
  643. injected_at_sequence: int,
  644. task: str,
  645. content: str
  646. ) -> None:
  647. """追加知识注入记录"""
  648. log = await self.get_knowledge_log(trace_id)
  649. log["entries"].append({
  650. "knowledge_id": knowledge_id,
  651. "goal_id": goal_id,
  652. "injected_at_sequence": injected_at_sequence,
  653. "injected_at": datetime.now().isoformat(),
  654. "task": task,
  655. "content": content[:500], # 限制长度
  656. "eval_result": None,
  657. "evaluated_at": None,
  658. "evaluated_at_trigger": None
  659. })
  660. log_file = self._get_knowledge_log_file(trace_id)
  661. log_file.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")
  662. async def update_knowledge_evaluation(
  663. self,
  664. trace_id: str,
  665. knowledge_id: str,
  666. eval_result: Dict[str, Any],
  667. trigger_event: str
  668. ) -> None:
  669. """更新知识评估结果
  670. 当同一个knowledge_id在不同goal中被多次注入时,
  671. 优先更新最近一个未评估的条目(按injected_at_sequence倒序)
  672. """
  673. log = await self.get_knowledge_log(trace_id)
  674. # 找到所有匹配且未评估的条目
  675. matching_entries = [
  676. (i, entry) for i, entry in enumerate(log["entries"])
  677. if entry["knowledge_id"] == knowledge_id and entry["eval_result"] is None
  678. ]
  679. if matching_entries:
  680. # 按injected_at_sequence倒序排序,取最近的一个
  681. matching_entries.sort(key=lambda x: x[1]["injected_at_sequence"], reverse=True)
  682. idx, entry = matching_entries[0]
  683. entry["eval_result"] = eval_result
  684. entry["evaluated_at"] = datetime.now().isoformat()
  685. entry["evaluated_at_trigger"] = trigger_event
  686. log_file = self._get_knowledge_log_file(trace_id)
  687. log_file.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")
  688. async def get_pending_knowledge_entries(self, trace_id: str) -> List[Dict[str, Any]]:
  689. """获取所有待评估的知识条目"""
  690. log = await self.get_knowledge_log(trace_id)
  691. return [e for e in log["entries"] if e["eval_result"] is None]
  692. async def update_user_feedback(
  693. self,
  694. trace_id: str,
  695. knowledge_id: str,
  696. user_feedback: Dict[str, Any]
  697. ) -> None:
  698. """记录用户对知识的反馈(confirm/override),不覆盖 agent 的 eval_result
  699. 当同一个 knowledge_id 被多次注入时,更新最近一次注入的条目。
  700. """
  701. log = await self.get_knowledge_log(trace_id)
  702. # 找到所有匹配的条目(不限 eval_result 是否为 None)
  703. matching_entries = [
  704. (i, entry) for i, entry in enumerate(log["entries"])
  705. if entry["knowledge_id"] == knowledge_id
  706. ]
  707. if matching_entries:
  708. # 按 injected_at_sequence 倒序,取最近一次注入的条目
  709. matching_entries.sort(key=lambda x: x[1]["injected_at_sequence"], reverse=True)
  710. idx, entry = matching_entries[0]
  711. entry["user_feedback"] = user_feedback
  712. log_file = self._get_knowledge_log_file(trace_id)
  713. log_file.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")