|
|
@@ -20,12 +20,16 @@ Sub-Trace 是完全独立的 Trace,有自己的目录:
|
|
|
└── events.jsonl
|
|
|
"""
|
|
|
|
|
|
+import hashlib
|
|
|
import json
|
|
|
import logging
|
|
|
+import os
|
|
|
+import uuid
|
|
|
+from datetime import datetime
|
|
|
from pathlib import Path
|
|
|
from typing import Dict, List, Optional, Any
|
|
|
-from datetime import datetime
|
|
|
|
|
|
+from .attachments import AttachmentRef
|
|
|
from .models import Trace, Message
|
|
|
from .goal_models import GoalTree, Goal, GoalStats
|
|
|
|
|
|
@@ -67,6 +71,10 @@ class FileSystemTraceStore:
|
|
|
"""获取 model_usage.json 文件路径"""
|
|
|
return self._get_trace_dir(trace_id) / "model_usage.json"
|
|
|
|
|
|
+ def _get_attachments_dir(self, trace_id: str) -> Path:
|
|
|
+ """Return the durable attachment directory for one trace."""
|
|
|
+ return self._get_trace_dir(trace_id) / "attachments"
|
|
|
+
|
|
|
# ===== Trace 操作 =====
|
|
|
|
|
|
async def create_trace(self, trace: Trace) -> str:
|
|
|
@@ -78,9 +86,13 @@ class FileSystemTraceStore:
|
|
|
messages_dir = self._get_messages_dir(trace.trace_id)
|
|
|
messages_dir.mkdir(exist_ok=True)
|
|
|
|
|
|
+ self._get_attachments_dir(trace.trace_id).mkdir(exist_ok=True)
|
|
|
+
|
|
|
# 写入 meta.json
|
|
|
meta_file = self._get_meta_file(trace.trace_id)
|
|
|
- meta_file.write_text(json.dumps(trace.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
+ meta_file.write_text(
|
|
|
+ json.dumps(trace.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8"
|
|
|
+ )
|
|
|
|
|
|
# 创建空的 events.jsonl
|
|
|
events_file = self._get_events_file(trace.trace_id)
|
|
|
@@ -88,6 +100,103 @@ class FileSystemTraceStore:
|
|
|
|
|
|
return trace.trace_id
|
|
|
|
|
|
+ async def store_message_attachment(
|
|
|
+ self,
|
|
|
+ *,
|
|
|
+ trace_id: str,
|
|
|
+ message_id: str,
|
|
|
+ media_type: str,
|
|
|
+ content: bytes,
|
|
|
+ sha256: str,
|
|
|
+ ) -> AttachmentRef:
|
|
|
+ """Persist verified bytes without exposing filesystem paths."""
|
|
|
+ _validate_path_segment(trace_id, "trace_id")
|
|
|
+ _validate_path_segment(message_id, "message_id")
|
|
|
+ if not isinstance(content, bytes):
|
|
|
+ raise TypeError("content must be bytes")
|
|
|
+ actual = f"sha256:{hashlib.sha256(content).hexdigest()}"
|
|
|
+ if sha256 != actual:
|
|
|
+ raise ValueError("attachment digest mismatch")
|
|
|
+ if not self._get_meta_file(trace_id).exists():
|
|
|
+ raise ValueError(f"Trace not found: {trace_id}")
|
|
|
+ message = await self.get_message(message_id)
|
|
|
+ if message is None or message.trace_id != trace_id:
|
|
|
+ raise ValueError("attachment message does not belong to the Trace")
|
|
|
+
|
|
|
+ reference = AttachmentRef(
|
|
|
+ trace_id=trace_id,
|
|
|
+ message_id=message_id,
|
|
|
+ media_type=media_type,
|
|
|
+ sha256=actual,
|
|
|
+ size_bytes=len(content),
|
|
|
+ )
|
|
|
+
|
|
|
+ digest_hex = actual.removeprefix("sha256:")
|
|
|
+ attachment_dir = self._get_attachments_dir(trace_id) / message_id
|
|
|
+ attachment_dir.mkdir(parents=True, exist_ok=True)
|
|
|
+ target = attachment_dir / f"{digest_hex}.bin"
|
|
|
+ metadata = attachment_dir / f"{digest_hex}.json"
|
|
|
+ if target.exists():
|
|
|
+ if target.read_bytes() != content:
|
|
|
+ raise ValueError("attachment content conflict")
|
|
|
+ else:
|
|
|
+ temporary = attachment_dir / f".{digest_hex}.{uuid.uuid4().hex}.tmp"
|
|
|
+ try:
|
|
|
+ with temporary.open("xb") as output:
|
|
|
+ output.write(content)
|
|
|
+ output.flush()
|
|
|
+ os.fsync(output.fileno())
|
|
|
+ os.replace(temporary, target)
|
|
|
+ _fsync_directory(attachment_dir)
|
|
|
+ finally:
|
|
|
+ temporary.unlink(missing_ok=True)
|
|
|
+
|
|
|
+ if metadata.exists():
|
|
|
+ try:
|
|
|
+ stored_reference = AttachmentRef.from_dict(
|
|
|
+ json.loads(metadata.read_text(encoding="utf-8"))
|
|
|
+ )
|
|
|
+ except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
|
+ raise ValueError("attachment metadata is invalid") from exc
|
|
|
+ if stored_reference != reference:
|
|
|
+ raise ValueError("attachment reference conflict")
|
|
|
+ else:
|
|
|
+ _write_atomic_json(metadata, reference.to_dict())
|
|
|
+ _fsync_directory(attachment_dir)
|
|
|
+
|
|
|
+ return reference
|
|
|
+
|
|
|
+ async def read_message_attachment(self, ref: AttachmentRef) -> bytes:
|
|
|
+ """Read and verify one content-addressed attachment."""
|
|
|
+ if not isinstance(ref, AttachmentRef):
|
|
|
+ raise TypeError("ref must be an AttachmentRef")
|
|
|
+ _validate_path_segment(ref.trace_id, "trace_id")
|
|
|
+ _validate_path_segment(ref.message_id, "message_id")
|
|
|
+ digest_hex = ref.sha256.removeprefix("sha256:")
|
|
|
+ target = (
|
|
|
+ self._get_attachments_dir(ref.trace_id)
|
|
|
+ / ref.message_id
|
|
|
+ / f"{digest_hex}.bin"
|
|
|
+ )
|
|
|
+ metadata = target.with_suffix(".json")
|
|
|
+ if not target.is_file():
|
|
|
+ raise FileNotFoundError("trace attachment not found")
|
|
|
+ if not metadata.is_file():
|
|
|
+ raise ValueError("trace attachment metadata not found")
|
|
|
+ try:
|
|
|
+ stored_reference = AttachmentRef.from_dict(
|
|
|
+ json.loads(metadata.read_text(encoding="utf-8"))
|
|
|
+ )
|
|
|
+ except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
|
+ raise ValueError("trace attachment metadata is invalid") from exc
|
|
|
+ if stored_reference != ref:
|
|
|
+ raise ValueError("trace attachment reference does not match metadata")
|
|
|
+ content = target.read_bytes()
|
|
|
+ actual = f"sha256:{hashlib.sha256(content).hexdigest()}"
|
|
|
+ if actual != ref.sha256 or len(content) != ref.size_bytes:
|
|
|
+ raise ValueError("trace attachment integrity check failed")
|
|
|
+ return content
|
|
|
+
|
|
|
async def get_trace(self, trace_id: str) -> Optional[Trace]:
|
|
|
"""获取 Trace"""
|
|
|
meta_file = self._get_meta_file(trace_id)
|
|
|
@@ -117,7 +226,9 @@ class FileSystemTraceStore:
|
|
|
|
|
|
# 写回文件
|
|
|
meta_file = self._get_meta_file(trace_id)
|
|
|
- meta_file.write_text(json.dumps(trace.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
+ meta_file.write_text(
|
|
|
+ json.dumps(trace.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8"
|
|
|
+ )
|
|
|
|
|
|
async def list_traces(
|
|
|
self,
|
|
|
@@ -132,7 +243,7 @@ class FileSystemTraceStore:
|
|
|
attempt_id: Optional[str] = None,
|
|
|
validation_id: Optional[str] = None,
|
|
|
operation_id: Optional[str] = None,
|
|
|
- limit: int = 50
|
|
|
+ limit: int = 50,
|
|
|
) -> List[Trace]:
|
|
|
"""列出 Traces"""
|
|
|
traces = []
|
|
|
@@ -213,7 +324,9 @@ class FileSystemTraceStore:
|
|
|
async def update_goal_tree(self, trace_id: str, tree: GoalTree) -> None:
|
|
|
"""更新完整 GoalTree"""
|
|
|
goal_file = self._get_goal_file(trace_id)
|
|
|
- goal_file.write_text(json.dumps(tree.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
+ goal_file.write_text(
|
|
|
+ json.dumps(tree.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8"
|
|
|
+ )
|
|
|
|
|
|
async def add_goal(self, trace_id: str, goal: Goal) -> None:
|
|
|
"""添加 Goal 到 GoalTree"""
|
|
|
@@ -225,18 +338,21 @@ class FileSystemTraceStore:
|
|
|
await self.update_goal_tree(trace_id, tree)
|
|
|
|
|
|
# 推送 goal_added 事件
|
|
|
- event_data = {
|
|
|
- "goal": goal.to_dict(),
|
|
|
- "parent_id": goal.parent_id
|
|
|
- }
|
|
|
+ event_data = {"goal": goal.to_dict(), "parent_id": goal.parent_id}
|
|
|
await self.append_event(trace_id, "goal_added", event_data)
|
|
|
|
|
|
# 打印详细的 goal 信息
|
|
|
- desc_preview = goal.description[:80] + "..." if len(goal.description) > 80 else goal.description
|
|
|
+ desc_preview = (
|
|
|
+ goal.description[:80] + "..."
|
|
|
+ if len(goal.description) > 80
|
|
|
+ else goal.description
|
|
|
+ )
|
|
|
print(f"[Goal Added] ID={goal.id}, Parent={goal.parent_id or 'root'}")
|
|
|
print(f" 📝 {desc_preview}")
|
|
|
if goal.reason:
|
|
|
- reason_preview = goal.reason[:60] + "..." if len(goal.reason) > 60 else goal.reason
|
|
|
+ reason_preview = (
|
|
|
+ goal.reason[:60] + "..." if len(goal.reason) > 60 else goal.reason
|
|
|
+ )
|
|
|
print(f" 💡 {reason_preview}")
|
|
|
|
|
|
async def update_goal(
|
|
|
@@ -259,7 +375,9 @@ class FileSystemTraceStore:
|
|
|
for key, value in updates.items():
|
|
|
if hasattr(goal, key):
|
|
|
# 特殊处理 stats 字段(可能是 dict)
|
|
|
- if key in ["self_stats", "cumulative_stats"] and isinstance(value, dict):
|
|
|
+ if key in ["self_stats", "cumulative_stats"] and isinstance(
|
|
|
+ value, dict
|
|
|
+ ):
|
|
|
value = GoalStats.from_dict(value)
|
|
|
setattr(goal, key, value)
|
|
|
|
|
|
@@ -274,12 +392,14 @@ class FileSystemTraceStore:
|
|
|
cascade_completed = await self._check_cascade_completion(trace_id, goal)
|
|
|
affected_goals.extend(cascade_completed)
|
|
|
|
|
|
- await self.append_event(trace_id, "goal_updated", {
|
|
|
- "goal_id": goal_id,
|
|
|
- "updates": updates,
|
|
|
- "affected_goals": affected_goals
|
|
|
- })
|
|
|
- print(f"[DEBUG] Pushed goal_updated event: goal_id={goal_id}, updates={updates}, affected={len(affected_goals)}")
|
|
|
+ await self.append_event(
|
|
|
+ trace_id,
|
|
|
+ "goal_updated",
|
|
|
+ {"goal_id": goal_id, "updates": updates, "affected_goals": affected_goals},
|
|
|
+ )
|
|
|
+ print(
|
|
|
+ f"[DEBUG] Pushed goal_updated event: goal_id={goal_id}, updates={updates}, affected={len(affected_goals)}"
|
|
|
+ )
|
|
|
|
|
|
# Goal 完成时触发知识评估
|
|
|
if updates.get("status") in ["completed", "abandoned"]:
|
|
|
@@ -293,12 +413,12 @@ class FileSystemTraceStore:
|
|
|
trace.context["pending_knowledge_eval"] = True
|
|
|
trace.context["knowledge_eval_trigger"] = "goal_completion"
|
|
|
await self.update_trace(trace_id, context=trace.context)
|
|
|
- logger.info(f"[Knowledge Eval] Goal {goal_id} 完成,设置评估标志,待评估知识: {len(pending)} 条")
|
|
|
+ logger.info(
|
|
|
+ f"[Knowledge Eval] Goal {goal_id} 完成,设置评估标志,待评估知识: {len(pending)} 条"
|
|
|
+ )
|
|
|
|
|
|
async def _check_cascade_completion(
|
|
|
- self,
|
|
|
- trace_id: str,
|
|
|
- completed_goal: Goal
|
|
|
+ self, trace_id: str, completed_goal: Goal
|
|
|
) -> List[Dict[str, Any]]:
|
|
|
"""
|
|
|
检查级联完成:如果一个 Goal 的所有子 Goal 都完成,则自动完成父 Goal
|
|
|
@@ -328,8 +448,7 @@ class FileSystemTraceStore:
|
|
|
|
|
|
# 检查是否所有子 Goal 都已完成(排除 abandoned)
|
|
|
all_completed = all(
|
|
|
- child.status in ["completed", "abandoned"]
|
|
|
- for child in children
|
|
|
+ child.status in ["completed", "abandoned"] for child in children
|
|
|
)
|
|
|
|
|
|
if all_completed and parent.status != "completed":
|
|
|
@@ -342,15 +461,19 @@ class FileSystemTraceStore:
|
|
|
|
|
|
await self.update_goal_tree(trace_id, tree)
|
|
|
|
|
|
- affected.append({
|
|
|
- "goal_id": parent.id,
|
|
|
- "status": "completed",
|
|
|
- "summary": parent.summary,
|
|
|
- "cumulative_stats": parent.cumulative_stats.to_dict()
|
|
|
- })
|
|
|
+ affected.append(
|
|
|
+ {
|
|
|
+ "goal_id": parent.id,
|
|
|
+ "status": "completed",
|
|
|
+ "summary": parent.summary,
|
|
|
+ "cumulative_stats": parent.cumulative_stats.to_dict(),
|
|
|
+ }
|
|
|
+ )
|
|
|
|
|
|
# 递归检查祖父 Goal
|
|
|
- grandparent_affected = await self._check_cascade_completion(trace_id, parent)
|
|
|
+ grandparent_affected = await self._check_cascade_completion(
|
|
|
+ trace_id, parent
|
|
|
+ )
|
|
|
affected.extend(grandparent_affected)
|
|
|
|
|
|
return affected
|
|
|
@@ -368,7 +491,10 @@ class FileSystemTraceStore:
|
|
|
# 1. 写入 message 文件
|
|
|
messages_dir = self._get_messages_dir(trace_id)
|
|
|
message_file = messages_dir / f"{message.message_id}.json"
|
|
|
- message_file.write_text(json.dumps(message.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
+ message_file.write_text(
|
|
|
+ json.dumps(message.to_dict(), indent=2, ensure_ascii=False),
|
|
|
+ encoding="utf-8",
|
|
|
+ )
|
|
|
|
|
|
# 2. 更新 trace 统计
|
|
|
trace = await self.get_trace(trace_id)
|
|
|
@@ -392,7 +518,9 @@ class FileSystemTraceStore:
|
|
|
if message.tokens:
|
|
|
trace.total_tokens += message.tokens
|
|
|
elif message.prompt_tokens or message.completion_tokens:
|
|
|
- trace.total_tokens += (message.prompt_tokens or 0) + (message.completion_tokens or 0)
|
|
|
+ trace.total_tokens += (message.prompt_tokens or 0) + (
|
|
|
+ message.completion_tokens or 0
|
|
|
+ )
|
|
|
|
|
|
if message.cost:
|
|
|
trace.total_cost += message.cost
|
|
|
@@ -411,7 +539,7 @@ class FileSystemTraceStore:
|
|
|
total_cache_creation_tokens=trace.total_cache_creation_tokens,
|
|
|
total_cache_read_tokens=trace.total_cache_read_tokens,
|
|
|
total_cost=trace.total_cost,
|
|
|
- total_duration_ms=trace.total_duration_ms
|
|
|
+ total_duration_ms=trace.total_duration_ms,
|
|
|
)
|
|
|
|
|
|
# 3. 更新 Goal stats
|
|
|
@@ -419,10 +547,11 @@ class FileSystemTraceStore:
|
|
|
|
|
|
# 4. 追加 message_added 事件
|
|
|
affected_goals = await self._get_affected_goals(trace_id, message)
|
|
|
- event_id = await self.append_event(trace_id, "message_added", {
|
|
|
- "message": message.to_dict(),
|
|
|
- "affected_goals": affected_goals
|
|
|
- })
|
|
|
+ event_id = await self.append_event(
|
|
|
+ trace_id,
|
|
|
+ "message_added",
|
|
|
+ {"message": message.to_dict(), "affected_goals": affected_goals},
|
|
|
+ )
|
|
|
if event_id:
|
|
|
try:
|
|
|
from . import websocket as trace_ws
|
|
|
@@ -434,7 +563,11 @@ class FileSystemTraceStore:
|
|
|
affected_goals=affected_goals,
|
|
|
)
|
|
|
except Exception:
|
|
|
- logger.exception("Failed to broadcast message_added (trace_id=%s, event_id=%s)", trace_id, event_id)
|
|
|
+ logger.exception(
|
|
|
+ "Failed to broadcast message_added (trace_id=%s, event_id=%s)",
|
|
|
+ trace_id,
|
|
|
+ event_id,
|
|
|
+ )
|
|
|
|
|
|
return message.message_id
|
|
|
|
|
|
@@ -483,7 +616,9 @@ class FileSystemTraceStore:
|
|
|
# 保存更新后的 tree
|
|
|
await self.update_goal_tree(trace_id, tree)
|
|
|
|
|
|
- async def _get_affected_goals(self, trace_id: str, message: Message) -> List[Dict[str, Any]]:
|
|
|
+ async def _get_affected_goals(
|
|
|
+ self, trace_id: str, message: Message
|
|
|
+ ) -> List[Dict[str, Any]]:
|
|
|
"""获取受影响的 Goals(自身 + 所有祖先)"""
|
|
|
tree = await self.get_goal_tree(trace_id)
|
|
|
|
|
|
@@ -497,11 +632,13 @@ class FileSystemTraceStore:
|
|
|
affected = []
|
|
|
|
|
|
# 添加自身(包含 self_stats 和 cumulative_stats)
|
|
|
- affected.append({
|
|
|
- "goal_id": goal.id,
|
|
|
- "self_stats": goal.self_stats.to_dict(),
|
|
|
- "cumulative_stats": goal.cumulative_stats.to_dict()
|
|
|
- })
|
|
|
+ affected.append(
|
|
|
+ {
|
|
|
+ "goal_id": goal.id,
|
|
|
+ "self_stats": goal.self_stats.to_dict(),
|
|
|
+ "cumulative_stats": goal.cumulative_stats.to_dict(),
|
|
|
+ }
|
|
|
+ )
|
|
|
|
|
|
# 添加所有祖先(仅 cumulative_stats)
|
|
|
current_goal = goal
|
|
|
@@ -510,10 +647,12 @@ class FileSystemTraceStore:
|
|
|
if not parent:
|
|
|
break
|
|
|
|
|
|
- affected.append({
|
|
|
- "goal_id": parent.id,
|
|
|
- "cumulative_stats": parent.cumulative_stats.to_dict()
|
|
|
- })
|
|
|
+ affected.append(
|
|
|
+ {
|
|
|
+ "goal_id": parent.id,
|
|
|
+ "cumulative_stats": parent.cumulative_stats.to_dict(),
|
|
|
+ }
|
|
|
+ )
|
|
|
|
|
|
current_goal = parent
|
|
|
|
|
|
@@ -560,9 +699,7 @@ class FileSystemTraceStore:
|
|
|
return messages
|
|
|
|
|
|
async def get_main_path_messages(
|
|
|
- self,
|
|
|
- trace_id: str,
|
|
|
- head_sequence: int
|
|
|
+ self, trace_id: str, head_sequence: int
|
|
|
) -> List[Message]:
|
|
|
"""
|
|
|
获取从 head_sequence 沿 parent_sequence 链回溯到 root 的完整路径
|
|
|
@@ -592,11 +729,7 @@ class FileSystemTraceStore:
|
|
|
path.reverse()
|
|
|
return path
|
|
|
|
|
|
- async def get_messages_by_goal(
|
|
|
- self,
|
|
|
- trace_id: str,
|
|
|
- goal_id: str
|
|
|
- ) -> List[Message]:
|
|
|
+ async def get_messages_by_goal(self, trace_id: str, goal_id: str) -> List[Message]:
|
|
|
"""获取指定 Goal 关联的所有 Messages"""
|
|
|
all_messages = await self.get_trace_messages(trace_id)
|
|
|
return [m for m in all_messages if m.goal_id == goal_id]
|
|
|
@@ -615,9 +748,14 @@ class FileSystemTraceStore:
|
|
|
# 确定文件路径
|
|
|
messages_dir = self._get_messages_dir(message.trace_id)
|
|
|
message_file = messages_dir / f"{message_id}.json"
|
|
|
- message_file.write_text(json.dumps(message.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
+ message_file.write_text(
|
|
|
+ json.dumps(message.to_dict(), indent=2, ensure_ascii=False),
|
|
|
+ encoding="utf-8",
|
|
|
+ )
|
|
|
|
|
|
- async def abandon_messages_after(self, trace_id: str, cutoff_sequence: int) -> List[str]:
|
|
|
+ async def abandon_messages_after(
|
|
|
+ self, trace_id: str, cutoff_sequence: int
|
|
|
+ ) -> List[str]:
|
|
|
"""
|
|
|
将 sequence > cutoff_sequence 的 active messages 标记为 abandoned。
|
|
|
返回被 abandon 的 message_id 列表。
|
|
|
@@ -631,10 +769,12 @@ class FileSystemTraceStore:
|
|
|
msg.status = "abandoned"
|
|
|
msg.abandoned_at = now
|
|
|
# 直接写回文件
|
|
|
- message_file = self._get_messages_dir(trace_id) / f"{msg.message_id}.json"
|
|
|
+ message_file = (
|
|
|
+ self._get_messages_dir(trace_id) / f"{msg.message_id}.json"
|
|
|
+ )
|
|
|
message_file.write_text(
|
|
|
json.dumps(msg.to_dict(), indent=2, ensure_ascii=False),
|
|
|
- encoding="utf-8"
|
|
|
+ encoding="utf-8",
|
|
|
)
|
|
|
abandoned_ids.append(msg.message_id)
|
|
|
|
|
|
@@ -710,15 +850,17 @@ class FileSystemTraceStore:
|
|
|
model_entry["cache_read_tokens"] += cache_read_tokens
|
|
|
model_entry["call_count"] += 1
|
|
|
else:
|
|
|
- data["models"].append({
|
|
|
- "model": model,
|
|
|
- "source": source,
|
|
|
- "prompt_tokens": prompt_tokens,
|
|
|
- "completion_tokens": completion_tokens,
|
|
|
- "total_tokens": total_tokens,
|
|
|
- "cache_read_tokens": cache_read_tokens,
|
|
|
- "call_count": 1,
|
|
|
- })
|
|
|
+ data["models"].append(
|
|
|
+ {
|
|
|
+ "model": model,
|
|
|
+ "source": source,
|
|
|
+ "prompt_tokens": prompt_tokens,
|
|
|
+ "completion_tokens": completion_tokens,
|
|
|
+ "total_tokens": total_tokens,
|
|
|
+ "cache_read_tokens": cache_read_tokens,
|
|
|
+ "call_count": 1,
|
|
|
+ }
|
|
|
+ )
|
|
|
data["summary"]["total_models"] = len(data["models"])
|
|
|
|
|
|
# 添加到timeline
|
|
|
@@ -736,14 +878,14 @@ class FileSystemTraceStore:
|
|
|
data["timeline"].append(timeline_entry)
|
|
|
|
|
|
# 写回文件
|
|
|
- usage_file.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
+ usage_file.write_text(
|
|
|
+ json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
|
+ )
|
|
|
|
|
|
# ===== 事件流操作(用于 WebSocket 断线续传)=====
|
|
|
|
|
|
async def get_events(
|
|
|
- self,
|
|
|
- trace_id: str,
|
|
|
- since_event_id: int = 0
|
|
|
+ self, trace_id: str, since_event_id: int = 0
|
|
|
) -> List[Dict[str, Any]]:
|
|
|
"""获取事件流"""
|
|
|
events_file = self._get_events_file(trace_id)
|
|
|
@@ -751,7 +893,7 @@ class FileSystemTraceStore:
|
|
|
return []
|
|
|
|
|
|
events = []
|
|
|
- with events_file.open('r', encoding='utf-8') as f:
|
|
|
+ with events_file.open("r", encoding="utf-8") as f:
|
|
|
for line in f:
|
|
|
try:
|
|
|
event = json.loads(line.strip())
|
|
|
@@ -763,10 +905,7 @@ class FileSystemTraceStore:
|
|
|
return events
|
|
|
|
|
|
async def append_event(
|
|
|
- self,
|
|
|
- trace_id: str,
|
|
|
- event_type: str,
|
|
|
- payload: Dict[str, Any]
|
|
|
+ self, trace_id: str, event_type: str, payload: Dict[str, Any]
|
|
|
) -> int:
|
|
|
"""追加事件,返回 event_id"""
|
|
|
# 获取 trace 并递增 event_id
|
|
|
@@ -785,13 +924,13 @@ class FileSystemTraceStore:
|
|
|
"event_id": event_id,
|
|
|
"event": event_type,
|
|
|
"ts": datetime.now().isoformat(),
|
|
|
- **payload
|
|
|
+ **payload,
|
|
|
}
|
|
|
|
|
|
# 追加到 events.jsonl
|
|
|
events_file = self._get_events_file(trace_id)
|
|
|
- with events_file.open('a', encoding='utf-8') as f:
|
|
|
- f.write(json.dumps(event, ensure_ascii=False) + '\n')
|
|
|
+ with events_file.open("a", encoding="utf-8") as f:
|
|
|
+ f.write(json.dumps(event, ensure_ascii=False) + "\n")
|
|
|
|
|
|
return event_id
|
|
|
|
|
|
@@ -872,7 +1011,9 @@ class FileSystemTraceStore:
|
|
|
event["timestamp"] = datetime.now().isoformat()
|
|
|
log["events"].append(event)
|
|
|
log_file = self._get_cognition_log_file(trace_id)
|
|
|
- log_file.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
+ log_file.write_text(
|
|
|
+ json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
|
+ )
|
|
|
|
|
|
async def append_knowledge_entry(
|
|
|
self,
|
|
|
@@ -881,7 +1022,7 @@ class FileSystemTraceStore:
|
|
|
goal_id: str,
|
|
|
injected_at_sequence: int,
|
|
|
task: str,
|
|
|
- content: str
|
|
|
+ content: str,
|
|
|
) -> None:
|
|
|
"""兼容旧接口:追加知识注入记录(转换为 query 事件)"""
|
|
|
await self.append_cognition_event(
|
|
|
@@ -893,8 +1034,10 @@ class FileSystemTraceStore:
|
|
|
"query": task,
|
|
|
"response": "",
|
|
|
"source_ids": [knowledge_id],
|
|
|
- "sources": [{"id": knowledge_id, "task": task, "content": content[:500]}],
|
|
|
- }
|
|
|
+ "sources": [
|
|
|
+ {"id": knowledge_id, "task": task, "content": content[:500]}
|
|
|
+ ],
|
|
|
+ },
|
|
|
)
|
|
|
|
|
|
async def update_knowledge_evaluation(
|
|
|
@@ -902,7 +1045,7 @@ class FileSystemTraceStore:
|
|
|
trace_id: str,
|
|
|
knowledge_id: str,
|
|
|
eval_result: Dict[str, Any],
|
|
|
- trigger_event: str
|
|
|
+ trigger_event: str,
|
|
|
) -> None:
|
|
|
"""更新知识评估结果(兼容旧格式 + 新 cognition_log 格式)
|
|
|
|
|
|
@@ -915,23 +1058,30 @@ class FileSystemTraceStore:
|
|
|
# 旧格式兼容:直接更新 entries 中的 eval_result 字段
|
|
|
if "entries" in log:
|
|
|
matching = [
|
|
|
- (i, e) for i, e in enumerate(log["entries"])
|
|
|
- if e.get("knowledge_id") == knowledge_id and e.get("eval_result") is None
|
|
|
+ (i, e)
|
|
|
+ for i, e in enumerate(log["entries"])
|
|
|
+ if e.get("knowledge_id") == knowledge_id
|
|
|
+ and e.get("eval_result") is None
|
|
|
]
|
|
|
if matching:
|
|
|
- matching.sort(key=lambda x: x[1].get("injected_at_sequence", 0), reverse=True)
|
|
|
+ matching.sort(
|
|
|
+ key=lambda x: x[1].get("injected_at_sequence", 0), reverse=True
|
|
|
+ )
|
|
|
_, entry = matching[0]
|
|
|
entry["eval_result"] = eval_result
|
|
|
entry["evaluated_at"] = datetime.now().isoformat()
|
|
|
entry["evaluated_at_trigger"] = trigger_event
|
|
|
log_file = self._get_knowledge_log_file(trace_id)
|
|
|
- log_file.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
+ log_file.write_text(
|
|
|
+ json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
|
+ )
|
|
|
return
|
|
|
|
|
|
# 新格式:追加 evaluation 事件
|
|
|
# 找到包含该 knowledge_id 的最近 query 事件
|
|
|
query_events = [
|
|
|
- e for e in events
|
|
|
+ e
|
|
|
+ for e in events
|
|
|
if e.get("type") == "query" and knowledge_id in e.get("source_ids", [])
|
|
|
]
|
|
|
query_sequence = query_events[-1]["sequence"] if query_events else None
|
|
|
@@ -944,12 +1094,18 @@ class FileSystemTraceStore:
|
|
|
"query_sequence": query_sequence,
|
|
|
"trigger": trigger_event,
|
|
|
"assessments": [
|
|
|
- {"source_id": knowledge_id, "status": eval_result.get("eval_status", ""), "reason": eval_result.get("reason", "")}
|
|
|
+ {
|
|
|
+ "source_id": knowledge_id,
|
|
|
+ "status": eval_result.get("eval_status", ""),
|
|
|
+ "reason": eval_result.get("reason", ""),
|
|
|
+ }
|
|
|
],
|
|
|
- }
|
|
|
+ },
|
|
|
)
|
|
|
|
|
|
- async def get_pending_knowledge_entries(self, trace_id: str) -> List[Dict[str, Any]]:
|
|
|
+ async def get_pending_knowledge_entries(
|
|
|
+ self, trace_id: str
|
|
|
+ ) -> List[Dict[str, Any]]:
|
|
|
"""获取所有待评估的知识条目(兼容旧格式 + 新格式)"""
|
|
|
log = await self.get_cognition_log(trace_id)
|
|
|
|
|
|
@@ -970,21 +1126,20 @@ class FileSystemTraceStore:
|
|
|
if qe.get("sequence") not in evaluated_sequences:
|
|
|
# 转为旧格式兼容(runner 中的评估逻辑期望此格式)
|
|
|
for source in qe.get("sources", []):
|
|
|
- pending.append({
|
|
|
- "knowledge_id": source.get("id", ""),
|
|
|
- "goal_id": qe.get("goal_id", ""),
|
|
|
- "injected_at_sequence": qe.get("sequence", 0),
|
|
|
- "task": source.get("task", ""),
|
|
|
- "content": source.get("content", ""),
|
|
|
- "query_sequence": qe.get("sequence"),
|
|
|
- })
|
|
|
+ pending.append(
|
|
|
+ {
|
|
|
+ "knowledge_id": source.get("id", ""),
|
|
|
+ "goal_id": qe.get("goal_id", ""),
|
|
|
+ "injected_at_sequence": qe.get("sequence", 0),
|
|
|
+ "task": source.get("task", ""),
|
|
|
+ "content": source.get("content", ""),
|
|
|
+ "query_sequence": qe.get("sequence"),
|
|
|
+ }
|
|
|
+ )
|
|
|
return pending
|
|
|
|
|
|
async def update_user_feedback(
|
|
|
- self,
|
|
|
- trace_id: str,
|
|
|
- knowledge_id: str,
|
|
|
- user_feedback: Dict[str, Any]
|
|
|
+ self, trace_id: str, knowledge_id: str, user_feedback: Dict[str, Any]
|
|
|
) -> None:
|
|
|
"""记录用户对知识的反馈(confirm/override)"""
|
|
|
log = await self.get_cognition_log(trace_id)
|
|
|
@@ -992,15 +1147,20 @@ class FileSystemTraceStore:
|
|
|
# 旧格式
|
|
|
if "entries" in log:
|
|
|
matching = [
|
|
|
- (i, e) for i, e in enumerate(log["entries"])
|
|
|
+ (i, e)
|
|
|
+ for i, e in enumerate(log["entries"])
|
|
|
if e.get("knowledge_id") == knowledge_id
|
|
|
]
|
|
|
if matching:
|
|
|
- matching.sort(key=lambda x: x[1].get("injected_at_sequence", 0), reverse=True)
|
|
|
+ matching.sort(
|
|
|
+ key=lambda x: x[1].get("injected_at_sequence", 0), reverse=True
|
|
|
+ )
|
|
|
_, entry = matching[0]
|
|
|
entry["user_feedback"] = user_feedback
|
|
|
log_file = self._get_knowledge_log_file(trace_id)
|
|
|
- log_file.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
+ log_file.write_text(
|
|
|
+ json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
|
+ )
|
|
|
return
|
|
|
|
|
|
# 新格式:追加 user_feedback 事件(或直接记录在 evaluation 上)
|
|
|
@@ -1010,5 +1170,37 @@ class FileSystemTraceStore:
|
|
|
"type": "user_feedback",
|
|
|
"knowledge_id": knowledge_id,
|
|
|
"feedback": user_feedback,
|
|
|
- }
|
|
|
+ },
|
|
|
)
|
|
|
+
|
|
|
+
|
|
|
+def _validate_path_segment(value: str, field_name: str) -> None:
|
|
|
+ if (
|
|
|
+ not isinstance(value, str)
|
|
|
+ or not value
|
|
|
+ or value in {".", ".."}
|
|
|
+ or Path(value).name != value
|
|
|
+ or "/" in value
|
|
|
+ or "\\" in value
|
|
|
+ ):
|
|
|
+ raise ValueError(f"{field_name} must be a safe path segment")
|
|
|
+
|
|
|
+
|
|
|
+def _fsync_directory(path: Path) -> None:
|
|
|
+ descriptor = os.open(path, os.O_RDONLY)
|
|
|
+ try:
|
|
|
+ os.fsync(descriptor)
|
|
|
+ finally:
|
|
|
+ os.close(descriptor)
|
|
|
+
|
|
|
+
|
|
|
+def _write_atomic_json(path: Path, value: Dict[str, Any]) -> None:
|
|
|
+ temporary = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp"
|
|
|
+ try:
|
|
|
+ with temporary.open("x", encoding="utf-8") as output:
|
|
|
+ json.dump(value, output, ensure_ascii=False, sort_keys=True)
|
|
|
+ output.flush()
|
|
|
+ os.fsync(output.fileno())
|
|
|
+ os.replace(temporary, path)
|
|
|
+ finally:
|
|
|
+ temporary.unlink(missing_ok=True)
|