store.py 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246
  1. """把本项目按 Run 落盘的业务文件投影成上游查看器使用的只读视图。"""
  2. from __future__ import annotations
  3. import hashlib
  4. import json
  5. import mimetypes
  6. import re
  7. from dataclasses import dataclass
  8. from datetime import datetime, timezone
  9. from pathlib import Path
  10. from typing import Any, Iterable
  11. PROJECT_ROOT = Path(__file__).resolve().parents[2]
  12. DEFAULT_RUN_ROOT = PROJECT_ROOT / "demo_output"
  13. def _load_json(path: Path, default: Any = None) -> Any:
  14. try:
  15. return json.loads(path.read_text(encoding="utf-8"))
  16. except (FileNotFoundError, json.JSONDecodeError, OSError):
  17. return default
  18. def _stable_id(value: str) -> int:
  19. """生成低于 JavaScript MAX_SAFE_INTEGER 的稳定数字 ID。"""
  20. return int(hashlib.sha256(value.encode("utf-8")).hexdigest()[:13], 16)
  21. def _iso(timestamp: float | None) -> str | None:
  22. if timestamp is None:
  23. return None
  24. return datetime.fromtimestamp(timestamp, tz=timezone.utc).isoformat()
  25. def _clip(value: Any, limit: int = 240) -> str:
  26. if value is None:
  27. return ""
  28. if not isinstance(value, str):
  29. value = json.dumps(value, ensure_ascii=False, default=str)
  30. return value if len(value) <= limit else value[: limit - 1] + "…"
  31. def _viewer_status(value: Any) -> str:
  32. normalized = str(value or "").upper()
  33. if normalized in {"COMPLETED", "PASS", "SUCCESS", "SUCCEEDED"} or (
  34. normalized.endswith("_COMPLETED")
  35. ):
  36. return "success"
  37. if normalized in {"FAILED", "FAIL", "ERROR"}:
  38. return "failed"
  39. return "running"
  40. def _files(root: Path, pattern: str) -> list[Path]:
  41. return sorted(root.glob(pattern))
  42. def _mtime_bounds(root: Path) -> tuple[float | None, float | None]:
  43. values = [
  44. path.stat().st_mtime
  45. for path in root.rglob("*")
  46. if path.is_file() and path.name != ".run.lock"
  47. ]
  48. return (min(values), max(values)) if values else (None, None)
  49. def _input_block(
  50. key: str,
  51. title: str,
  52. value: Any,
  53. *,
  54. source: str = "",
  55. images: Iterable[str] = (),
  56. ) -> dict[str, Any]:
  57. return {
  58. "key": key,
  59. "name": key,
  60. "title": title,
  61. "hint": source,
  62. "source": source,
  63. "filled": value not in (None, "", [], {}),
  64. "text": _clip(value, 1600),
  65. "images": list(images),
  66. }
  67. def _node_card(
  68. index: int,
  69. *,
  70. node_type: str,
  71. label: str,
  72. summary: Any,
  73. ok: bool | None = None,
  74. detail: Any = None,
  75. tool_call_id: str = "",
  76. ) -> dict[str, Any]:
  77. return {
  78. "seq": index,
  79. "kind": "node",
  80. "fanout_group": "",
  81. "card": {
  82. "i": index,
  83. "type": node_type,
  84. "label": label,
  85. "summary": _clip(summary, 300),
  86. "ok": ok,
  87. "images": [],
  88. "branch": "",
  89. "node_id": None,
  90. "tool_call_id": tool_call_id,
  91. "io": None,
  92. "calls": [],
  93. "reasoning": "",
  94. "tokens": "",
  95. "detail": detail if detail is not None else summary,
  96. },
  97. }
  98. @dataclass(frozen=True)
  99. class RunRef:
  100. numeric_id: int
  101. key: str
  102. path: Path
  103. kind: str
  104. class FileRunStore:
  105. """扫描 ``demo_output``,不读取或修改数据库、Checkpoint。"""
  106. def __init__(self, run_root: Path | None = None):
  107. self.run_root = (run_root or DEFAULT_RUN_ROOT).resolve()
  108. def discover(self) -> list[RunRef]:
  109. if not self.run_root.is_dir():
  110. return []
  111. refs: list[RunRef] = []
  112. for path in sorted(self.run_root.iterdir()):
  113. if not path.is_dir():
  114. continue
  115. kind = self._kind(path)
  116. if kind is None:
  117. continue
  118. refs.append(
  119. RunRef(
  120. numeric_id=_stable_id(path.name),
  121. key=path.name,
  122. path=path.resolve(),
  123. kind=kind,
  124. )
  125. )
  126. return sorted(
  127. refs,
  128. key=lambda item: _mtime_bounds(item.path)[1] or 0,
  129. reverse=True,
  130. )
  131. def get(self, numeric_id: int) -> RunRef:
  132. for ref in self.discover():
  133. if ref.numeric_id == numeric_id:
  134. return ref
  135. raise KeyError(numeric_id)
  136. @staticmethod
  137. def _kind(path: Path) -> str | None:
  138. if (path / "run_summary.json").is_file() or (
  139. path / "global_data_stage_delivery.json"
  140. ).is_file():
  141. return "global_data"
  142. if _files(path, "production_plans/production_dag.v*.json"):
  143. return "segment"
  144. return None
  145. def brief(self, ref: RunRef) -> dict[str, Any]:
  146. start, end = _mtime_bounds(ref.path)
  147. if ref.kind == "global_data":
  148. summary = _load_json(ref.path / "run_summary.json", {}) or {}
  149. stage_delivery = _load_json(
  150. ref.path / "global_data_stage_delivery.json", {}
  151. ) or {}
  152. metrics = _load_json(ref.path / "run_metrics.json", {}) or {}
  153. totals = metrics.get("model_totals") or {}
  154. invocation = metrics.get("run_invocations") or {}
  155. duration_ms = invocation.get("total_duration_ms")
  156. plan = self._latest_plan(ref.path)
  157. raw_status = (
  158. summary.get("status")
  159. or stage_delivery.get("status")
  160. or "RUNNING"
  161. )
  162. return {
  163. "id": ref.numeric_id,
  164. "run_key": ref.key,
  165. "run_kind": ref.kind,
  166. "agent_name": "global_data",
  167. "model_name": self._model_names(metrics),
  168. "objective": (plan or {}).get("goal") or ref.key,
  169. "status": _viewer_status(raw_status),
  170. "raw_status": raw_status,
  171. "parent_run_id": None,
  172. "start_time": _iso(start),
  173. "end_time": _iso(end),
  174. "duration_sec": (
  175. round(float(duration_ms) / 1000, 1)
  176. if isinstance(duration_ms, (int, float))
  177. else None
  178. ),
  179. "step_count": len(summary.get("event_log") or []),
  180. "replan_count": summary.get("replan_count") or 0,
  181. "input_tokens": totals.get("input_tokens") or 0,
  182. "output_tokens": totals.get("output_tokens") or 0,
  183. "cost_usd": totals.get("reported_cost_usd") or 0,
  184. "cost_status": (
  185. "reported"
  186. if totals.get("reported_cost_usd") is not None
  187. else "unavailable"
  188. ),
  189. }
  190. plan = self._latest_production_plan(ref.path) or {}
  191. reports = _files(
  192. ref.path, "segment_validation_results/segment.*.v*.json"
  193. )
  194. summaries = _files(ref.path, "segment_summaries/segment.*.v*.json")
  195. metrics = _load_json(ref.path / "run_metrics.json", {}) or {}
  196. totals = metrics.get("model_totals") or {}
  197. raw_status = "RUNNING"
  198. if reports:
  199. verdicts = [
  200. (_load_json(path, {}) or {}).get("verdict") for path in reports
  201. ]
  202. raw_status = "COMPLETED" if verdicts and all(
  203. value == "PASS" for value in verdicts
  204. ) else "FAILED"
  205. elif summaries:
  206. raw_status = (_load_json(summaries[-1], {}) or {}).get(
  207. "status", "RUNNING"
  208. )
  209. return {
  210. "id": ref.numeric_id,
  211. "run_key": ref.key,
  212. "run_kind": ref.kind,
  213. "agent_name": "segment_production",
  214. "model_name": self._model_names(metrics),
  215. "objective": plan.get("summary") or f"Production Segment · {ref.key}",
  216. "status": _viewer_status(raw_status),
  217. "raw_status": raw_status,
  218. "parent_run_id": None,
  219. "start_time": _iso(start),
  220. "end_time": _iso(end),
  221. "duration_sec": (
  222. round(end - start, 1)
  223. if start is not None and end is not None
  224. else None
  225. ),
  226. "step_count": len(_files(ref.path, "tool_operations/*.json")),
  227. "replan_count": 0,
  228. "input_tokens": totals.get("input_tokens") or 0,
  229. "output_tokens": totals.get("output_tokens") or 0,
  230. "cost_usd": totals.get("reported_cost_usd") or 0,
  231. "cost_status": (
  232. "reported"
  233. if totals.get("reported_cost_usd") is not None
  234. else "unavailable"
  235. ),
  236. }
  237. @staticmethod
  238. def _model_names(metrics: dict[str, Any]) -> str:
  239. names: list[str] = []
  240. for item in (metrics.get("models") or {}).values():
  241. for name in item.get("models") or []:
  242. if name and name not in names:
  243. names.append(str(name))
  244. return " · ".join(names)
  245. @staticmethod
  246. def _latest_plan(run_dir: Path) -> dict[str, Any] | None:
  247. plans = _files(run_dir, "plans/global_data_dag.v*.json")
  248. return _load_json(plans[-1], {}) if plans else None
  249. @staticmethod
  250. def _latest_production_plan(
  251. run_dir: Path,
  252. ) -> dict[str, Any] | None:
  253. plans = _files(
  254. run_dir,
  255. "production_plans/production_dag.v*.json",
  256. )
  257. if not plans:
  258. return None
  259. latest = max(
  260. plans,
  261. key=lambda path: int(
  262. re.search(r"\.v(\d+)\.json$", path.name).group(1)
  263. ),
  264. )
  265. return _load_json(latest, {})
  266. def detail(self, ref: RunRef) -> dict[str, Any]:
  267. result = self.brief(ref)
  268. if ref.kind == "global_data":
  269. summary = _load_json(ref.path / "run_summary.json", {}) or {}
  270. delivery = _load_json(
  271. ref.path / "global_data_stage_delivery.json", {}
  272. ) or {}
  273. result.update(
  274. {
  275. "final_output": delivery.get("summary"),
  276. "error_message": summary.get("error"),
  277. "input_payload": {
  278. "input_path": summary.get("input_path"),
  279. "input_sha256": summary.get("input_sha256"),
  280. "protocol_version": summary.get("protocol_version"),
  281. },
  282. "overview": self._global_overview(ref, summary),
  283. "steps": self._global_steps(summary),
  284. }
  285. )
  286. else:
  287. plan = self._latest_production_plan(ref.path) or {}
  288. result.update(
  289. {
  290. "final_output": self._segment_final_output(ref),
  291. "error_message": None,
  292. "input_payload": plan,
  293. "overview": self._segment_overview(ref),
  294. "steps": self._segment_steps(ref),
  295. }
  296. )
  297. return result
  298. def _global_overview(
  299. self, ref: RunRef, summary: dict[str, Any]
  300. ) -> dict[str, int]:
  301. plans = _files(ref.path, "plans/global_data_dag.v*.json")
  302. latest = _load_json(plans[-1], {}) if plans else {}
  303. tasks = (latest or {}).get("tasks") or []
  304. version = int((latest or {}).get("plan_version") or len(plans) or 1)
  305. reports = [
  306. _load_json(
  307. ref.path
  308. / "validation_results"
  309. / f"{task.get('task_id')}.v{version}.json",
  310. {},
  311. )
  312. or {}
  313. for task in tasks
  314. ]
  315. result_files = _files(ref.path, "executor_results/Task*.v*.json")
  316. distinct_tasks = {
  317. re.sub(r"\.v\d+$", "", path.stem) for path in result_files
  318. }
  319. return {
  320. "plan_rounds": len(plans),
  321. "execute_rounds": int(
  322. summary.get("executor_calls") or len(result_files)
  323. ),
  324. "task_count": len(tasks),
  325. "passed": sum(report.get("verdict") == "PASS" for report in reports),
  326. "failed": sum(report.get("verdict") == "FAIL" for report in reports),
  327. "retries": max(len(result_files) - len(distinct_tasks), 0),
  328. }
  329. def _segment_overview(self, ref: RunRef) -> dict[str, int]:
  330. plan = self._latest_production_plan(ref.path) or {}
  331. packages = _files(ref.path, "segment_packages/*.json")
  332. latest_reports: dict[str, dict[str, Any]] = {}
  333. for path in _files(
  334. ref.path, "segment_validation_results/segment.*.v*.json"
  335. ):
  336. parts = path.stem.split(".")
  337. segment_id = parts[1] if len(parts) > 2 else path.stem
  338. latest_reports[segment_id] = _load_json(path, {}) or {}
  339. reports = list(latest_reports.values())
  340. return {
  341. "plan_rounds": len(
  342. _files(
  343. ref.path,
  344. "production_plans/production_dag.v*.json",
  345. )
  346. ),
  347. "execute_rounds": len(_files(ref.path, "tool_operations/*.json")),
  348. "task_count": len(plan.get("segments") or packages),
  349. "passed": sum(report.get("verdict") == "PASS" for report in reports),
  350. "failed": sum(report.get("verdict") == "FAIL" for report in reports),
  351. "retries": 0,
  352. }
  353. @staticmethod
  354. def _global_steps(summary: dict[str, Any]) -> list[dict[str, Any]]:
  355. steps = []
  356. for index, message in enumerate(summary.get("event_log") or [], start=1):
  357. lower = message.lower()
  358. if "planner" in lower:
  359. node_type = "plan"
  360. elif "executor" in lower:
  361. node_type = "execute"
  362. elif "validator" in lower or "验收" in message:
  363. node_type = "validate"
  364. elif "预处理" in message:
  365. node_type = "preprocess"
  366. elif "完成" in message:
  367. node_type = "finalize"
  368. else:
  369. node_type = "route"
  370. steps.append(
  371. {
  372. "step_id": index,
  373. "step_index": index,
  374. "node_type": node_type,
  375. "content": message,
  376. "delta": {"event": message},
  377. "input_tokens": 0,
  378. "output_tokens": 0,
  379. "created_at": None,
  380. }
  381. )
  382. return steps
  383. def _segment_steps(self, ref: RunRef) -> list[dict[str, Any]]:
  384. steps: list[dict[str, Any]] = []
  385. package_paths = _files(ref.path, "segment_packages/*.json")
  386. if package_paths:
  387. package = _load_json(package_paths[0], {}) or {}
  388. steps.append(
  389. {
  390. "step_id": 1,
  391. "step_index": 1,
  392. "node_type": "prepare_segment",
  393. "content": f"物化 {package.get('segment_id', 'Segment')} Package",
  394. "delta": package,
  395. "input_tokens": 0,
  396. "output_tokens": 0,
  397. "created_at": _iso(package_paths[0].stat().st_mtime),
  398. }
  399. )
  400. for index, path in enumerate(
  401. _files(ref.path, "tool_operations/*.json"), start=2
  402. ):
  403. operation = _load_json(path, {}) or {}
  404. steps.append(
  405. {
  406. "step_id": index,
  407. "step_index": index,
  408. "node_type": "execute",
  409. "content": (
  410. f"{operation.get('tool_name', 'tool')} · "
  411. f"{operation.get('status', 'UNKNOWN')}"
  412. ),
  413. "delta": operation,
  414. "input_tokens": 0,
  415. "output_tokens": 0,
  416. "created_at": _iso(path.stat().st_mtime),
  417. }
  418. )
  419. return steps
  420. @staticmethod
  421. def _segment_final_output(ref: RunRef) -> Any:
  422. reports = _files(
  423. ref.path, "segment_validation_results/segment.*.v*.json"
  424. )
  425. if reports:
  426. return _load_json(reports[-1], {})
  427. packages = _files(ref.path, "segment_packages/*.json")
  428. if packages:
  429. package = _load_json(packages[-1], {}) or {}
  430. return {
  431. "status": "RUNNING",
  432. "segment_id": package.get("segment_id"),
  433. "message": "Segment 已物化,尚无终态 ValidationReport",
  434. }
  435. return None
  436. def snapshots(self, ref: RunRef) -> dict[str, Any]:
  437. if ref.kind == "global_data":
  438. snapshots = self._global_snapshots(ref)
  439. else:
  440. snapshots = self._segment_snapshots(ref)
  441. return {
  442. "run_id": ref.numeric_id,
  443. "run_key": ref.key,
  444. "objective": self.brief(ref)["objective"],
  445. "status": self.brief(ref)["status"],
  446. "snapshots": snapshots,
  447. }
  448. def _global_snapshots(self, ref: RunRef) -> list[dict[str, Any]]:
  449. paths = _files(ref.path, "plans/global_data_dag.v*.json")
  450. output: list[dict[str, Any]] = []
  451. previous_ids: set[str] = set()
  452. for index, path in enumerate(paths):
  453. plan = _load_json(path, {}) or {}
  454. version = plan.get("plan_version") or index + 1
  455. tasks = plan.get("tasks") or []
  456. current_ids = {str(item.get("task_id")) for item in tasks}
  457. added = sorted(current_ids - previous_ids)
  458. removed = sorted(previous_ids - current_ids)
  459. nodes = [
  460. self._snapshot_task(ref, task, version) for task in tasks
  461. ]
  462. output.append(
  463. {
  464. "loop_index": index,
  465. "kind": "initial" if index == 0 else "replan",
  466. "title": f"Plan v{version}",
  467. "step_id": index + 1,
  468. "step_index": index + 1,
  469. "is_final": index == len(paths) - 1,
  470. "created_at": _iso(path.stat().st_mtime),
  471. "reasoning": plan.get("revision_summary") or plan.get("goal"),
  472. "tree_changed": bool(added or removed),
  473. "diff": {
  474. "added": added,
  475. "removed": removed,
  476. "modified": [],
  477. },
  478. "lessons_added": [],
  479. "lessons_cumulative": [],
  480. "tree": {"nodes": nodes},
  481. "node_change": {
  482. **{item: "added" for item in added},
  483. **{item: "removed" for item in removed},
  484. },
  485. "round_execution": None,
  486. }
  487. )
  488. previous_ids = current_ids
  489. return output
  490. def _snapshot_task(
  491. self, ref: RunRef, task: dict[str, Any], version: int
  492. ) -> dict[str, Any]:
  493. task_id = str(task.get("task_id") or "")
  494. delivery = _load_json(
  495. ref.path / "executor_results" / f"{task_id}.v{version}.json", {}
  496. ) or {}
  497. report = _load_json(
  498. ref.path / "validation_results" / f"{task_id}.v{version}.json", {}
  499. ) or {}
  500. verdict = report.get("verdict")
  501. status = "completed" if verdict == "PASS" else (
  502. "failed" if verdict == "FAIL" else "pending"
  503. )
  504. return {
  505. "step_id": task_id,
  506. "task_id": task_id,
  507. "parent_id": None,
  508. "name": task.get("objective") or task_id,
  509. "goal": task.get("reason") or "",
  510. "status": status,
  511. "kind": task.get("skill_id"),
  512. "acceptance": task.get("expectation_ids") or [],
  513. "inputs": task.get("depends_on") or [],
  514. "stage": "GLOBAL_DATA",
  515. "category": task.get("deliverable_type") or "",
  516. "executing": False,
  517. "result": (
  518. {
  519. "summary": delivery.get("summary") or report.get("summary"),
  520. "product_url": None,
  521. "product_text": None,
  522. "verdict": report,
  523. "attempts": 1,
  524. }
  525. if delivery or report
  526. else None
  527. ),
  528. "exec_step_id": None,
  529. "tools": [
  530. item.get("tool_name")
  531. for item in delivery.get("tool_calls") or []
  532. if item.get("tool_name")
  533. ],
  534. "trace_summary": {
  535. "llm": 0,
  536. "tool": len(delivery.get("tool_calls") or []),
  537. },
  538. }
  539. def _segment_snapshots(self, ref: RunRef) -> list[dict[str, Any]]:
  540. plan = self._latest_production_plan(ref.path) or {}
  541. packages = [
  542. _load_json(path, {}) or {}
  543. for path in _files(ref.path, "segment_packages/*.json")
  544. ]
  545. nodes = []
  546. for package in packages:
  547. segment_id = package.get("segment_id") or "Segment"
  548. reports = _files(
  549. ref.path,
  550. f"segment_validation_results/segment.{segment_id}.v*.json",
  551. )
  552. report = _load_json(reports[-1], {}) if reports else {}
  553. verdict = (report or {}).get("verdict")
  554. nodes.append(
  555. {
  556. "step_id": segment_id,
  557. "task_id": segment_id,
  558. "parent_id": None,
  559. "name": segment_id,
  560. "goal": f"生产并验收 {segment_id}",
  561. "status": (
  562. "completed" if verdict == "PASS"
  563. else "failed" if verdict == "FAIL"
  564. else "running"
  565. ),
  566. "kind": "segment-production",
  567. "acceptance": [],
  568. "inputs": [
  569. item.get("input_id")
  570. for item in package.get("production_inputs") or []
  571. ],
  572. "stage": "PRODUCTION",
  573. "category": "video",
  574. "executing": verdict is None,
  575. "result": report or None,
  576. "exec_step_id": None,
  577. "tools": [],
  578. "trace_summary": {"llm": 0, "tool": 0},
  579. }
  580. )
  581. return [
  582. {
  583. "loop_index": 0,
  584. "kind": "initial",
  585. "title": f"Production Plan v{plan.get('plan_version', 1)}",
  586. "step_id": 1,
  587. "step_index": 1,
  588. "is_final": False,
  589. "created_at": None,
  590. "reasoning": plan.get("summary") or "",
  591. "tree_changed": True,
  592. "diff": {
  593. "added": [node["step_id"] for node in nodes],
  594. "removed": [],
  595. "modified": [],
  596. },
  597. "lessons_added": [],
  598. "lessons_cumulative": [],
  599. "tree": {"nodes": nodes},
  600. "node_change": {
  601. node["step_id"]: "added" for node in nodes
  602. },
  603. "round_execution": None,
  604. }
  605. ]
  606. def flat(self, ref: RunRef, round_index: int | None) -> dict[str, Any]:
  607. if ref.kind == "global_data":
  608. return self._global_flat(ref, round_index)
  609. return self._segment_flat(ref, round_index)
  610. @staticmethod
  611. def _module_def(
  612. module_id: int,
  613. key: str,
  614. *,
  615. name: str,
  616. summary: str,
  617. tools: Iterable[str] = (),
  618. inputs: Iterable[tuple[str, str]] = (),
  619. model: str = "",
  620. refs: Iterable[tuple[str, str]] = (),
  621. ) -> dict[str, Any]:
  622. return {
  623. "id": module_id,
  624. "module_key": key,
  625. "kind": "agent",
  626. "name": name,
  627. "summary": summary,
  628. "spec": {
  629. "node_kind": "agent",
  630. "summary": summary,
  631. "system_prompt": "",
  632. "input_schema": [
  633. {
  634. "key": input_key,
  635. "name": input_key,
  636. "title": title,
  637. "required": True,
  638. "source": title,
  639. }
  640. for input_key, title in inputs
  641. ],
  642. "tools": [{"name": value} for value in tools],
  643. "model": model,
  644. },
  645. "view": None,
  646. "fingerprint": hashlib.sha256(
  647. f"{key}:{summary}".encode("utf-8")
  648. ).hexdigest()[:8],
  649. "refs": [
  650. {
  651. "node_key": node_key,
  652. "to_module_key": target,
  653. "seq": index,
  654. "title": target,
  655. "cardinality": "one",
  656. "fanout_over": "",
  657. }
  658. for index, (node_key, target) in enumerate(refs)
  659. ],
  660. }
  661. @staticmethod
  662. def _instance(
  663. instance_id: int,
  664. module_id: int,
  665. module_key: str,
  666. *,
  667. label: str,
  668. task_id: str | None,
  669. ok: bool | None,
  670. inputs: list[dict[str, Any]],
  671. output: Any,
  672. flow: list[dict[str, Any]],
  673. used_tools: Iterable[str] = (),
  674. images: Iterable[str] = (),
  675. ) -> dict[str, Any]:
  676. return {
  677. "id": instance_id,
  678. "module_id": module_id,
  679. "module_key": module_key,
  680. "label": label,
  681. "task_id": task_id,
  682. "step_id": task_id,
  683. "step_index": instance_id,
  684. "branch": "",
  685. "ok": ok,
  686. "input": {"blocks": inputs},
  687. "output": {
  688. "data": output,
  689. "ok": ok,
  690. "images": list(images),
  691. } if output is not None else None,
  692. "timing": {"sec": None},
  693. "tokens": "",
  694. "caller": None,
  695. "used_tools": sorted(set(used_tools)),
  696. "flow": flow,
  697. }
  698. def _global_flat(
  699. self, ref: RunRef, round_index: int | None
  700. ) -> dict[str, Any]:
  701. plans = _files(ref.path, "plans/global_data_dag.v*.json")
  702. metrics = _load_json(ref.path / "run_metrics.json", {}) or {}
  703. model = self._model_names(metrics)
  704. module_defs = {
  705. 1: self._module_def(
  706. 1, "production.global_data.plan",
  707. name="Global Data Planner",
  708. summary="读取 Production Brief,生成或修订完整 Global Data DAG。",
  709. inputs=(("brief", "Production Brief"),),
  710. model=model,
  711. ),
  712. 2: self._module_def(
  713. 2, "production.global_data.execute",
  714. name="Global Data Executor",
  715. summary="按 TaskPackage 与 Skill 白名单执行一个 Task。",
  716. inputs=(("task", "TaskPackage"),),
  717. model=model,
  718. ),
  719. 3: self._module_def(
  720. 3, "production.global_data.validate_task",
  721. name="Task Validator",
  722. summary="在独立上下文中逐项验收 Executor Delivery。",
  723. inputs=(("delivery", "Executor Delivery"),),
  724. model=model,
  725. ),
  726. 4: self._module_def(
  727. 4, "production.global_data.validate_stage",
  728. name="Stage Validator",
  729. summary="全部 Task 通过后检查阶段完整性、冲突和漏项。",
  730. inputs=(("plan", "Global Data Plan"), ("tasks", "PASS Deliveries")),
  731. model=model,
  732. ),
  733. }
  734. rounds: list[dict[str, Any]] = []
  735. for index, path in enumerate(plans):
  736. if round_index is not None and index != round_index:
  737. continue
  738. plan = _load_json(path, {}) or {}
  739. version = int(plan.get("plan_version") or index + 1)
  740. instances: list[dict[str, Any]] = []
  741. next_id = (index + 1) * 1000
  742. next_id += 1
  743. planner_id = next_id
  744. instances.append(
  745. self._instance(
  746. planner_id, 1, "production.global_data.plan",
  747. label=f"Plan v{version}",
  748. task_id=None,
  749. ok=True,
  750. inputs=[
  751. _input_block(
  752. "brief", "Production Brief",
  753. plan.get("goal"), source="production_brief.json",
  754. )
  755. ],
  756. output=plan,
  757. flow=[
  758. _node_card(
  759. 0, node_type="llm", label="规划",
  760. summary=plan.get("revision_summary") or plan.get("goal"),
  761. ok=True, detail=plan,
  762. )
  763. ],
  764. )
  765. )
  766. entry_ids = [planner_id]
  767. for task in self._ordered_tasks(ref, plan.get("tasks") or []):
  768. task_id = str(task.get("task_id"))
  769. delivery = _load_json(
  770. ref.path / "executor_results" / f"{task_id}.v{version}.json",
  771. {},
  772. ) or {}
  773. report = _load_json(
  774. ref.path / "validation_results" / f"{task_id}.v{version}.json",
  775. {},
  776. ) or {}
  777. next_id += 1
  778. executor_id = next_id
  779. tools = [
  780. item.get("tool_name")
  781. for item in delivery.get("tool_calls") or []
  782. if item.get("tool_name")
  783. ]
  784. flow = [
  785. _node_card(
  786. flow_index,
  787. node_type="tool",
  788. label=call.get("tool_name") or "tool",
  789. summary=call.get("output_excerpt"),
  790. ok=call.get("success"),
  791. detail={
  792. "input": call.get("arguments"),
  793. "output": call.get("output_excerpt"),
  794. },
  795. tool_call_id=call.get("tool_call_id") or "",
  796. )
  797. for flow_index, call in enumerate(
  798. delivery.get("tool_calls") or []
  799. )
  800. ]
  801. if not flow and delivery:
  802. flow = [
  803. _node_card(
  804. 0,
  805. node_type="tool",
  806. label="确定性交付",
  807. summary=delivery.get("summary") or "已写入交付结果",
  808. ok=True,
  809. detail=delivery,
  810. )
  811. ]
  812. delivery_view = self._delivery_view(ref, delivery)
  813. image_urls = [
  814. artifact["browser_uri"]
  815. for artifact in (delivery_view or {}).get("artifacts") or []
  816. if str(
  817. artifact.get("mime_type")
  818. or artifact.get("media_type")
  819. or ""
  820. ).startswith("image/")
  821. and artifact.get("browser_uri")
  822. ]
  823. instances.append(
  824. self._instance(
  825. executor_id, 2, "production.global_data.execute",
  826. label=task.get("objective") or task_id,
  827. task_id=task_id,
  828. ok=bool(delivery),
  829. inputs=[
  830. _input_block(
  831. "task", "TaskPackage", task,
  832. source=f"tasks/{task_id}.v{version}.json",
  833. )
  834. ],
  835. output=delivery_view,
  836. flow=flow or [
  837. _node_card(
  838. 0, node_type="note", label="等待执行",
  839. summary="尚无 Executor Delivery", ok=None,
  840. )
  841. ],
  842. used_tools=tools,
  843. images=image_urls,
  844. )
  845. )
  846. entry_ids.append(executor_id)
  847. next_id += 1
  848. validator_id = next_id
  849. instances.append(
  850. self._instance(
  851. validator_id, 3,
  852. "production.global_data.validate_task",
  853. label=f"{task_id} 验收",
  854. task_id=task_id,
  855. ok=report.get("verdict") == "PASS" if report else None,
  856. inputs=[
  857. _input_block(
  858. "delivery", "Executor Delivery",
  859. delivery.get("summary"),
  860. source=f"executor_results/{task_id}.v{version}.json",
  861. )
  862. ],
  863. output=report or None,
  864. flow=[
  865. _node_card(
  866. 0, node_type="llm", label="独立验收",
  867. summary=report.get("summary") or "尚无验收报告",
  868. ok=(
  869. report.get("verdict") == "PASS"
  870. if report else None
  871. ),
  872. detail=report or None,
  873. )
  874. ],
  875. )
  876. )
  877. entry_ids.append(validator_id)
  878. stage_paths = _files(
  879. ref.path,
  880. f"stage_validation_results/global_data.v{version}.json",
  881. )
  882. if stage_paths:
  883. stage = _load_json(stage_paths[-1], {}) or {}
  884. next_id += 1
  885. stage_id = next_id
  886. instances.append(
  887. self._instance(
  888. stage_id, 4, "production.global_data.validate_stage",
  889. label="Global Data Stage 验收",
  890. task_id=None,
  891. ok=stage.get("verdict") == "PASS",
  892. inputs=[
  893. _input_block(
  894. "plan", "Global Data Plan", f"v{version}",
  895. source=path.name,
  896. ),
  897. _input_block(
  898. "tasks", "PASS Deliveries",
  899. [item.get("task_id") for item in plan.get("tasks") or []],
  900. source="executor_results/",
  901. ),
  902. ],
  903. output=stage,
  904. flow=[
  905. _node_card(
  906. 0, node_type="llm", label="阶段总验收",
  907. summary=stage.get("summary"),
  908. ok=stage.get("verdict") == "PASS",
  909. detail=stage,
  910. )
  911. ],
  912. )
  913. )
  914. entry_ids.append(stage_id)
  915. rounds.append(
  916. {"index": index, "entry_ids": entry_ids, "instances": instances}
  917. )
  918. root = {
  919. "id": 0,
  920. "module_id": 0,
  921. "module_key": "production.global_data.workflow",
  922. "label": "Global Data 0.3",
  923. "task_id": None,
  924. "step_id": None,
  925. "step_index": 0,
  926. "branch": "",
  927. "ok": self.brief(ref)["status"] == "success",
  928. "input": {"blocks": []},
  929. "output": None,
  930. "timing": {"sec": self.brief(ref)["duration_sec"]},
  931. "tokens": "",
  932. "caller": None,
  933. "used_tools": [],
  934. "flow": [],
  935. "module": {
  936. "id": 0,
  937. "module_key": "production.global_data.workflow",
  938. "kind": "workflow",
  939. "name": "Global Data Workflow",
  940. "summary": "Preprocess → Plan → Execute → Validate → Replan/Finalize",
  941. "spec": {},
  942. "view": None,
  943. "fingerprint": "0.3",
  944. "refs": [],
  945. },
  946. }
  947. return {
  948. "run_id": ref.numeric_id,
  949. "run_key": ref.key,
  950. "root": root,
  951. "modules": module_defs,
  952. "round_count": len(plans),
  953. "rounds": rounds,
  954. }
  955. @staticmethod
  956. def _ordered_tasks(
  957. ref: RunRef, tasks: list[dict[str, Any]]
  958. ) -> list[dict[str, Any]]:
  959. summary = _load_json(ref.path / "run_summary.json", {}) or {}
  960. order: list[str] = []
  961. for event in summary.get("event_log") or []:
  962. match = re.search(r"(Task\d+)", event)
  963. if match and match.group(1) not in order:
  964. order.append(match.group(1))
  965. rank = {task_id: index for index, task_id in enumerate(order)}
  966. return sorted(
  967. tasks,
  968. key=lambda item: (
  969. rank.get(str(item.get("task_id")), len(rank)),
  970. str(item.get("task_id")),
  971. ),
  972. )
  973. def _delivery_view(
  974. self, ref: RunRef, delivery: dict[str, Any]
  975. ) -> dict[str, Any] | None:
  976. if not delivery:
  977. return None
  978. result = dict(delivery)
  979. result["artifacts"] = [
  980. {
  981. **artifact,
  982. "browser_uri": self.media_url(ref, artifact.get("uri")),
  983. }
  984. for artifact in delivery.get("artifacts") or []
  985. ]
  986. return result
  987. def _segment_flat(
  988. self, ref: RunRef, round_index: int | None
  989. ) -> dict[str, Any]:
  990. plan = self._latest_production_plan(ref.path) or {}
  991. packages = [
  992. (path, _load_json(path, {}) or {})
  993. for path in _files(ref.path, "segment_packages/*.json")
  994. ]
  995. modules = {
  996. 11: self._module_def(
  997. 11, "production.segment.execute",
  998. name="Segment Executor",
  999. summary="按 SegmentPackage 生产镜头、声音、字幕和最终片段。",
  1000. inputs=(("package", "SegmentPackage"),),
  1001. ),
  1002. 12: self._module_def(
  1003. 12, "production.segment.validate",
  1004. name="Segment Validator",
  1005. summary="确定性证据前检后,独立判断六项 Segment 质量。",
  1006. inputs=(("delivery", "Segment Delivery"),),
  1007. ),
  1008. }
  1009. instances: list[dict[str, Any]] = []
  1010. entry_ids: list[int] = []
  1011. operations = [
  1012. _load_json(path, {}) or {}
  1013. for path in _files(ref.path, "tool_operations/*.json")
  1014. ]
  1015. for index, (package_path, package) in enumerate(packages):
  1016. segment_id = str(package.get("segment_id") or f"Segment{index + 1}")
  1017. executor_id = 2000 + index * 10 + 1
  1018. flow = [
  1019. _node_card(
  1020. flow_index,
  1021. node_type="tool",
  1022. label=operation.get("tool_name") or "tool",
  1023. summary=operation.get("status"),
  1024. ok=(
  1025. True
  1026. if operation.get("status") == "SUCCEEDED"
  1027. else False
  1028. if operation.get("status") == "FAILED"
  1029. else None
  1030. ),
  1031. detail=operation,
  1032. )
  1033. for flow_index, operation in enumerate(operations)
  1034. ]
  1035. deliveries = _files(
  1036. ref.path, f"segment_deliveries/segment.{segment_id}.v*.json"
  1037. )
  1038. delivery = _load_json(deliveries[-1], {}) if deliveries else {}
  1039. instances.append(
  1040. self._instance(
  1041. executor_id, 11, "production.segment.execute",
  1042. label=segment_id,
  1043. task_id=segment_id,
  1044. ok=bool(delivery) if deliveries else None,
  1045. inputs=[
  1046. _input_block(
  1047. "package", "SegmentPackage", package,
  1048. source=str(package_path.relative_to(ref.path)),
  1049. )
  1050. ],
  1051. output=delivery or None,
  1052. flow=flow or [
  1053. _node_card(
  1054. 0, node_type="note", label="等待执行",
  1055. summary="尚未形成 Segment Delivery", ok=None,
  1056. )
  1057. ],
  1058. used_tools=[
  1059. item.get("tool_name")
  1060. for item in operations
  1061. if item.get("tool_name")
  1062. ],
  1063. )
  1064. )
  1065. entry_ids.append(executor_id)
  1066. reports = _files(
  1067. ref.path,
  1068. f"segment_validation_results/segment.{segment_id}.v*.json",
  1069. )
  1070. if reports:
  1071. report = _load_json(reports[-1], {}) or {}
  1072. validator_id = executor_id + 1
  1073. instances.append(
  1074. self._instance(
  1075. validator_id, 12, "production.segment.validate",
  1076. label=f"{segment_id} 验收",
  1077. task_id=segment_id,
  1078. ok=report.get("verdict") == "PASS",
  1079. inputs=[
  1080. _input_block(
  1081. "delivery", "Segment Delivery",
  1082. delivery, source=deliveries[-1].name,
  1083. )
  1084. ],
  1085. output=report,
  1086. flow=[
  1087. _node_card(
  1088. 0, node_type="llm", label="六项独立验收",
  1089. summary=report.get("summary"),
  1090. ok=report.get("verdict") == "PASS",
  1091. detail=report,
  1092. )
  1093. ],
  1094. )
  1095. )
  1096. entry_ids.append(validator_id)
  1097. rounds = (
  1098. [{"index": 0, "entry_ids": entry_ids, "instances": instances}]
  1099. if round_index in (None, 0)
  1100. else []
  1101. )
  1102. root = {
  1103. "id": 10,
  1104. "module_id": 10,
  1105. "module_key": "production.segment.workflow",
  1106. "label": f"Production 0.6 · {plan.get('plan_id', ref.key)}",
  1107. "task_id": None,
  1108. "step_id": None,
  1109. "step_index": 0,
  1110. "branch": "",
  1111. "ok": self.brief(ref)["status"] == "success",
  1112. "input": {"blocks": []},
  1113. "output": None,
  1114. "timing": {"sec": self.brief(ref)["duration_sec"]},
  1115. "tokens": "",
  1116. "caller": None,
  1117. "used_tools": [],
  1118. "flow": [],
  1119. "module": {
  1120. "id": 10,
  1121. "module_key": "production.segment.workflow",
  1122. "kind": "workflow",
  1123. "name": "Segment Workflow",
  1124. "summary": "SegmentPackage → Executor → Validator",
  1125. "spec": {},
  1126. "view": None,
  1127. "fingerprint": "0.6",
  1128. "refs": [],
  1129. },
  1130. }
  1131. return {
  1132. "run_id": ref.numeric_id,
  1133. "run_key": ref.key,
  1134. "root": root,
  1135. "modules": modules,
  1136. "round_count": 1,
  1137. "rounds": rounds,
  1138. }
  1139. def media_path(self, ref: RunRef, relative: str) -> Path:
  1140. candidate = (ref.path / relative).resolve()
  1141. if not candidate.is_relative_to(ref.path) or not candidate.is_file():
  1142. raise FileNotFoundError(relative)
  1143. return candidate
  1144. def media_url(self, ref: RunRef, uri: Any) -> str | None:
  1145. if not isinstance(uri, str) or not uri:
  1146. return None
  1147. try:
  1148. path = Path(uri).resolve()
  1149. relative = path.relative_to(ref.path)
  1150. except (ValueError, OSError):
  1151. return uri if uri.startswith(("http://", "https://")) else None
  1152. return f"/api/runs/{ref.numeric_id}/media/{relative.as_posix()}"
  1153. def tools(self) -> dict[str, Any]:
  1154. from production_build_agents.tools.registry import (
  1155. create_default_tool_registry,
  1156. )
  1157. # ToolRegistry 没有只读定义接口;从 Skill 白名单和工具 README 返回稳定说明,
  1158. # 不实例化远端客户端,也不触发环境校验。
  1159. names = {
  1160. "read_production_brief",
  1161. "search_tool",
  1162. "inspect_tool",
  1163. "run_tool",
  1164. "probe_media",
  1165. "view_images",
  1166. "extract_frames",
  1167. "publish_media_reference",
  1168. "transcribe_audio",
  1169. "create_ass_subtitles",
  1170. "inspect_ass_subtitles",
  1171. "render_ass_subtitles",
  1172. "audio_trim",
  1173. "mix_audio_tracks",
  1174. "video_trim",
  1175. "video_concat",
  1176. "video_mux_audio",
  1177. }
  1178. _ = create_default_tool_registry # 保留到实现正式定义投影时使用。
  1179. return {
  1180. name: {
  1181. "description": "Production Build 运行时工具",
  1182. "params": [],
  1183. }
  1184. for name in sorted(names)
  1185. }
  1186. def media_type(path: Path) -> str:
  1187. return mimetypes.guess_type(path.name)[0] or "application/octet-stream"