service.py 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812
  1. """Transactional service for the isolated ``find_agent_v2_*`` tables."""
  2. from __future__ import annotations
  3. import json
  4. import uuid
  5. from dataclasses import asdict
  6. from decimal import Decimal
  7. from typing import Any
  8. from sqlalchemy import case, func, select
  9. from find_agent_v2.models import (
  10. FindAgentV2Candidate,
  11. FindAgentV2Evidence,
  12. FindAgentV2Round,
  13. FindAgentV2Run,
  14. FindAgentV2Search,
  15. )
  16. from find_agent_v2.state import DiscoverySnapshot, ExecutionPlan
  17. from supply_infra.db.session import get_session
  18. from find_agent_v2.gates import (
  19. build_rule_snapshot,
  20. evaluate_candidate_gate,
  21. parse_datetime_value,
  22. )
  23. RUN_TIMEOUT_MINUTES = 60
  24. RUN_TIMEOUT_SECONDS = RUN_TIMEOUT_MINUTES * 60
  25. RUN_TIMEOUT_REASON = "运行时间超过 60 分钟,系统自动标记失败"
  26. class FindAgentV2RunNotFound(LookupError):
  27. pass
  28. def _json(value: Any) -> str:
  29. return json.dumps(value, ensure_ascii=False, default=str)
  30. def _loads(value: str | None, default: Any) -> Any:
  31. try:
  32. return json.loads(value) if value else default
  33. except (TypeError, ValueError):
  34. return default
  35. def _ratio(value: Any) -> Decimal | None:
  36. if value in (None, "") or isinstance(value, bool):
  37. return None
  38. number = Decimal(str(value))
  39. if number > 1 and number <= 100:
  40. number /= 100
  41. if number < 0 or number > 1:
  42. raise ValueError("ratio 必须在 0~1")
  43. return number.quantize(Decimal("0.000001"))
  44. def _fails_search_share_gate(
  45. provider: str, share_count: Any, min_share_count: int,
  46. ) -> bool:
  47. """Reject any provider's explicit share metric below the configured threshold."""
  48. del provider
  49. return share_count is not None and int(share_count) < int(min_share_count)
  50. def _candidate_dict(row: FindAgentV2Candidate) -> dict[str, Any]:
  51. detail = _loads(row.detail_json, {})
  52. return {
  53. "candidate_id": int(row.id),
  54. "aweme_id": row.aweme_id,
  55. "title": row.title,
  56. "content_link": row.content_link,
  57. "video_url": detail.get("video_url") if isinstance(detail, dict) else None,
  58. "author_name": row.author_name,
  59. "author_sec_uid": row.author_sec_uid,
  60. "source_keywords": _loads(row.source_keywords_json, []),
  61. "tags": _loads(row.tags_json, []),
  62. "publish_at": row.publish_at.isoformat() if row.publish_at else None,
  63. "duration_seconds": float(row.duration_seconds) if row.duration_seconds is not None else None,
  64. "play_count": row.play_count,
  65. "like_count": row.like_count,
  66. "comment_count": row.comment_count,
  67. "collect_count": row.collect_count,
  68. "share_count": row.share_count,
  69. "detail_status": row.detail_status,
  70. "portrait_status": row.portrait_status,
  71. "content_50_plus_ratio": float(row.content_50_plus_ratio) if row.content_50_plus_ratio is not None else None,
  72. "account_50_plus_ratio": float(row.account_50_plus_ratio) if row.account_50_plus_ratio is not None else None,
  73. "relevance_score": float(row.relevance_score) if row.relevance_score is not None else None,
  74. "elder_score": float(row.elder_score) if row.elder_score is not None else None,
  75. "share_score": float(row.share_score) if row.share_score is not None else None,
  76. "value_score": float(row.value_score) if row.value_score is not None else None,
  77. "gate_status": row.gate_status,
  78. "gate_result": _loads(row.gate_result_json, {}),
  79. "decision_bucket": row.decision_bucket,
  80. "decision_reason": row.decision_reason,
  81. "reject_reason_code": row.reject_reason_code,
  82. }
  83. class FindAgentV2Service:
  84. def create_run(
  85. self,
  86. *,
  87. user_input: str,
  88. demand_word: str,
  89. demand_grade_id: int | None = None,
  90. run_id: str | None = None,
  91. rule_config: dict[str, Any] | None = None,
  92. triggered_by_user_id: int | None = None,
  93. triggered_by_username: str | None = None,
  94. trigger_source: str = "cli",
  95. ) -> str:
  96. run_key = str(run_id or uuid.uuid4().hex)[:64]
  97. with get_session() as session:
  98. exists = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_key))
  99. if exists is not None:
  100. raise ValueError(f"find_agent_v2 run_id 已存在: {run_key}")
  101. rules = rule_config or build_rule_snapshot()
  102. session.add(FindAgentV2Run(
  103. run_id=run_key,
  104. demand_grade_id=demand_grade_id,
  105. demand_word=str(demand_word)[:256],
  106. triggered_by_user_id=triggered_by_user_id,
  107. triggered_by_username=(str(triggered_by_username)[:64] if triggered_by_username else None),
  108. trigger_source=str(trigger_source or "cli")[:32],
  109. input_json=_json({"user_input": user_input}),
  110. rule_config_json=_json(rules),
  111. status="running",
  112. ))
  113. return run_key
  114. def lookup_run(self, run_id: str) -> dict[str, Any] | None:
  115. with get_session() as session:
  116. row = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_id))
  117. if row is None:
  118. return None
  119. return {
  120. "run_id": row.run_id,
  121. "demand_grade_id": row.demand_grade_id,
  122. "demand_word": row.demand_word,
  123. "triggered_by_user_id": row.triggered_by_user_id,
  124. "triggered_by_username": row.triggered_by_username,
  125. "trigger_source": row.trigger_source,
  126. "status": row.status,
  127. "outcome_status": row.outcome_status,
  128. "current_round": row.current_round,
  129. "search_count": row.search_count,
  130. "candidate_count": row.candidate_count,
  131. "valid_primary_count": row.valid_primary_count,
  132. "input_tokens": row.input_tokens,
  133. "output_tokens": row.output_tokens,
  134. "total_tokens": row.total_tokens,
  135. "cost_usd": float(row.cost_usd or 0),
  136. "intent_summary": row.intent_summary,
  137. "stop_reason": row.stop_reason,
  138. "obagent_run_uid": row.obagent_run_uid,
  139. "rule_config": _loads(row.rule_config_json, {}),
  140. "create_time": row.create_time.isoformat() if row.create_time else None,
  141. "update_time": row.update_time.isoformat() if row.update_time else None,
  142. }
  143. def fail_run(self, run_id: str, reason: str) -> None:
  144. """Persist failures that happen outside the normal Agent finalization path."""
  145. with get_session() as session:
  146. row = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_id))
  147. if row is None:
  148. raise FindAgentV2RunNotFound(run_id)
  149. row.status = "failed"
  150. row.outcome_status = "failed"
  151. row.stop_reason = str(reason)[:2000]
  152. def set_obagent_run_uid(self, run_id: str, run_uid: str | None) -> None:
  153. if not run_uid:
  154. return
  155. with get_session() as session:
  156. row = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_id))
  157. if row is None:
  158. raise FindAgentV2RunNotFound(run_id)
  159. row.obagent_run_uid = str(run_uid)[:64]
  160. def add_usage(self, run_id: str, usage: dict[str, Any]) -> None:
  161. """Accumulate one execution attempt; resume never erases earlier usage."""
  162. with get_session() as session:
  163. row = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_id))
  164. if row is None:
  165. raise FindAgentV2RunNotFound(run_id)
  166. row.input_tokens += int(usage.get("input_tokens") or 0)
  167. row.output_tokens += int(usage.get("output_tokens") or 0)
  168. row.total_tokens += int(usage.get("total_tokens") or 0)
  169. row.cost_usd = (
  170. Decimal(str(row.cost_usd or 0)) + Decimal(str(usage.get("cost") or 0))
  171. ).quantize(Decimal("0.00000001"))
  172. def prepare_resume(self, run_id: str) -> dict[str, Any]:
  173. """Reset a terminal run for a recovery round without deleting audit rows."""
  174. with get_session() as session:
  175. row = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_id))
  176. if row is None:
  177. raise FindAgentV2RunNotFound(run_id)
  178. if row.status == "running":
  179. raise ValueError(f"run_id={run_id} 已处于 running,不能重复 resume")
  180. row.status = "running"
  181. row.outcome_status = None
  182. row.stop_reason = None
  183. return self.require_run(run_id)
  184. def require_run(self, run_id: str) -> dict[str, Any]:
  185. run = self.lookup_run(run_id)
  186. if run is None:
  187. raise FindAgentV2RunNotFound(f"find_agent_v2 run_id 不存在: {run_id}")
  188. return run
  189. def get_run_user_input(self, run_id: str) -> str:
  190. """Return the immutable task input stored when a v2 run was prepared."""
  191. with get_session() as session:
  192. row = session.scalar(select(FindAgentV2Run).where(
  193. FindAgentV2Run.run_id == run_id
  194. ))
  195. if row is None:
  196. raise FindAgentV2RunNotFound(run_id)
  197. user_input = _loads(row.input_json, {}).get("user_input")
  198. if not isinstance(user_input, str) or not user_input.strip():
  199. raise ValueError(f"run_id={run_id} 缺少有效 user_input")
  200. return user_input
  201. def get_latest_execution_plan(self, run_id: str) -> ExecutionPlan | None:
  202. """Restore the newest validated plan for continuation/resume without replanning raw input."""
  203. with get_session() as session:
  204. raw = session.scalar(
  205. select(FindAgentV2Round.plan_json)
  206. .where(
  207. FindAgentV2Round.run_id == run_id,
  208. FindAgentV2Round.plan_json.is_not(None),
  209. )
  210. .order_by(FindAgentV2Round.round_index.desc(), FindAgentV2Round.id.desc())
  211. .limit(1)
  212. )
  213. payload = _loads(raw, None)
  214. if not isinstance(payload, dict):
  215. return None
  216. try:
  217. return ExecutionPlan.model_validate(payload)
  218. except ValueError:
  219. # Old rounds may contain the pre-contract free-form plan shape.
  220. return None
  221. def begin_round(self, run_id: str, round_index: int, snapshot: DiscoverySnapshot) -> None:
  222. with get_session() as session:
  223. run = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_id))
  224. if run is None:
  225. raise FindAgentV2RunNotFound(run_id)
  226. run.current_round = int(round_index)
  227. session.add(FindAgentV2Round(
  228. run_id=run_id,
  229. round_index=int(round_index),
  230. phase="planning",
  231. status="open",
  232. start_snapshot_json=_json(asdict(snapshot)),
  233. ))
  234. def update_round(
  235. self,
  236. run_id: str,
  237. round_index: int,
  238. *,
  239. phase: str | None = None,
  240. plan: str | None = None,
  241. status: str | None = None,
  242. snapshot: DiscoverySnapshot | None = None,
  243. error: str | None = None,
  244. ) -> None:
  245. with get_session() as session:
  246. row = session.scalar(select(FindAgentV2Round).where(
  247. FindAgentV2Round.run_id == run_id,
  248. FindAgentV2Round.round_index == int(round_index),
  249. ))
  250. if row is None:
  251. raise FindAgentV2RunNotFound(f"round 不存在: {run_id}/{round_index}")
  252. if phase is not None:
  253. row.phase = phase
  254. if plan is not None:
  255. row.plan_json = plan
  256. if status is not None:
  257. row.status = status
  258. if snapshot is not None:
  259. row.end_snapshot_json = _json(asdict(snapshot))
  260. if error is not None:
  261. row.error_message = error[:2000]
  262. def save_search(
  263. self,
  264. *,
  265. run_id: str,
  266. round_index: int,
  267. keyword: str,
  268. query_reason: str,
  269. source_type: str,
  270. provider: str,
  271. cursor: str,
  272. page_no: int,
  273. payload: dict[str, Any],
  274. ) -> dict[str, Any]:
  275. results = list(payload.get("search_results") or [])
  276. with get_session() as session:
  277. run = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_id))
  278. if run is None:
  279. raise FindAgentV2RunNotFound(run_id)
  280. search = FindAgentV2Search(
  281. run_id=run_id,
  282. round_index=round_index,
  283. keyword=keyword[:256],
  284. query_reason=query_reason,
  285. source_type=source_type[:32],
  286. provider=provider[:32],
  287. cursor=str(cursor)[:128],
  288. page_no=page_no,
  289. has_more=int(bool(payload.get("has_more"))),
  290. next_cursor=str(payload.get("next_cursor") or "")[:128] or None,
  291. provider_state_json=_json({
  292. "search_id": payload.get("search_id"),
  293. "backtrace": payload.get("backtrace"),
  294. }),
  295. result_count=len(results),
  296. status="failed" if payload.get("error") else "success",
  297. error_message=str(payload.get("error") or "") or None,
  298. raw_response_json=_json(payload),
  299. )
  300. session.add(search)
  301. session.flush()
  302. new_count = 0
  303. share_gate_rejected_count = 0
  304. min_share_count = int(_loads(run.rule_config_json, {}).get("min_share_count", 1000))
  305. for item in results:
  306. aweme_id = str(item.get("aweme_id") or "").strip()
  307. if not aweme_id:
  308. continue
  309. candidate = session.scalar(select(FindAgentV2Candidate).where(
  310. FindAgentV2Candidate.run_id == run_id,
  311. FindAgentV2Candidate.aweme_id == aweme_id,
  312. ))
  313. author = item.get("author") if isinstance(item.get("author"), dict) else {}
  314. stats = item.get("statistics") if isinstance(item.get("statistics"), dict) else {}
  315. if candidate is None:
  316. candidate = FindAgentV2Candidate(
  317. run_id=run_id,
  318. first_search_id=search.id,
  319. aweme_id=aweme_id,
  320. decision_bucket="pending_evaluation",
  321. )
  322. session.add(candidate)
  323. new_count += 1
  324. keywords = _loads(candidate.source_keywords_json, [])
  325. if keyword not in keywords:
  326. keywords.append(keyword)
  327. candidate.source_keywords_json = _json(keywords)
  328. candidate.title = str(item.get("desc") or item.get("title") or candidate.title or "")[:512] or None
  329. candidate.content_link = str(item.get("url") or candidate.content_link or "")[:1024] or None
  330. candidate.author_name = str(author.get("nickname") or candidate.author_name or "")[:256] or None
  331. candidate.author_sec_uid = str(author.get("sec_uid") or candidate.author_sec_uid or "")[:256] or None
  332. candidate.tags_json = _json(item.get("topics") or item.get("tags") or [])
  333. duration_ms = item.get("duration_ms")
  334. if duration_ms:
  335. candidate.duration_seconds = Decimal(str(duration_ms)) / 1000
  336. metric_fields = {
  337. "play_count": stats.get("play_count"),
  338. "like_count": (
  339. stats.get("digg_count")
  340. if stats.get("digg_count") is not None
  341. else stats.get("like_count")
  342. ),
  343. "comment_count": stats.get("comment_count"),
  344. "collect_count": stats.get("collect_count"),
  345. "share_count": stats.get("share_count"),
  346. }
  347. for field, value in metric_fields.items():
  348. if value is not None:
  349. setattr(candidate, field, int(value))
  350. search_share_count = stats.get("share_count")
  351. if _fails_search_share_gate(provider, search_share_count, min_share_count):
  352. gate = {
  353. "stage": "search",
  354. "status": "fail",
  355. "primary_eligible": False,
  356. "failed_reason_codes": ["SHARE_COUNT_TOO_LOW"],
  357. "checks": [{
  358. "name": "share_count",
  359. "status": "fail",
  360. "reason_code": "SHARE_COUNT_TOO_LOW",
  361. "actual": int(search_share_count),
  362. "threshold": min_share_count,
  363. "compensated": False,
  364. }],
  365. }
  366. candidate.gate_status = "fail"
  367. candidate.gate_result_json = _json(gate)
  368. candidate.decision_bucket = "rejected"
  369. candidate.decision_reason = (
  370. f"搜索结果分享数 {int(search_share_count)} 低于门槛 {min_share_count}"
  371. )
  372. candidate.reject_reason_code = "SHARE_COUNT_TOO_LOW"
  373. share_gate_rejected_count += 1
  374. elif (
  375. search_share_count is not None
  376. and candidate.reject_reason_code == "SHARE_COUNT_TOO_LOW"
  377. ):
  378. # A later search response may contain a refreshed metric.
  379. candidate.gate_status = None
  380. candidate.gate_result_json = None
  381. candidate.decision_bucket = "pending_evaluation"
  382. candidate.decision_reason = None
  383. candidate.reject_reason_code = None
  384. run.search_count = int(session.scalar(select(func.count()).select_from(FindAgentV2Search).where(FindAgentV2Search.run_id == run_id)) or 0)
  385. session.flush()
  386. run.candidate_count = int(session.scalar(select(func.count()).select_from(FindAgentV2Candidate).where(FindAgentV2Candidate.run_id == run_id)) or 0)
  387. return {
  388. "search_id": int(search.id),
  389. "new_candidate_count": new_count,
  390. "result_count": len(results),
  391. "share_gate_rejected_count": share_gate_rejected_count,
  392. }
  393. def candidate_inputs(self, run_id: str, candidate_ids: list[int]) -> list[dict[str, Any]]:
  394. with get_session() as session:
  395. rows = list(session.scalars(select(FindAgentV2Candidate).where(
  396. FindAgentV2Candidate.run_id == run_id,
  397. FindAgentV2Candidate.id.in_([int(v) for v in candidate_ids]),
  398. )))
  399. return [_candidate_dict(row) for row in rows]
  400. def candidate_input(self, run_id: str, candidate_id: int) -> dict[str, Any]:
  401. """Return one candidate only when it belongs to the requested v2 run."""
  402. items = self.candidate_inputs(run_id, [candidate_id])
  403. if not items or int(items[0]["candidate_id"]) != int(candidate_id):
  404. raise ValueError(f"candidate_id 不属于 run: {candidate_id}")
  405. return items[0]
  406. def get_search_summaries(self, run_id: str) -> list[dict[str, Any]]:
  407. """Return compact search execution metadata without loading candidates."""
  408. self.require_run(run_id)
  409. with get_session() as session:
  410. rows = list(session.scalars(select(FindAgentV2Search).where(
  411. FindAgentV2Search.run_id == run_id,
  412. ).order_by(FindAgentV2Search.id)))
  413. return [{
  414. "search_id": int(row.id),
  415. "round_index": row.round_index,
  416. "keyword": row.keyword,
  417. "query_reason": row.query_reason,
  418. "provider": row.provider,
  419. "page_no": row.page_no,
  420. "has_more": bool(row.has_more),
  421. "next_cursor": row.next_cursor,
  422. "status": row.status,
  423. "result_count": row.result_count,
  424. } for row in rows]
  425. def get_candidate_progress(self, run_id: str) -> dict[str, int]:
  426. """Compute full-run candidate progress with SQL aggregates, never a row limit."""
  427. self.require_run(run_id)
  428. pending = FindAgentV2Candidate.decision_bucket == "pending_evaluation"
  429. detail_pending = FindAgentV2Candidate.detail_status == "pending"
  430. portrait_pending = FindAgentV2Candidate.portrait_status == "pending"
  431. with get_session() as session:
  432. row = session.execute(select(
  433. func.count(FindAgentV2Candidate.id),
  434. func.sum(case((pending, 1), else_=0)),
  435. func.sum(case((FindAgentV2Candidate.decision_bucket == "primary", 1), else_=0)),
  436. func.sum(case((FindAgentV2Candidate.decision_bucket == "rejected", 1), else_=0)),
  437. func.sum(case((pending & detail_pending, 1), else_=0)),
  438. func.sum(case((pending & (FindAgentV2Candidate.detail_status == "success"), 1), else_=0)),
  439. func.sum(case((pending & (FindAgentV2Candidate.detail_status == "failed"), 1), else_=0)),
  440. func.sum(case((pending & portrait_pending, 1), else_=0)),
  441. func.sum(case((pending & (FindAgentV2Candidate.portrait_status == "success"), 1), else_=0)),
  442. func.sum(case((pending & (FindAgentV2Candidate.portrait_status == "failed"), 1), else_=0)),
  443. func.sum(case((pending & ~detail_pending & ~portrait_pending, 1), else_=0)),
  444. func.sum(case((
  445. pending
  446. & (FindAgentV2Candidate.detail_status == "success")
  447. & (FindAgentV2Candidate.portrait_status == "success"),
  448. 1,
  449. ), else_=0)),
  450. ).where(FindAgentV2Candidate.run_id == run_id)).one()
  451. values = [int(value or 0) for value in row]
  452. keys = (
  453. "total_count", "pending_count", "primary_count", "rejected_count",
  454. "detail_pending_count", "detail_success_count", "detail_failed_count",
  455. "portrait_pending_count", "portrait_success_count", "portrait_failed_count",
  456. "evidence_completed_count", "evidence_success_count",
  457. )
  458. return dict(zip(keys, values, strict=True))
  459. def list_pending_evidence_ids(
  460. self, run_id: str, evidence_type: str, *, limit: int,
  461. ) -> list[int]:
  462. """Claimable evidence work projection; callers repeatedly drain this queue."""
  463. if evidence_type not in {"detail", "portrait"}:
  464. raise ValueError("evidence_type 只能是 detail/portrait")
  465. status_column = (
  466. FindAgentV2Candidate.detail_status
  467. if evidence_type == "detail"
  468. else FindAgentV2Candidate.portrait_status
  469. )
  470. with get_session() as session:
  471. return [int(value) for value in session.scalars(select(
  472. FindAgentV2Candidate.id,
  473. ).where(
  474. FindAgentV2Candidate.run_id == run_id,
  475. FindAgentV2Candidate.decision_bucket == "pending_evaluation",
  476. status_column == "pending",
  477. ).order_by(FindAgentV2Candidate.id).limit(max(1, int(limit))))]
  478. def list_ready_evaluation_ids(self, run_id: str, *, limit: int) -> list[int]:
  479. """Return pending candidates whose detail and portrait attempts have completed."""
  480. with get_session() as session:
  481. return [int(value) for value in session.scalars(select(
  482. FindAgentV2Candidate.id,
  483. ).where(
  484. FindAgentV2Candidate.run_id == run_id,
  485. FindAgentV2Candidate.decision_bucket == "pending_evaluation",
  486. FindAgentV2Candidate.detail_status != "pending",
  487. FindAgentV2Candidate.portrait_status != "pending",
  488. ).order_by(FindAgentV2Candidate.id).limit(max(1, int(limit))))]
  489. def count_pending_candidates(self, run_id: str) -> int:
  490. with get_session() as session:
  491. return int(session.scalar(select(func.count()).select_from(
  492. FindAgentV2Candidate,
  493. ).where(
  494. FindAgentV2Candidate.run_id == run_id,
  495. FindAgentV2Candidate.decision_bucket == "pending_evaluation",
  496. )) or 0)
  497. def get_report_state(self, run_id: str) -> dict[str, Any]:
  498. """Report-only projection: aggregate summary, primaries and rejection distribution."""
  499. run = self.require_run(run_id)
  500. progress = self.get_candidate_progress(run_id)
  501. with get_session() as session:
  502. primaries = list(session.scalars(select(FindAgentV2Candidate).where(
  503. FindAgentV2Candidate.run_id == run_id,
  504. FindAgentV2Candidate.decision_bucket == "primary",
  505. ).order_by(
  506. FindAgentV2Candidate.value_score.desc(),
  507. FindAgentV2Candidate.id,
  508. )))
  509. rejection_rows = session.execute(select(
  510. FindAgentV2Candidate.reject_reason_code,
  511. func.count(FindAgentV2Candidate.id),
  512. ).where(
  513. FindAgentV2Candidate.run_id == run_id,
  514. FindAgentV2Candidate.decision_bucket == "rejected",
  515. ).group_by(FindAgentV2Candidate.reject_reason_code)).all()
  516. primary_candidates = [_candidate_dict(row) for row in primaries]
  517. return {
  518. "run": run,
  519. "summary": progress,
  520. "primary_candidates": primary_candidates,
  521. "rejection_reason_distribution": {
  522. str(reason or "UNSPECIFIED"): int(count)
  523. for reason, count in rejection_rows
  524. },
  525. }
  526. def save_details(
  527. self,
  528. run_id: str,
  529. details: list[dict[str, Any]],
  530. errors: list[dict[str, Any]],
  531. *,
  532. requested_aweme_ids: list[str] | None = None,
  533. ) -> None:
  534. by_id = {str(item.get("content_id") or ""): item for item in details}
  535. error_by_id = {str(item.get("content_id") or ""): item for item in errors}
  536. requested = {str(value) for value in (requested_aweme_ids or []) if str(value)}
  537. target_ids = set(by_id) | set(error_by_id) | requested
  538. with get_session() as session:
  539. rows = list(session.scalars(select(FindAgentV2Candidate).where(
  540. FindAgentV2Candidate.run_id == run_id,
  541. FindAgentV2Candidate.aweme_id.in_(target_ids),
  542. )))
  543. for row in rows:
  544. detail = by_id.get(row.aweme_id)
  545. error = error_by_id.get(row.aweme_id)
  546. if detail:
  547. row.detail_status = "success"
  548. row.detail_json = _json(detail)
  549. row.title = str(detail.get("title") or detail.get("body_text") or row.title or "")[:512] or None
  550. row.content_link = str(detail.get("content_link") or row.content_link or "")[:1024] or None
  551. row.author_name = str(detail.get("channel_account_name") or row.author_name or "")[:256] or None
  552. row.author_sec_uid = str(detail.get("channel_account_id") or row.author_sec_uid or "")[:256] or None
  553. row.tags_json = _json(detail.get("topic_list") or [])
  554. parsed = parse_datetime_value(detail.get("publish_at"))
  555. row.publish_at = parsed.replace(tzinfo=None) if parsed else None
  556. row.duration_seconds = detail.get("duration_seconds") or None
  557. for key in ("play_count", "like_count", "comment_count", "collect_count", "share_count"):
  558. value = detail.get(key)
  559. if value is not None:
  560. setattr(row, key, value)
  561. status, raw = "success", detail
  562. else:
  563. error = error or {
  564. "error": "上游详情响应未包含已请求候选",
  565. "error_code": "UPSTREAM_RESULT_MISSING",
  566. }
  567. row.detail_status = "failed"
  568. status, raw = "failed", error
  569. session.add(FindAgentV2Evidence(
  570. run_id=run_id,
  571. candidate_id=row.id,
  572. evidence_type="detail",
  573. provider="crawler",
  574. status=status,
  575. raw_json=_json(raw),
  576. error_message=str((error or {}).get("error") or "") or None,
  577. ))
  578. def save_portraits(
  579. self,
  580. run_id: str,
  581. results: list[dict[str, Any]],
  582. *,
  583. requested_aweme_ids: list[str] | None = None,
  584. ) -> None:
  585. by_id = {str(item.get("aweme_id") or ""): item for item in results}
  586. requested = {str(value) for value in (requested_aweme_ids or []) if str(value)}
  587. target_ids = set(by_id) | requested
  588. with get_session() as session:
  589. rows = list(session.scalars(select(FindAgentV2Candidate).where(
  590. FindAgentV2Candidate.run_id == run_id,
  591. FindAgentV2Candidate.aweme_id.in_(target_ids),
  592. )))
  593. for row in rows:
  594. item = by_id.get(row.aweme_id) or {
  595. "aweme_id": row.aweme_id,
  596. "error": "上游画像响应未包含已请求候选",
  597. "error_code": "UPSTREAM_RESULT_MISSING",
  598. }
  599. normalization = item.get("age_normalization") or {}
  600. content = normalization.get("content") or {}
  601. account = normalization.get("account") or {}
  602. row.portrait_json = _json(item)
  603. row.portrait_status = "failed" if item.get("error") else "success"
  604. row.content_50_plus_ratio = _ratio(
  605. content.get("older_ratio") if content.get("has_age_portrait") else None
  606. )
  607. row.account_50_plus_ratio = _ratio(
  608. account.get("older_ratio") if account.get("has_age_portrait") else None
  609. )
  610. session.add(FindAgentV2Evidence(
  611. run_id=run_id,
  612. candidate_id=row.id,
  613. evidence_type="portrait",
  614. provider="douhot",
  615. status=row.portrait_status,
  616. raw_json=_json(item),
  617. normalized_json=_json(normalization),
  618. error_message=str(item.get("error") or "") or None,
  619. ))
  620. def evaluate(self, run_id: str, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
  621. run_data = self.require_run(run_id)
  622. output: list[dict[str, Any]] = []
  623. with get_session() as session:
  624. run = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_id))
  625. assert run is not None
  626. for item in items:
  627. row = session.scalar(select(FindAgentV2Candidate).where(
  628. FindAgentV2Candidate.run_id == run_id,
  629. FindAgentV2Candidate.id == int(item.get("candidate_id") or 0),
  630. ))
  631. if row is None:
  632. raise ValueError(f"candidate_id 不属于 run: {item.get('candidate_id')}")
  633. for key in ("relevance_score", "elder_score", "share_score", "value_score"):
  634. setattr(row, key, _ratio(item.get(key)))
  635. requested = str(item.get("decision_bucket") or "rejected")
  636. if requested not in {"primary", "rejected"}:
  637. raise ValueError("decision_bucket 只能是 primary/rejected")
  638. gate_input = _candidate_dict(row)
  639. gate = evaluate_candidate_gate(gate_input, run_data["rule_config"])
  640. row.gate_status = gate["status"]
  641. row.gate_result_json = _json(gate)
  642. row.decision_bucket = "primary" if requested == "primary" and gate["status"] == "pass" else "rejected"
  643. row.decision_reason = str(item.get("decision_reason") or "")
  644. failed = list(gate.get("failed_reason_codes") or [])
  645. row.reject_reason_code = (str(item.get("reject_reason_code") or "") or (failed[0] if failed else None))
  646. output.append({"candidate_id": int(row.id), "decision_bucket": row.decision_bucket, "gate_status": row.gate_status})
  647. session.flush()
  648. primaries = list(session.scalars(select(FindAgentV2Candidate).where(
  649. FindAgentV2Candidate.run_id == run_id,
  650. FindAgentV2Candidate.decision_bucket == "primary",
  651. )))
  652. run.valid_primary_count = len({row.aweme_id for row in primaries if row.gate_status == "pass"})
  653. return output
  654. def reject_failed_gates(
  655. self, run_id: str, failures: list[tuple[int, dict[str, Any]]],
  656. ) -> list[int]:
  657. """Reject hard-gate failures before any evaluator model invocation."""
  658. rejected: list[int] = []
  659. if not failures:
  660. return rejected
  661. by_id = {int(candidate_id): gate for candidate_id, gate in failures}
  662. with get_session() as session:
  663. rows = list(session.scalars(select(FindAgentV2Candidate).where(
  664. FindAgentV2Candidate.run_id == run_id,
  665. FindAgentV2Candidate.id.in_(list(by_id)),
  666. FindAgentV2Candidate.decision_bucket == "pending_evaluation",
  667. )))
  668. for row in rows:
  669. gate = by_id[int(row.id)]
  670. failed = list(gate.get("failed_reason_codes") or [])
  671. row.gate_status = "fail"
  672. row.gate_result_json = _json(gate)
  673. row.decision_bucket = "rejected"
  674. row.decision_reason = "硬门禁未通过:" + ", ".join(failed)
  675. row.reject_reason_code = failed[0] if failed else "HARD_GATE_FAILED"
  676. rejected.append(int(row.id))
  677. return rejected
  678. def recount_valid_primary(self, run_id: str) -> int:
  679. """Recompute the denormalized primary counter after parallel worker writes."""
  680. with get_session() as session:
  681. run = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_id))
  682. if run is None:
  683. raise FindAgentV2RunNotFound(run_id)
  684. count = int(session.scalar(select(func.count(func.distinct(
  685. FindAgentV2Candidate.aweme_id,
  686. ))).where(
  687. FindAgentV2Candidate.run_id == run_id,
  688. FindAgentV2Candidate.decision_bucket == "primary",
  689. FindAgentV2Candidate.gate_status == "pass",
  690. )) or 0)
  691. run.valid_primary_count = count
  692. return count
  693. def get_full_state(
  694. self, run_id: str, *, limit: int = 100, pending_only: bool = False,
  695. ) -> dict[str, Any]:
  696. run = self.require_run(run_id)
  697. with get_session() as session:
  698. searches = list(session.scalars(select(FindAgentV2Search).where(
  699. FindAgentV2Search.run_id == run_id,
  700. ).order_by(FindAgentV2Search.id)))
  701. candidate_query = select(FindAgentV2Candidate).where(
  702. FindAgentV2Candidate.run_id == run_id,
  703. )
  704. if pending_only:
  705. candidate_query = candidate_query.where(
  706. FindAgentV2Candidate.decision_bucket == "pending_evaluation",
  707. )
  708. candidates = list(session.scalars(candidate_query.order_by(
  709. FindAgentV2Candidate.value_score.desc(), FindAgentV2Candidate.id,
  710. ).limit(max(1, min(limit, 500)))))
  711. return {
  712. "run": run,
  713. "searches": [{
  714. "search_id": int(row.id), "round_index": row.round_index,
  715. "keyword": row.keyword, "query_reason": row.query_reason,
  716. "provider": row.provider, "page_no": row.page_no,
  717. "has_more": bool(row.has_more), "next_cursor": row.next_cursor,
  718. "status": row.status, "result_count": row.result_count,
  719. } for row in searches],
  720. "candidates": [_candidate_dict(row) for row in candidates],
  721. }
  722. def snapshot(self, run_id: str) -> DiscoverySnapshot:
  723. run = self.require_run(run_id)
  724. progress = self.get_candidate_progress(run_id)
  725. return DiscoverySnapshot(
  726. status=run["status"],
  727. search_count=run["search_count"],
  728. candidate_count=progress["total_count"],
  729. pending_count=progress["pending_count"],
  730. primary_count=progress["primary_count"],
  731. valid_primary_count=run["valid_primary_count"],
  732. rejected_count=progress["rejected_count"],
  733. outcome_status=run.get("outcome_status") or "",
  734. )
  735. def finalize(
  736. self,
  737. run_id: str,
  738. *,
  739. failed: bool = False,
  740. reason: str = "",
  741. ) -> dict[str, Any]:
  742. snapshot = self.snapshot(run_id)
  743. if failed:
  744. outcome, status = "failed", "failed"
  745. elif snapshot.valid_primary_count > 0:
  746. outcome, status = "goal_met", "finished"
  747. else:
  748. outcome, status = "no_match", "finished"
  749. with get_session() as session:
  750. run = session.scalar(select(FindAgentV2Run).where(FindAgentV2Run.run_id == run_id))
  751. if run is None:
  752. raise FindAgentV2RunNotFound(run_id)
  753. if not (run.status == "failed" and run.outcome_status == "failed"):
  754. run.status = status
  755. run.outcome_status = outcome
  756. run.stop_reason = reason[:2000] or None
  757. return self.require_run(run_id)
  758. _SERVICE = FindAgentV2Service()
  759. def get_find_agent_v2_service() -> FindAgentV2Service:
  760. return _SERVICE