learning_review.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796
  1. from __future__ import annotations
  2. from collections import Counter
  3. import hashlib
  4. from typing import Any
  5. from content_agent.constants import RUNTIME_SCHEMA_VERSION
  6. from content_agent.interfaces import RuntimeFileStore
  7. from content_agent.record_payload import with_raw_payload
  8. CURRENT_DECISION_ACTIONS = {
  9. "ADD_TO_CONTENT_POOL",
  10. "KEEP_CONTENT_FOR_REVIEW",
  11. "REJECT_CONTENT",
  12. "TECHNICAL_RETRY_REQUIRED",
  13. }
  14. CURRENT_QUERY_EFFECT_STATUSES = {"success", "pending", "failed", "rule_blocked"}
  15. def run(
  16. run_id: str,
  17. policy_run_id: str,
  18. runtime: RuntimeFileStore,
  19. review_id: str | None = None,
  20. ) -> dict[str, Any]:
  21. final_output = runtime.read_json(run_id, "final_output.json")
  22. search_clues = _current_contract_search_clues(runtime.read_jsonl(run_id, "search_clues.jsonl"))
  23. decisions = _current_contract_decisions(runtime.read_jsonl(run_id, "rule_decisions.jsonl"))
  24. discovered_content_items = runtime.read_jsonl(run_id, "discovered_content_items.jsonl")
  25. source_path_records = runtime.read_jsonl(run_id, "source_path_records.jsonl")
  26. walk_actions = runtime.read_jsonl(run_id, "walk_actions.jsonl")
  27. performance_rows = runtime.read_performance_feedback(run_id, policy_run_id)
  28. query_review = _build_query_review(search_clues)
  29. rule_review = _build_rule_review(decisions)
  30. performance_feedback = _build_performance_feedback_summary(performance_rows)
  31. productive_paths = _build_productive_paths(source_path_records)
  32. walk_review = _build_walk_review(walk_actions)
  33. v4_summary = _build_v4_summary(
  34. decisions,
  35. walk_actions,
  36. final_output,
  37. source_path_records,
  38. )
  39. search_clue_assets, search_clue_evidence = _build_search_clue_asset_promotions(
  40. run_id,
  41. policy_run_id,
  42. final_output,
  43. search_clues,
  44. decisions,
  45. discovered_content_items,
  46. source_path_records,
  47. performance_rows,
  48. )
  49. if search_clue_assets:
  50. runtime.write_search_clue_assets(search_clue_assets)
  51. runtime.write_search_clue_asset_evidence(search_clue_evidence)
  52. recommendations = _build_recommendations(
  53. final_output.get("summary", {}),
  54. query_review,
  55. rule_review["top_reject_reasons"],
  56. performance_feedback,
  57. )
  58. suggestions = [
  59. {"suggestion": item["suggested_action"], "basis": item["basis"]}
  60. for item in recommendations
  61. ]
  62. review = {
  63. "schema_version": RUNTIME_SCHEMA_VERSION,
  64. "run_id": run_id,
  65. "policy_run_id": policy_run_id,
  66. "review_id": review_id or f"review_{policy_run_id}",
  67. "review_status": "generated",
  68. "data_window": _build_data_window(run_id, policy_run_id, final_output),
  69. "summary": final_output["summary"],
  70. "metric_summary": _build_metric_summary(final_output, search_clues, decisions),
  71. "query_review": query_review,
  72. "rule_review": rule_review,
  73. "walk_review": walk_review,
  74. "asset_review": _build_asset_review(final_output, search_clue_assets),
  75. "performance_feedback": performance_feedback,
  76. "v4_summary": v4_summary,
  77. "recommendations": recommendations,
  78. "decision_distribution": rule_review["decision_distribution"],
  79. "effective_search_queries": [
  80. item["search_query"] for item in query_review["effective_queries"]
  81. ],
  82. "weak_search_queries": [
  83. item["search_query"] for item in query_review["review_queries"]
  84. ],
  85. "top_reject_reasons": rule_review["top_reject_reasons"],
  86. "productive_paths": productive_paths,
  87. "suggestions": suggestions,
  88. }
  89. review = with_raw_payload(review)
  90. runtime.write_json(run_id, "strategy_review.json", review)
  91. return review
  92. def _current_contract_search_clues(search_clues: list[dict[str, Any]]) -> list[dict[str, Any]]:
  93. return [
  94. clue
  95. for clue in search_clues
  96. if clue.get("search_query_effect_status") in CURRENT_QUERY_EFFECT_STATUSES
  97. ]
  98. def _current_contract_decisions(decisions: list[dict[str, Any]]) -> list[dict[str, Any]]:
  99. return [
  100. decision
  101. for decision in decisions
  102. if decision.get("decision_action") in CURRENT_DECISION_ACTIONS
  103. and decision.get("search_query_effect_status") in CURRENT_QUERY_EFFECT_STATUSES
  104. ]
  105. def _build_data_window(
  106. run_id: str,
  107. policy_run_id: str,
  108. final_output: dict[str, Any],
  109. ) -> dict[str, Any]:
  110. return {
  111. "scope": "single_run",
  112. "run_id": run_id,
  113. "policy_run_id": policy_run_id,
  114. "source_files": [
  115. "final_output.json",
  116. "search_clues.jsonl",
  117. "rule_decisions.jsonl",
  118. "discovered_content_items.jsonl",
  119. "source_path_records.jsonl",
  120. "walk_actions.jsonl",
  121. ],
  122. "policy_context": _policy_context(final_output),
  123. }
  124. def _policy_context(final_output: dict[str, Any]) -> dict[str, Any]:
  125. policy = final_output.get("policy") or {}
  126. walk_strategy = final_output.get("walk_strategy") or {}
  127. return {
  128. "policy_context_status": "available" if policy or walk_strategy else "missing",
  129. "strategy_version": policy.get("strategy_version"),
  130. "policy_bundle_id": policy.get("policy_bundle_id"),
  131. "rule_pack_id": policy.get("rule_pack_id"),
  132. "rule_pack_version": policy.get("rule_pack_version"),
  133. "walk_strategy_version": walk_strategy.get("walk_strategy_version"),
  134. "policy_bundle_hash": policy.get("policy_bundle_hash"),
  135. }
  136. def _build_metric_summary(
  137. final_output: dict[str, Any],
  138. search_clues: list[dict[str, Any]],
  139. decisions: list[dict[str, Any]],
  140. ) -> dict[str, Any]:
  141. summary = dict(final_output.get("summary") or {})
  142. status_counts = Counter(clue["search_query_effect_status"] for clue in search_clues)
  143. action_counts = Counter(decision["decision_action"] for decision in decisions)
  144. summary.update(
  145. {
  146. "search_query_count": len(search_clues),
  147. "success_query_count": status_counts["success"],
  148. "pending_query_count": status_counts["pending"],
  149. "failed_query_count": status_counts["failed"],
  150. "rule_blocked_query_count": status_counts["rule_blocked"],
  151. "pooled_decision_count": action_counts["ADD_TO_CONTENT_POOL"],
  152. "review_decision_count": action_counts["KEEP_CONTENT_FOR_REVIEW"],
  153. "technical_retry_decision_count": action_counts["TECHNICAL_RETRY_REQUIRED"],
  154. "rejected_decision_count": action_counts["REJECT_CONTENT"],
  155. }
  156. )
  157. return summary
  158. def _build_query_review(search_clues: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
  159. return {
  160. "effective_queries": _queries_with_status(search_clues, "success"),
  161. "review_queries": _queries_with_status(search_clues, "pending"),
  162. "failed_queries": _queries_with_status(search_clues, "failed"),
  163. "rule_blocked_queries": _queries_with_status(search_clues, "rule_blocked"),
  164. }
  165. def _queries_with_status(
  166. search_clues: list[dict[str, Any]],
  167. status: str,
  168. ) -> list[dict[str, Any]]:
  169. return [
  170. {
  171. "clue_id": clue.get("clue_id"),
  172. "search_query_id": clue.get("search_query_id"),
  173. "search_query": clue.get("search_query"),
  174. "search_query_effect_status": clue.get("search_query_effect_status"),
  175. "result_count": clue.get("result_count", 0),
  176. "pooled_content_count": clue.get("pooled_content_count", 0),
  177. "review_content_count": clue.get("review_content_count", 0),
  178. "rejected_content_count": clue.get("rejected_content_count", 0),
  179. }
  180. for clue in search_clues
  181. if clue.get("search_query_effect_status") == status
  182. ]
  183. def _build_rule_review(decisions: list[dict[str, Any]]) -> dict[str, Any]:
  184. reject_reasons = Counter(
  185. decision["decision_reason_code"]
  186. for decision in decisions
  187. if decision["decision_action"] == "REJECT_CONTENT"
  188. )
  189. return {
  190. "decision_distribution": dict(
  191. Counter(decision["decision_action"] for decision in decisions)
  192. ),
  193. "top_reject_reasons": [
  194. {"decision_reason_code": reason, "count": count}
  195. for reason, count in reject_reasons.most_common()
  196. ],
  197. "hard_gate_block_count": sum(
  198. 1
  199. for decision in decisions
  200. if decision.get("search_query_effect_status") == "rule_blocked"
  201. ),
  202. }
  203. def _build_walk_review(walk_actions: list[dict[str, Any]]) -> dict[str, Any]:
  204. status_counts = Counter(action.get("walk_status") for action in walk_actions)
  205. edge_counts = Counter(action.get("edge_id") for action in walk_actions)
  206. gate_review = _walk_gate_review(walk_actions)
  207. return {
  208. "walk_action_count": len(walk_actions),
  209. "walk_status_distribution": dict(status_counts),
  210. "edge_distribution": dict(edge_counts),
  211. "v4_gate_distribution": gate_review["v4_gate_distribution"],
  212. "deny_reasons": gate_review["deny_reasons"],
  213. }
  214. def _build_v4_summary(
  215. decisions: list[dict[str, Any]],
  216. walk_actions: list[dict[str, Any]],
  217. final_output: dict[str, Any],
  218. source_path_records: list[dict[str, Any]],
  219. ) -> dict[str, Any]:
  220. v4_decisions = [decision for decision in decisions if _is_v4_decision(decision)]
  221. walk_gate_review = _walk_gate_review(walk_actions)
  222. return {
  223. "schema_version": "v4_strategy_review_summary.v1",
  224. "summary_status": "available" if v4_decisions else "missing",
  225. "v4_decision_count": len(v4_decisions),
  226. "content_asset_count": len(final_output.get("content_assets") or []),
  227. "source_path_record_count": len(source_path_records),
  228. "decision_distribution": dict(
  229. Counter(decision.get("decision_action") for decision in v4_decisions)
  230. ),
  231. "score_buckets": _score_buckets(v4_decisions),
  232. "score_ranges": _score_ranges(v4_decisions),
  233. "missing_observable_fields": _missing_observable_field_counts(v4_decisions),
  234. "technical_retry": _technical_retry_summary(v4_decisions),
  235. "allow_walk_distribution": _allow_walk_distribution(v4_decisions),
  236. "walk_gate_review": walk_gate_review,
  237. }
  238. def _is_v4_decision(decision: dict[str, Any]) -> bool:
  239. scorecard = decision.get("scorecard") or {}
  240. return isinstance(scorecard, dict) and scorecard.get("schema_version") == "v4_scorecard.v1"
  241. def _score_bucket(decision: dict[str, Any]) -> str:
  242. if decision.get("decision_reason_code") == "v4_technical_retry_needed":
  243. return "technical_retry"
  244. action = decision.get("decision_action")
  245. if action == "ADD_TO_CONTENT_POOL":
  246. return "pool"
  247. if action == "KEEP_CONTENT_FOR_REVIEW":
  248. return "review"
  249. if action == "REJECT_CONTENT":
  250. return "reject"
  251. return "unknown"
  252. def _score_buckets(decisions: list[dict[str, Any]]) -> dict[str, int]:
  253. counts = Counter(_score_bucket(decision) for decision in decisions)
  254. return {
  255. "pool": counts["pool"],
  256. "review": counts["review"],
  257. "reject": counts["reject"],
  258. "technical_retry": counts["technical_retry"],
  259. "unknown": counts["unknown"],
  260. }
  261. def _score_ranges(decisions: list[dict[str, Any]]) -> dict[str, dict[str, float | int | None]]:
  262. return {
  263. "query_relevance_score": _range_summary(
  264. (decision.get("scorecard") or {}).get("query_relevance_score")
  265. for decision in decisions
  266. ),
  267. "platform_performance_score": _range_summary(
  268. (decision.get("scorecard") or {}).get("platform_performance_score")
  269. for decision in decisions
  270. ),
  271. "score": _range_summary(decision.get("score") for decision in decisions),
  272. }
  273. def _range_summary(values: Any) -> dict[str, float | int | None]:
  274. numeric_values = [
  275. float(value)
  276. for value in values
  277. if isinstance(value, (int, float)) and not isinstance(value, bool)
  278. ]
  279. if not numeric_values:
  280. return {"min": None, "max": None, "avg": None}
  281. return {
  282. "min": min(numeric_values),
  283. "max": max(numeric_values),
  284. "avg": round(sum(numeric_values) / len(numeric_values), 4),
  285. }
  286. def _missing_observable_field_counts(decisions: list[dict[str, Any]]) -> list[dict[str, Any]]:
  287. counts: Counter[tuple[str, str]] = Counter()
  288. for decision in decisions:
  289. scorecard = decision.get("scorecard") or {}
  290. for item in scorecard.get("missing_observable_fields") or []:
  291. if isinstance(item, dict):
  292. field = str(item.get("field") or "unknown")
  293. reason = str(item.get("reason") or "unknown")
  294. else:
  295. field = str(item)
  296. reason = "unknown"
  297. counts[(field, reason)] += 1
  298. return [
  299. {"field": field, "reason": reason, "count": count}
  300. for (field, reason), count in sorted(counts.items())
  301. ]
  302. def _technical_retry_summary(decisions: list[dict[str, Any]]) -> dict[str, Any]:
  303. retry_decisions = [
  304. decision
  305. for decision in decisions
  306. if decision.get("decision_reason_code") == "v4_technical_retry_needed"
  307. ]
  308. failure_types: Counter[str] = Counter()
  309. for decision in retry_decisions:
  310. scorecard = decision.get("scorecard") or {}
  311. replay_data = decision.get("decision_replay_data") or {}
  312. failure_type = (
  313. scorecard.get("failure_type")
  314. or replay_data.get("failure_type")
  315. or scorecard.get("final_status")
  316. or "unknown"
  317. )
  318. failure_types[str(failure_type)] += 1
  319. return {
  320. "count": len(retry_decisions),
  321. "failure_types": dict(failure_types),
  322. }
  323. def _allow_walk_distribution(decisions: list[dict[str, Any]]) -> dict[str, int]:
  324. counts = {"allowed": 0, "denied": 0, "missing": 0}
  325. for decision in decisions:
  326. replay_data = decision.get("decision_replay_data") or {}
  327. if replay_data.get("allow_walk") is True:
  328. counts["allowed"] += 1
  329. elif replay_data.get("allow_walk") is False:
  330. counts["denied"] += 1
  331. else:
  332. counts["missing"] += 1
  333. return counts
  334. def _walk_gate_review(walk_actions: list[dict[str, Any]]) -> dict[str, Any]:
  335. gate_counts = {"allowed": 0, "denied": 0, "missing": 0}
  336. deny_reasons: Counter[str] = Counter()
  337. allowed_success_count = 0
  338. denied_skip_count = 0
  339. for action in walk_actions:
  340. raw_payload = action.get("raw_payload") or {}
  341. status = raw_payload.get("walk_gate_status")
  342. if status == "allowed" or raw_payload.get("allow_walk") is True:
  343. gate_counts["allowed"] += 1
  344. if action.get("walk_status") == "success":
  345. allowed_success_count += 1
  346. elif status == "denied" or raw_payload.get("allow_walk") is False:
  347. gate_counts["denied"] += 1
  348. reason = (
  349. raw_payload.get("walk_gate_reason_code")
  350. or action.get("reason_code")
  351. or "unknown"
  352. )
  353. deny_reasons[str(reason)] += 1
  354. if action.get("walk_status") in {"skipped", "failed", "rule_blocked"}:
  355. denied_skip_count += 1
  356. else:
  357. gate_counts["missing"] += 1
  358. return {
  359. "walk_action_count": len(walk_actions),
  360. "v4_gate_distribution": gate_counts,
  361. "allowed_success_count": allowed_success_count,
  362. "denied_skip_count": denied_skip_count,
  363. "deny_reasons": [
  364. {"reason_code": reason, "count": count}
  365. for reason, count in deny_reasons.most_common()
  366. ],
  367. }
  368. def _build_asset_review(
  369. final_output: dict[str, Any],
  370. search_clue_assets: list[dict[str, Any]],
  371. ) -> dict[str, Any]:
  372. return {
  373. "content_asset_count": len(final_output.get("content_assets") or []),
  374. "author_asset_count": len(final_output.get("author_assets") or []),
  375. "search_clue_assets": {
  376. "promoted_count": len(search_clue_assets),
  377. "assets": [
  378. {
  379. "search_clue_asset_id": asset["search_clue_asset_id"],
  380. "platform": asset["platform"],
  381. "clue_type": asset["clue_type"],
  382. "normalized_clue_text": asset["normalized_clue_text"],
  383. "can_seed_next_run": asset["can_seed_next_run"],
  384. }
  385. for asset in search_clue_assets
  386. ],
  387. },
  388. }
  389. def _build_search_clue_asset_promotions(
  390. run_id: str,
  391. policy_run_id: str,
  392. final_output: dict[str, Any],
  393. search_clues: list[dict[str, Any]],
  394. decisions: list[dict[str, Any]],
  395. discovered_content_items: list[dict[str, Any]],
  396. source_path_records: list[dict[str, Any]],
  397. performance_rows: list[dict[str, Any]],
  398. ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
  399. content_assets = final_output.get("content_assets") or []
  400. if not content_assets:
  401. return [], []
  402. decisions_by_id = {
  403. decision["decision_id"]: decision
  404. for decision in decisions
  405. if decision.get("decision_action") == "ADD_TO_CONTENT_POOL"
  406. }
  407. source_paths_by_id = {
  408. path["source_path_record_id"]: path
  409. for path in source_path_records
  410. if path.get("source_path_type") == "decision_to_asset"
  411. }
  412. discovered_by_content_id = {
  413. item["platform_content_id"]: item
  414. for item in discovered_content_items
  415. if item.get("platform_content_id")
  416. }
  417. discovered_by_discovery_id = {
  418. item["content_discovery_id"]: item
  419. for item in discovered_content_items
  420. if item.get("content_discovery_id")
  421. }
  422. feedback_refs = [
  423. row["feedback_id"]
  424. for row in performance_rows
  425. if row.get("feedback_id")
  426. ]
  427. assets: list[dict[str, Any]] = []
  428. evidence: list[dict[str, Any]] = []
  429. for clue in search_clues:
  430. if clue.get("search_query_effect_status") != "success":
  431. continue
  432. if int(clue.get("pooled_content_count") or 0) <= 0:
  433. continue
  434. normalized = _normalize_clue_text(clue.get("search_query"))
  435. if not normalized:
  436. continue
  437. matched = _matched_asset_lineage_for_clue(
  438. clue,
  439. content_assets,
  440. decisions_by_id,
  441. source_paths_by_id,
  442. discovered_by_content_id,
  443. discovered_by_discovery_id,
  444. )
  445. if not matched["source_path_record_ids"] or not matched["decision_ids"]:
  446. continue
  447. platform = matched["platform"] or "douyin"
  448. asset_id = _stable_id("search_clue_asset", platform, "search_query", normalized)
  449. asset = {
  450. "search_clue_asset_id": asset_id,
  451. "platform": platform,
  452. "clue_type": "search_query",
  453. "normalized_clue_text": normalized,
  454. "display_clue_text": clue.get("search_query"),
  455. "promotion_status": "promoted",
  456. "reusable_priority": int(clue.get("pooled_content_count") or 0),
  457. "can_seed_next_run": 1,
  458. "first_seen_run_id": run_id,
  459. "first_seen_policy_run_id": policy_run_id,
  460. "summary_metrics": {
  461. "result_count": clue.get("result_count", 0),
  462. "pooled_content_count": clue.get("pooled_content_count", 0),
  463. "review_content_count": clue.get("review_content_count", 0),
  464. "failed_content_count": clue.get("rejected_content_count", 0),
  465. "matched_content_asset_count": len(matched["content_asset_ids"]),
  466. },
  467. }
  468. asset["raw_payload"] = {
  469. **asset,
  470. "promotion_reason": "success_search_clue_with_content_asset_lineage",
  471. }
  472. assets.append(asset)
  473. evidence_row = {
  474. "evidence_id": _stable_id("search_clue_evidence", run_id, policy_run_id, clue["clue_id"]),
  475. "search_clue_asset_id": asset_id,
  476. "run_id": run_id,
  477. "policy_run_id": policy_run_id,
  478. "clue_id": clue["clue_id"],
  479. "search_query_id": clue.get("search_query_id"),
  480. "pooled_content_count": int(clue.get("pooled_content_count") or 0),
  481. "review_content_count": int(clue.get("review_content_count") or 0),
  482. "failed_content_count": int(clue.get("rejected_content_count") or 0),
  483. "source_path_record_ids": matched["source_path_record_ids"],
  484. "decision_ids": matched["decision_ids"],
  485. "performance_feedback_refs": feedback_refs,
  486. }
  487. evidence_row["raw_payload"] = dict(evidence_row)
  488. evidence.append(evidence_row)
  489. return assets, evidence
  490. def _matched_asset_lineage_for_clue(
  491. clue: dict[str, Any],
  492. content_assets: list[dict[str, Any]],
  493. decisions_by_id: dict[str, dict[str, Any]],
  494. source_paths_by_id: dict[str, dict[str, Any]],
  495. discovered_by_content_id: dict[str, dict[str, Any]],
  496. discovered_by_discovery_id: dict[str, dict[str, Any]],
  497. ) -> dict[str, Any]:
  498. clue_query_id = clue.get("search_query_id")
  499. result = {
  500. "platform": None,
  501. "content_asset_ids": [],
  502. "source_path_record_ids": [],
  503. "decision_ids": [],
  504. }
  505. for asset in content_assets:
  506. decision_id = asset.get("decision_id") or next(iter(asset.get("decision_ids", [])), None)
  507. decision = decisions_by_id.get(decision_id)
  508. if not decision:
  509. continue
  510. if clue_query_id not in _asset_query_ids(asset, decision, discovered_by_content_id, discovered_by_discovery_id):
  511. continue
  512. path_ids = _matched_decision_to_asset_path_ids(asset, decision, source_paths_by_id)
  513. if not path_ids:
  514. continue
  515. result["platform"] = result["platform"] or asset.get("platform")
  516. result["content_asset_ids"].append(
  517. asset.get("content_asset_id") or asset.get("platform_content_id")
  518. )
  519. result["source_path_record_ids"].extend(path_ids)
  520. result["decision_ids"].append(decision_id)
  521. return {
  522. **result,
  523. "content_asset_ids": sorted({value for value in result["content_asset_ids"] if value}),
  524. "source_path_record_ids": sorted(set(result["source_path_record_ids"])),
  525. "decision_ids": sorted(set(result["decision_ids"])),
  526. }
  527. def _asset_query_ids(
  528. asset: dict[str, Any],
  529. decision: dict[str, Any],
  530. discovered_by_content_id: dict[str, dict[str, Any]],
  531. discovered_by_discovery_id: dict[str, dict[str, Any]],
  532. ) -> set[str]:
  533. item = (
  534. discovered_by_discovery_id.get(asset.get("content_discovery_id"))
  535. or discovered_by_content_id.get(asset.get("platform_content_id"))
  536. or {}
  537. )
  538. evidence = decision.get("source_evidence") or {}
  539. asset_evidence = asset.get("source_evidence") or {}
  540. values = {
  541. item.get("search_query_id"),
  542. evidence.get("search_query_id"),
  543. asset_evidence.get("search_query_id"),
  544. }
  545. values.update(evidence.get("matched_search_query_ids") or [])
  546. values.update(asset_evidence.get("matched_search_query_ids") or [])
  547. return {value for value in values if value}
  548. def _matched_decision_to_asset_path_ids(
  549. asset: dict[str, Any],
  550. decision: dict[str, Any],
  551. source_paths_by_id: dict[str, dict[str, Any]],
  552. ) -> list[str]:
  553. asset_path_ids = set(asset.get("source_path_record_ids") or [])
  554. asset_node_ids = {
  555. asset.get("content_asset_id"),
  556. asset.get("platform_content_id"),
  557. }
  558. matched = []
  559. for path_id in asset_path_ids:
  560. path = source_paths_by_id.get(path_id)
  561. if not path:
  562. continue
  563. if path.get("decision_id") != decision.get("decision_id"):
  564. continue
  565. if path.get("from_node_id") != decision.get("decision_id"):
  566. continue
  567. if path.get("to_node_id") not in asset_node_ids:
  568. continue
  569. matched.append(path_id)
  570. return matched
  571. def _normalize_clue_text(value: Any) -> str:
  572. if not isinstance(value, str):
  573. return ""
  574. return " ".join(value.strip().lower().split())
  575. def _stable_id(prefix: str, *parts: str) -> str:
  576. digest = hashlib.sha1("|".join(parts).encode("utf-8")).hexdigest()[:16]
  577. return f"{prefix}_{digest}"
  578. def _build_performance_feedback_summary(rows: list[dict[str, Any]]) -> dict[str, Any]:
  579. if not rows:
  580. return {
  581. "performance_feedback_status": "missing",
  582. "feedback_source": "none",
  583. "feedback_count": 0,
  584. }
  585. status_counts = Counter(row.get("feedback_status", "available") for row in rows)
  586. return {
  587. "performance_feedback_status": "available",
  588. "feedback_count": len(rows),
  589. "feedback_status_distribution": dict(status_counts),
  590. "average_completion_rate": _average_metric(rows, "completion_rate"),
  591. "average_share_rate": _average_metric(rows, "share_rate"),
  592. "average_watch_seconds": _average_metric(rows, "average_watch_seconds"),
  593. }
  594. def _average_metric(rows: list[dict[str, Any]], key: str) -> float | None:
  595. values = [row.get(key) for row in rows if row.get(key) is not None]
  596. if not values:
  597. return None
  598. return sum(float(value) for value in values) / len(values)
  599. def _build_productive_paths(source_path_records: list[dict[str, Any]]) -> list[dict[str, Any]]:
  600. return [
  601. {
  602. "source_path_record_id": path["source_path_record_id"],
  603. "from": f"{path['from_node_type']}:{path['from_node_id']}",
  604. "to": f"{path['to_node_type']}:{path['to_node_id']}",
  605. }
  606. for path in source_path_records
  607. if path["source_path_type"] == "decision_to_asset"
  608. ]
  609. def _build_recommendations(
  610. summary: dict[str, Any],
  611. query_review: dict[str, list[dict[str, Any]]],
  612. reject_reasons: list[dict[str, Any]],
  613. performance_feedback: dict[str, Any],
  614. ) -> list[dict[str, Any]]:
  615. recommendations: list[dict[str, Any]] = []
  616. if summary.get("pooled_content_count", 0) > 0:
  617. recommendations.append(
  618. _recommendation(
  619. "rec_query_keep_success",
  620. "query_strategy",
  621. "search_query",
  622. "success_queries",
  623. "keep",
  624. "本次已有搜索词产生 ADD_TO_CONTENT_POOL 结果。",
  625. [
  626. f"search_clues.jsonl:{item['clue_id']}"
  627. for item in query_review["effective_queries"]
  628. if item.get("clue_id")
  629. ],
  630. )
  631. )
  632. if query_review["review_queries"]:
  633. recommendations.append(
  634. _recommendation(
  635. "rec_query_review_budget",
  636. "query_strategy",
  637. "search_query",
  638. "pending_queries",
  639. "review_with_small_budget",
  640. "存在只产生 KEEP_CONTENT_FOR_REVIEW 的搜索词。",
  641. [
  642. f"search_clues.jsonl:{item['clue_id']}"
  643. for item in query_review["review_queries"]
  644. if item.get("clue_id")
  645. ],
  646. )
  647. )
  648. if reject_reasons:
  649. reason = reject_reasons[0]["decision_reason_code"]
  650. recommendations.append(
  651. _recommendation(
  652. "rec_rule_fix_reject_reason",
  653. "rule_evidence",
  654. "decision_reason_code",
  655. reason,
  656. "inspect_reject_evidence",
  657. f"最高频淘汰原因是 {reason}。",
  658. [f"rule_decisions.jsonl:{reason}"],
  659. )
  660. )
  661. if performance_feedback["performance_feedback_status"] == "missing":
  662. recommendations.append(
  663. _recommendation(
  664. "rec_performance_feedback_missing",
  665. "performance_feedback",
  666. "run",
  667. "performance_feedback",
  668. "keep_feedback_empty",
  669. "当前没有后验表现反馈,先不据此修改策略。",
  670. [],
  671. )
  672. )
  673. else:
  674. recommendations.append(
  675. _recommendation(
  676. "rec_performance_feedback_review",
  677. "performance_feedback",
  678. "run",
  679. "performance_feedback",
  680. "review_feedback_before_strategy_change",
  681. "已有后验表现反馈,只作为下一轮建议证据,不覆盖本轮规则判断。",
  682. ["performance_feedback"],
  683. )
  684. )
  685. return recommendations
  686. def _recommendation(
  687. recommendation_id: str,
  688. recommendation_type: str,
  689. target_type: str,
  690. target_id: str,
  691. suggested_action: str,
  692. basis: str,
  693. evidence_refs: list[str],
  694. ) -> dict[str, Any]:
  695. return {
  696. "recommendation_id": recommendation_id,
  697. "recommendation_type": recommendation_type,
  698. "target_type": target_type,
  699. "target_id": target_id,
  700. "suggested_action": suggested_action,
  701. "basis": basis,
  702. "evidence_refs": evidence_refs,
  703. "requires_human_approval": True,
  704. }
  705. def _build_suggestions(
  706. summary: dict[str, Any],
  707. search_clues: list[dict[str, Any]],
  708. reject_reasons: Counter[str],
  709. ) -> list[dict[str, str]]:
  710. query_review = _build_query_review(_current_contract_search_clues(search_clues))
  711. recommendations = _build_recommendations(
  712. summary,
  713. query_review,
  714. [
  715. {"decision_reason_code": reason, "count": count}
  716. for reason, count in reject_reasons.most_common()
  717. ],
  718. _build_performance_feedback_summary([]),
  719. )
  720. return [
  721. {"suggestion": item["suggested_action"], "basis": item["basis"]}
  722. for item in recommendations
  723. ]