test_find_agent_v2.py 51 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450
  1. from __future__ import annotations
  2. import json
  3. from dataclasses import replace
  4. from pathlib import Path
  5. from types import SimpleNamespace
  6. import pytest
  7. from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel
  8. from langchain_core.messages import AIMessage, ToolMessage
  9. from find_agent_v2.graph import FindAgentRoundGraph
  10. from find_agent_v2 import tools as find_agent_tools
  11. from find_agent_v2 import runtime as find_agent_runtime
  12. from find_agent_v2.agent import decide_continued_exploration
  13. from find_agent_v2.runtime import (
  14. DelegateArgs,
  15. FindAgentNodeHost,
  16. _events,
  17. _message_dict,
  18. _usage,
  19. )
  20. from find_agent_v2.demand_context import (
  21. V2DemandContext,
  22. V2ReferencePoint,
  23. V2ReferenceVideo,
  24. _latest_demand_grade_query,
  25. _points_from_expansions,
  26. build_v2_user_input,
  27. )
  28. from find_agent_v2.gates import build_rule_snapshot, evaluate_candidate_gate
  29. from find_agent_v2.models import (
  30. FindAgentV2Candidate,
  31. FindAgentV2Evidence,
  32. FindAgentV2Round,
  33. FindAgentV2Run,
  34. FindAgentV2Search,
  35. )
  36. from find_agent_v2.observability import (
  37. GRAPH_SPEC,
  38. MODULE_TITLES,
  39. OBAGENT_AGENT,
  40. OBAGENT_PROJECT,
  41. OBAGENT_ROUND_ANCHOR,
  42. NullObserver,
  43. InputSlot,
  44. _ModuleHandle,
  45. )
  46. from find_agent_v2.prompts import COMMON_RULES, EVALUATOR_PROMPT
  47. from find_agent_v2.providers import _normalize_search_item, normalize_age_pair
  48. from find_agent_v2.state import (
  49. DemandBrief,
  50. DiscoverySnapshot,
  51. EvaluationBrief,
  52. EvidenceAssignment,
  53. ExecutionPlan,
  54. FindAgentState,
  55. NodeRun,
  56. PlanningDecision,
  57. SearchTask,
  58. SearchAssignment,
  59. SupervisorDecision,
  60. )
  61. from find_agent_v2.service import _fails_search_share_gate
  62. from find_agent_v2.tools import (
  63. CandidateEvaluation,
  64. EVALUATION_TOOLS,
  65. EVIDENCE_TOOLS,
  66. REPORT_TOOLS,
  67. SEARCH_TOOLS,
  68. bound_candidate_tools,
  69. normalize_evaluation_items,
  70. )
  71. def _names(functions) -> set[str]:
  72. return {getattr(fn, "_tool_name", fn.__name__) for fn in functions}
  73. def test_video_tool_error_code_is_observed_as_failed() -> None:
  74. message = ToolMessage(
  75. content=json.dumps({
  76. "success": False,
  77. "error": "OSS timeout",
  78. "error_code": "video_understanding_timeout",
  79. }),
  80. name="understand_candidate_video_30s_v2",
  81. tool_call_id="call-1",
  82. )
  83. assert _message_dict(message)["status"] == "error"
  84. assert _events([message])[0]["status"] == "error"
  85. assert _events([message])[0]["error_code"] == "video_understanding_timeout"
  86. def test_runtime_usage_reads_openrouter_token_usage_cost() -> None:
  87. message = AIMessage(
  88. content="done",
  89. usage_metadata={"input_tokens": 10, "output_tokens": 2, "total_tokens": 12},
  90. response_metadata={"token_usage": {"cost": 0.00123}},
  91. )
  92. assert _usage([message]) == {
  93. "input_tokens": 10,
  94. "output_tokens": 2,
  95. "total_tokens": 12,
  96. "cost": 0.00123,
  97. }
  98. @pytest.mark.asyncio
  99. async def test_host_direct_search_and_evidence_paths_use_no_llm(monkeypatch) -> None:
  100. calls: list[tuple[str, object]] = []
  101. async def fake_search(**kwargs):
  102. calls.append(("search", kwargs["searches"]))
  103. return '{"run_id":"run","searches":[]}'
  104. async def fake_details(**kwargs):
  105. calls.append(("detail", kwargs["candidate_ids"]))
  106. return '{"run_id":"run","success_count":2,"failed_count":0}'
  107. monkeypatch.setattr(find_agent_runtime, "search_videos_v2", fake_search)
  108. monkeypatch.setattr(find_agent_runtime, "fetch_candidate_details_v2", fake_details)
  109. host = FindAgentNodeHost(observer=NullObserver())
  110. search_run = await host.run_search_assignment(
  111. round_index=1,
  112. assignment=SearchAssignment(
  113. run_id="run",
  114. round_index=1,
  115. tasks=[SearchTask(
  116. task_id="task-1", keyword="日本间谍", query_reason="测试",
  117. )],
  118. ),
  119. system_prompt="search",
  120. user_content="{}",
  121. )
  122. evidence_run = await host.run_evidence_assignment(
  123. round_index=1,
  124. assignment=EvidenceAssignment(
  125. run_id="run",
  126. round_index=1,
  127. candidate_ids=[101, 102],
  128. evidence_type="detail",
  129. candidates=[],
  130. ),
  131. system_prompt="evidence",
  132. user_content="{}",
  133. )
  134. assert calls == [
  135. ("search", [{
  136. "task_id": "task-1",
  137. "keyword": "日本间谍",
  138. "query_reason": "测试",
  139. "source_type": "mixed",
  140. "provider": None,
  141. "max_pages": 1,
  142. "coverage_targets": [],
  143. }]),
  144. ("detail", [101, 102]),
  145. ]
  146. assert (search_run.iterations, evidence_run.iterations) == (0, 0)
  147. assert (search_run.tool_calls_made, evidence_run.tool_calls_made) == (1, 1)
  148. def _evaluation(**overrides):
  149. value = {
  150. "candidate_id": 11,
  151. "relevance_score": 0.9,
  152. "elder_score": 0.8,
  153. "share_score": 0.7,
  154. "value_score": 0.85,
  155. "decision_bucket": "primary",
  156. "decision_reason": "证据充分且满足推荐要求",
  157. }
  158. value.update(overrides)
  159. return value
  160. def test_evaluation_items_resolve_aweme_id_only_inside_worker_shard() -> None:
  161. allowed = [
  162. {"candidate_id": 11, "aweme_id": "video-11"},
  163. {"candidate_id": 12, "aweme_id": "video-12"},
  164. ]
  165. normalized = normalize_evaluation_items([
  166. _evaluation(candidate_id=None, aweme_id="video-11"),
  167. _evaluation(candidate_id=12, aweme_id=None, decision_bucket="rejected"),
  168. ], allowed_candidates=allowed)
  169. assert [item["candidate_id"] for item in normalized] == [11, 12]
  170. assert all("aweme_id" not in item for item in normalized)
  171. @pytest.mark.parametrize("items, message", [
  172. ([_evaluation()], "缺少 candidate_ids=[12]"),
  173. ([_evaluation(), _evaluation()], "不能重复评估"),
  174. ([_evaluation(candidate_id=99), _evaluation(candidate_id=12)], "不属于当前 Worker 分片"),
  175. ])
  176. def test_evaluation_items_fail_fast_on_incomplete_duplicate_or_foreign_items(items, message) -> None:
  177. allowed = [
  178. {"candidate_id": 11, "aweme_id": "video-11"},
  179. {"candidate_id": 12, "aweme_id": "video-12"},
  180. ]
  181. with pytest.raises(ValueError, match=message.replace("[", r"\[").replace("]", r"\]")):
  182. normalize_evaluation_items(items, allowed_candidates=allowed)
  183. def test_candidate_evaluation_schema_rejects_missing_identity() -> None:
  184. payload = _evaluation(candidate_id=None)
  185. with pytest.raises(ValueError, match="至少提供一个"):
  186. CandidateEvaluation.model_validate(payload)
  187. def test_v2_orm_uses_only_new_table_namespace() -> None:
  188. assert {
  189. FindAgentV2Run.__tablename__,
  190. FindAgentV2Round.__tablename__,
  191. FindAgentV2Search.__tablename__,
  192. FindAgentV2Candidate.__tablename__,
  193. FindAgentV2Evidence.__tablename__,
  194. } == {
  195. "find_agent_v2_run",
  196. "find_agent_v2_round",
  197. "find_agent_v2_search",
  198. "find_agent_v2_candidate",
  199. "find_agent_v2_evidence",
  200. }
  201. assert "obagent_run_uid" in FindAgentV2Run.__table__.columns
  202. assert {"input_tokens", "output_tokens", "total_tokens", "cost_usd"} <= {
  203. column.name for column in FindAgentV2Run.__table__.columns
  204. }
  205. assert not any(table.foreign_key_constraints for table in (
  206. FindAgentV2Run.__table__,
  207. FindAgentV2Round.__table__,
  208. FindAgentV2Search.__table__,
  209. FindAgentV2Candidate.__table__,
  210. FindAgentV2Evidence.__table__,
  211. ))
  212. def test_v2_package_has_no_legacy_business_imports() -> None:
  213. package = Path(__file__).parents[2] / "find_agent_v2"
  214. source = "\n".join(path.read_text(encoding="utf-8") for path in package.glob("*.py"))
  215. assert "agents.find_agent" not in source
  216. assert "video_discovery_gates" not in source
  217. assert "services.video_discovery" not in source
  218. assert "system_prompt.md" not in source
  219. def test_project_orm_defines_no_database_foreign_keys() -> None:
  220. import supply_infra.db.models # noqa: F401
  221. from supply_infra.db.base import Base
  222. assert not {
  223. table.name: sorted(fk.name or "<unnamed>" for fk in table.foreign_key_constraints)
  224. for table in Base.metadata.tables.values()
  225. if table.foreign_key_constraints
  226. }
  227. def test_v2_owns_age_portrait_normalization() -> None:
  228. normalized = normalize_age_pair(
  229. {"年龄": {"50-": {"percentage": "35%", "preference": 120}}},
  230. {"年龄": {"50岁以上": {"percentage": 0.25, "preference": 110}}},
  231. )
  232. assert normalized["content"]["older_ratio"] == 0.35
  233. assert normalized["account"]["older_ratio"] == 0.25
  234. assert normalized["consistency"] == "aligned"
  235. def test_search_metrics_preserve_missing_share_count_and_real_zero() -> None:
  236. missing = _normalize_search_item({"aweme_id": "missing", "statistics": {}})
  237. zero = _normalize_search_item({
  238. "aweme_id": "zero", "statistics": {"share_count": 0},
  239. })
  240. assert missing is not None and missing["statistics"]["share_count"] is None
  241. assert zero is not None and zero["statistics"]["share_count"] == 0
  242. @pytest.mark.parametrize(
  243. "provider,share_count,expected",
  244. [
  245. ("tikhub", 999, True),
  246. ("tikhub", 1000, False),
  247. ("tikhub", None, False),
  248. ("internal_keyword", 0, True),
  249. ("internal_keyword", 999, True),
  250. ("internal_keyword", 1000, False),
  251. ("internal_keyword", None, False),
  252. ],
  253. )
  254. def test_search_share_gate_applies_to_explicit_metrics_from_all_providers(
  255. provider, share_count, expected,
  256. ) -> None:
  257. assert _fails_search_share_gate(provider, share_count, 1000) is expected
  258. def test_v2_owns_primary_candidate_gate() -> None:
  259. rules = build_rule_snapshot()
  260. candidate = {
  261. "title": "适合父母的实用生活建议",
  262. "publish_at": rules["current_datetime"],
  263. "duration_seconds": 60,
  264. "share_count": 2000,
  265. "content_50_plus_ratio": 0.35,
  266. "account_50_plus_ratio": 0.25,
  267. "relevance_score": 0.8,
  268. "elder_score": 0.8,
  269. "share_score": 0.8,
  270. "value_score": 0.8,
  271. }
  272. result = evaluate_candidate_gate(candidate, rules)
  273. assert result["status"] == "pass"
  274. assert result["primary_eligible"] is True
  275. def test_v2_gate_rejects_explicitly_low_evidence() -> None:
  276. rules = build_rule_snapshot()
  277. candidate = {
  278. "title": "普通视频",
  279. "publish_at": rules["current_datetime"],
  280. "duration_seconds": 10,
  281. "share_count": 3,
  282. "content_50_plus_ratio": 0.01,
  283. "account_50_plus_ratio": 0.02,
  284. }
  285. result = evaluate_candidate_gate(candidate, rules)
  286. assert result["status"] == "fail"
  287. assert {"DURATION_TOO_SHORT", "SHARE_COUNT_TOO_LOW", "PORTRAIT_50_PLUS_TOO_LOW"} <= set(
  288. result["failed_reason_codes"]
  289. )
  290. def test_v2_gate_ignores_publish_time_but_rejects_missing_elder_portrait() -> None:
  291. rules = build_rule_snapshot()
  292. candidate = {
  293. "title": "高分但缺少硬性证据的视频",
  294. "publish_at": None,
  295. "duration_seconds": 60,
  296. "share_count": 2000,
  297. "content_50_plus_ratio": None,
  298. "account_50_plus_ratio": None,
  299. "relevance_score": 0.99,
  300. "elder_score": 0.99,
  301. "share_score": 0.99,
  302. "value_score": 0.99,
  303. }
  304. result = evaluate_candidate_gate(candidate, rules)
  305. assert result["status"] == "fail"
  306. assert result["failed_reason_codes"] == ["CONTENT_PORTRAIT_MISSING"]
  307. assert all(check["name"] != "temporal" for check in result["checks"])
  308. @pytest.mark.parametrize(
  309. "content_ratio,account_ratio",
  310. [(0.30, None), (None, 0.30), (0.05, 0.30), (0.30, 0.05)],
  311. )
  312. def test_v2_gate_accepts_either_elder_portrait_side(content_ratio, account_ratio) -> None:
  313. rules = build_rule_snapshot()
  314. result = evaluate_candidate_gate({
  315. "title": "符合要求的视频",
  316. "publish_at": rules["current_datetime"],
  317. "duration_seconds": 60,
  318. "share_count": 2000,
  319. "content_50_plus_ratio": content_ratio,
  320. "account_50_plus_ratio": account_ratio,
  321. }, rules)
  322. assert result["status"] == "pass"
  323. assert result["primary_eligible"] is True
  324. def test_v2_gate_does_not_compensate_missing_duration_or_share_count() -> None:
  325. rules = build_rule_snapshot()
  326. result = evaluate_candidate_gate({
  327. "title": "其他信号很强但硬指标缺失",
  328. "publish_at": rules["current_datetime"],
  329. "duration_seconds": None,
  330. "share_count": None,
  331. "content_50_plus_ratio": 0.30,
  332. "like_count": 100000,
  333. "play_count": 1000000,
  334. "relevance_score": 0.99,
  335. "elder_score": 0.99,
  336. "share_score": 0.99,
  337. "value_score": 0.99,
  338. }, rules)
  339. assert result["status"] == "fail"
  340. assert {"DURATION_UNKNOWN", "SHARE_COUNT_UNKNOWN"} <= set(
  341. result["failed_reason_codes"]
  342. )
  343. def test_v2_context_deduplicates_expansion_points() -> None:
  344. rows = [
  345. SimpleNamespace(
  346. video_id="v1", point_type="purpose", expanded_text="照顾父母",
  347. point_desc="描述一",
  348. ),
  349. SimpleNamespace(
  350. video_id="v1", point_type="purpose", expanded_text="照顾父母",
  351. point_desc="重复描述",
  352. ),
  353. SimpleNamespace(
  354. video_id="v1", point_type="invalid", expanded_text="忽略",
  355. point_desc=None,
  356. ),
  357. ]
  358. order, points = _points_from_expansions(rows)
  359. assert order == ["v1"]
  360. assert len(points["v1"]) == 1
  361. assert points["v1"][0].point == "照顾父母"
  362. def test_v2_context_builds_self_contained_user_input() -> None:
  363. context = V2DemandContext(
  364. biz_dt="20260811",
  365. demand_grade_id=123,
  366. demand_name="测试需求",
  367. grade="S",
  368. score=95.0,
  369. videos=[V2ReferenceVideo(
  370. video_id="video-1",
  371. title="参考视频",
  372. points=[V2ReferencePoint("关键内容", "key", "关键描述")],
  373. )],
  374. )
  375. raw = build_v2_user_input(
  376. context,
  377. run_id="v2-test-run",
  378. rules={"rule_version": "v2-test"},
  379. )
  380. assert '"run_id": "v2-test-run"' in raw
  381. assert '"demand_grade_id": 123' in raw
  382. assert '"reference_videos"' in raw
  383. assert '"关键内容"' in raw
  384. def test_demand_word_lookup_uses_exact_name_and_newest_record_order() -> None:
  385. sql = str(
  386. _latest_demand_grade_query("照顾父母").compile(
  387. compile_kwargs={"literal_binds": True},
  388. )
  389. ).lower()
  390. assert "demand_grade.demand_name = '照顾父母'" in sql
  391. assert "order by demand_grade.biz_dt desc, demand_grade.create_time desc, demand_grade.id desc" in sql
  392. def test_obagent_identity_and_round_structure_are_stable() -> None:
  393. assert OBAGENT_PROJECT == "find_agent_v2"
  394. assert OBAGENT_AGENT == "find_agent_v2"
  395. assert OBAGENT_ROUND_ANCHOR == {"in": "run", "on": ["graph"]}
  396. assert [node["key"] for node in GRAPH_SPEC["nodes"]] == [
  397. "supervisor", "search", "evidence", "evaluator",
  398. ]
  399. assert set(MODULE_TITLES) == {"supervisor", "search", "evidence", "evaluator", "report"}
  400. def test_round_graph_is_a_compiled_langgraph() -> None:
  401. service = _FakeService(pending_after_search=0)
  402. graph = FindAgentRoundGraph(service=service, runner=_FakeRunner(service))
  403. drawable = graph.app.get_graph()
  404. assert {"supervisor", "search", "evidence", "evaluator"} <= set(drawable.nodes)
  405. assert graph.obagent_spec.get("nodes")
  406. def test_runtime_exposes_bounded_delegate_schema() -> None:
  407. field = DelegateArgs.model_fields["requests"]
  408. assert field.metadata
  409. assert FindAgentNodeHost.__module__ == "find_agent_v2.runtime"
  410. def test_runtime_usage_can_be_reset_between_resume_attempts() -> None:
  411. host = FindAgentNodeHost(observer=NullObserver())
  412. host.usage["total_tokens"] = 99
  413. host.reset_usage()
  414. assert host.usage == {
  415. "input_tokens": 0,
  416. "output_tokens": 0,
  417. "total_tokens": 0,
  418. "cost": 0.0,
  419. }
  420. @pytest.mark.asyncio
  421. async def test_langchain_runtime_runs_without_network(monkeypatch) -> None:
  422. host = FindAgentNodeHost(observer=NullObserver())
  423. fake_model = FakeMessagesListChatModel(responses=[AIMessage(content="done")])
  424. monkeypatch.setattr(host, "_model", lambda _role: fake_model)
  425. result = await host.run_node(
  426. node="supervisor",
  427. round_index=1,
  428. system_prompt="plan",
  429. user_content="task",
  430. tools=(),
  431. max_iterations=2,
  432. allow_delegation=False,
  433. )
  434. assert result.content == "done"
  435. assert result.iterations == 1
  436. assert result.tool_calls_made == 0
  437. class _FakeSearchService:
  438. def __init__(self) -> None:
  439. self.saved: list[dict] = []
  440. def save_search(self, **kwargs):
  441. self.saved.append(kwargs)
  442. results = list((kwargs.get("payload") or {}).get("search_results") or [])
  443. return {
  444. "search_id": len(self.saved),
  445. "new_candidate_count": len(results),
  446. "result_count": len(results),
  447. "share_gate_rejected_count": 0,
  448. }
  449. def _empty_search_payload(*, provider: str, error: str | None = None) -> dict:
  450. return {
  451. "provider": provider,
  452. "search_results": [],
  453. "has_more": False,
  454. "next_cursor": "",
  455. "error": error,
  456. }
  457. def _hit_search_payload(*, provider: str, aweme_id: str) -> dict:
  458. return {
  459. "provider": provider,
  460. "search_results": [{"aweme_id": aweme_id, "desc": "命中"}],
  461. "has_more": False,
  462. "next_cursor": "",
  463. }
  464. @pytest.mark.asyncio
  465. async def test_default_search_falls_back_to_tikhub_when_internal_is_empty(
  466. monkeypatch,
  467. ) -> None:
  468. service = _FakeSearchService()
  469. providers: list[str] = []
  470. async def fake_internal(**_kwargs):
  471. providers.append("internal_keyword")
  472. return _empty_search_payload(provider="internal_keyword")
  473. async def fake_tikhub(**_kwargs):
  474. providers.append("tikhub")
  475. return _hit_search_payload(provider="tikhub", aweme_id="1")
  476. monkeypatch.setattr(find_agent_tools, "get_find_agent_v2_service", lambda: service)
  477. monkeypatch.setattr(find_agent_tools, "search_internal", fake_internal)
  478. monkeypatch.setattr(find_agent_tools, "search_tikhub", fake_tikhub)
  479. raw = await find_agent_tools.search_videos_v2(
  480. run_id="run",
  481. round_index=1,
  482. searches=[{
  483. "keyword": "甲午战争 民族觉醒 历史真相",
  484. "query_reason": "扩大候选池",
  485. }],
  486. )
  487. payload = json.loads(raw)
  488. assert providers == ["internal_keyword", "tikhub"]
  489. assert [item["provider"] for item in payload["searches"]] == [
  490. "internal_keyword",
  491. "tikhub",
  492. ]
  493. assert payload["searches"][1]["fallback_from"] == "internal_keyword"
  494. assert [item["provider"] for item in service.saved] == [
  495. "internal_keyword",
  496. "tikhub",
  497. ]
  498. @pytest.mark.asyncio
  499. async def test_default_search_skips_tikhub_when_internal_has_results(
  500. monkeypatch,
  501. ) -> None:
  502. service = _FakeSearchService()
  503. providers: list[str] = []
  504. async def fake_internal(**_kwargs):
  505. providers.append("internal_keyword")
  506. return _hit_search_payload(provider="internal_keyword", aweme_id="2")
  507. async def fake_tikhub(**_kwargs):
  508. providers.append("tikhub")
  509. return _hit_search_payload(provider="tikhub", aweme_id="3")
  510. monkeypatch.setattr(find_agent_tools, "get_find_agent_v2_service", lambda: service)
  511. monkeypatch.setattr(find_agent_tools, "search_internal", fake_internal)
  512. monkeypatch.setattr(find_agent_tools, "search_tikhub", fake_tikhub)
  513. raw = await find_agent_tools.search_videos_v2(
  514. run_id="run",
  515. round_index=1,
  516. searches=[{
  517. "keyword": "国家安全 反间谍",
  518. "query_reason": "未指定 provider 且内部有结果",
  519. }],
  520. )
  521. payload = json.loads(raw)
  522. assert providers == ["internal_keyword"]
  523. assert [item["provider"] for item in payload["searches"]] == ["internal_keyword"]
  524. assert "fallback_from" not in payload["searches"][0]
  525. @pytest.mark.asyncio
  526. async def test_explicit_internal_search_does_not_fallback_when_empty(
  527. monkeypatch,
  528. ) -> None:
  529. service = _FakeSearchService()
  530. providers: list[str] = []
  531. async def fake_internal(**_kwargs):
  532. providers.append("internal_keyword")
  533. return _empty_search_payload(provider="internal_keyword")
  534. async def fake_tikhub(**_kwargs):
  535. providers.append("tikhub")
  536. return _hit_search_payload(provider="tikhub", aweme_id="3")
  537. monkeypatch.setattr(find_agent_tools, "get_find_agent_v2_service", lambda: service)
  538. monkeypatch.setattr(find_agent_tools, "search_internal", fake_internal)
  539. monkeypatch.setattr(find_agent_tools, "search_tikhub", fake_tikhub)
  540. raw = await find_agent_tools.search_videos_v2(
  541. run_id="run",
  542. round_index=1,
  543. searches=[{
  544. "keyword": "国家安全 反间谍",
  545. "query_reason": "指定只用内部搜索",
  546. "provider": "internal_keyword",
  547. }],
  548. )
  549. payload = json.loads(raw)
  550. assert providers == ["internal_keyword"]
  551. assert [item["provider"] for item in payload["searches"]] == ["internal_keyword"]
  552. assert "fallback_from" not in payload["searches"][0]
  553. @pytest.mark.asyncio
  554. async def test_explicit_tikhub_search_does_not_call_internal(monkeypatch) -> None:
  555. service = _FakeSearchService()
  556. providers: list[str] = []
  557. async def fake_internal(**_kwargs):
  558. providers.append("internal_keyword")
  559. return _hit_search_payload(provider="internal_keyword", aweme_id="4")
  560. async def fake_tikhub(**_kwargs):
  561. providers.append("tikhub")
  562. return _hit_search_payload(provider="tikhub", aweme_id="5")
  563. monkeypatch.setattr(find_agent_tools, "get_find_agent_v2_service", lambda: service)
  564. monkeypatch.setattr(find_agent_tools, "search_internal", fake_internal)
  565. monkeypatch.setattr(find_agent_tools, "search_tikhub", fake_tikhub)
  566. raw = await find_agent_tools.search_videos_v2(
  567. run_id="run",
  568. round_index=1,
  569. searches=[{
  570. "keyword": "抗日战争 历史档案",
  571. "query_reason": "指定 TikHub",
  572. "provider": "tikhub",
  573. }],
  574. )
  575. payload = json.loads(raw)
  576. assert providers == ["tikhub"]
  577. assert [item["provider"] for item in payload["searches"]] == ["tikhub"]
  578. def test_stage_tool_allowlists_are_physical_and_isolated() -> None:
  579. assert _names(SEARCH_TOOLS) == {"search_videos_v2", "query_find_agent_v2_state"}
  580. assert _names(EVIDENCE_TOOLS) == {
  581. "fetch_candidate_details_v2",
  582. "fetch_candidate_portraits_v2",
  583. "query_pending_candidates_v2",
  584. }
  585. assert _names(EVALUATION_TOOLS) == {
  586. "understand_candidate_video_30s_v2",
  587. "evaluate_candidates_v2",
  588. "query_pending_candidates_v2",
  589. }
  590. assert _names(REPORT_TOOLS) == {"query_find_agent_v2_state"}
  591. all_names = _names((*SEARCH_TOOLS, *EVIDENCE_TOOLS, *EVALUATION_TOOLS, *REPORT_TOOLS))
  592. assert not any(name.startswith("batch_update_video_discovery") for name in all_names)
  593. assert "query_video_discovery_state" not in all_names
  594. def test_bound_worker_tools_query_exact_ids_beyond_global_display_limit(monkeypatch) -> None:
  595. class Service:
  596. def require_run(self, run_id):
  597. return {"run_id": run_id, "status": "running"}
  598. def candidate_inputs(self, run_id, candidate_ids):
  599. assert run_id == "large-run"
  600. assert candidate_ids == [101, 102]
  601. return [
  602. {
  603. "candidate_id": candidate_id,
  604. "aweme_id": f"video-{candidate_id}",
  605. "decision_bucket": "pending_evaluation",
  606. }
  607. for candidate_id in candidate_ids
  608. ]
  609. def get_full_state(self, *_args, **_kwargs):
  610. raise AssertionError("Worker 不应通过全局展示投影查询自己的分片")
  611. def evaluate(self, run_id, items):
  612. assert run_id == "large-run"
  613. return [
  614. {"candidate_id": item["candidate_id"], "decision_bucket": "primary"}
  615. for item in items
  616. ]
  617. monkeypatch.setattr(find_agent_tools, "get_find_agent_v2_service", lambda: Service())
  618. query_tool = bound_candidate_tools(
  619. (EVIDENCE_TOOLS[2],), run_id="large-run", candidate_ids=[101, 102],
  620. )[0]
  621. query_result = json.loads(query_tool(run_id="large-run", limit=100))
  622. assert [item["candidate_id"] for item in query_result["candidates"]] == [101, 102]
  623. evaluation_tool = bound_candidate_tools(
  624. (EVALUATION_TOOLS[1],), run_id="large-run", candidate_ids=[101, 102],
  625. )[0]
  626. evaluation_result = json.loads(evaluation_tool(
  627. run_id="large-run",
  628. items=[
  629. _evaluation(candidate_id=101),
  630. _evaluation(candidate_id=102),
  631. ],
  632. ))
  633. assert [item["candidate_id"] for item in evaluation_result["updated"]] == [101, 102]
  634. def test_common_prompt_points_to_v2_tables_and_tools() -> None:
  635. assert "find_agent_v2_run" in COMMON_RULES
  636. assert "video_discovery_run" not in COMMON_RULES
  637. assert "batch_search_and_record" not in COMMON_RULES
  638. assert "batch_update_video_discovery_candidates" not in COMMON_RULES
  639. def test_evaluator_prompt_defines_video_capability_and_judgment_boundaries() -> None:
  640. assert "understand_candidate_video_30s_v2" in EVALUATOR_PROMPT
  641. assert "同一候选" in EVALUATOR_PROMPT and "最多调用一次" in EVALUATOR_PROMPT
  642. assert "已经通过硬门禁" in EVALUATOR_PROMPT
  643. assert "不得把视频人物年龄当作受众画像" in EVALUATOR_PROMPT
  644. assert "不得用点赞量冒充分享量" in EVALUATOR_PROMPT
  645. assert "没有强日期依赖线索" in EVALUATOR_PROMPT
  646. assert "无法确认时按关键时间证据不足 rejected" in EVALUATOR_PROMPT
  647. assert "均属于强日期" in EVALUATOR_PROMPT
  648. assert "不能当作常青内容处理" in EVALUATOR_PROMPT
  649. assert "仍无法确认时,应 rejected" in EVALUATOR_PROMPT
  650. @pytest.mark.parametrize(
  651. "previous,current,expected,reason",
  652. [
  653. (
  654. DiscoverySnapshot("running", 1, 10, 0, 3, 3, 7),
  655. DiscoverySnapshot("running", 2, 10, 0, 3, 3, 7),
  656. False,
  657. "没有新增候选",
  658. ),
  659. (
  660. DiscoverySnapshot("running", 0, 0, 0, 0, 0, 0),
  661. DiscoverySnapshot("running", 1, 4, 0, 1, 1, 3),
  662. True,
  663. "样本不足",
  664. ),
  665. (
  666. DiscoverySnapshot("running", 1, 4, 0, 1, 1, 3),
  667. DiscoverySnapshot("running", 2, 12, 0, 1, 1, 11),
  668. False,
  669. "通过率为 0",
  670. ),
  671. (
  672. DiscoverySnapshot("running", 1, 8, 0, 1, 1, 7),
  673. DiscoverySnapshot("running", 2, 12, 0, 2, 2, 10),
  674. True,
  675. "正向通过率",
  676. ),
  677. ],
  678. )
  679. def test_exploration_decision_uses_volume_and_pass_rate(
  680. previous, current, expected, reason,
  681. ) -> None:
  682. decision = decide_continued_exploration(previous, current)
  683. assert decision.continue_exploring is expected
  684. assert reason in decision.reason
  685. def test_video_understanding_only_allows_candidates_passing_all_hard_gates(monkeypatch) -> None:
  686. service = _FakeService(pending_after_search=0)
  687. graph = FindAgentRoundGraph(service=service, runner=object())
  688. monkeypatch.setattr(service, "require_run", lambda _run_id: {
  689. "run_id": "run", "rule_config": build_rule_snapshot(),
  690. })
  691. common = {
  692. "video_url": "https://example.test/video.mp4",
  693. "publish_at": build_rule_snapshot()["current_datetime"],
  694. "content_50_plus_ratio": 0.30,
  695. "duration_seconds": 60,
  696. "share_count": 2000,
  697. }
  698. items = [
  699. {"candidate_id": 1, **common},
  700. {"candidate_id": 2, **common, "content_50_plus_ratio": 0.05},
  701. {"candidate_id": 3, **common, "publish_at": None},
  702. {"candidate_id": 4, **common, "video_url": None},
  703. {"candidate_id": 5, **common, "duration_seconds": 10},
  704. {"candidate_id": 6, **common, "share_count": 1},
  705. {"candidate_id": 7, **common, "duration_seconds": None},
  706. {"candidate_id": 8, **common, "share_count": None},
  707. ]
  708. assert graph._video_understanding_ids({"run_id": "run"}, items) == [1, 3]
  709. class _FakeService:
  710. def __init__(self, *, pending_after_search: int) -> None:
  711. self.pending_after_search = pending_after_search
  712. self.stage = "start"
  713. self.updates: list[dict] = []
  714. self.rejected_ids: set[int] = set()
  715. def require_run(self, run_id: str):
  716. snapshot = self.snapshot(run_id)
  717. return {
  718. "run_id": run_id,
  719. "status": "running",
  720. "outcome_status": None,
  721. "current_round": 1,
  722. "search_count": snapshot.search_count,
  723. "candidate_count": snapshot.candidate_count,
  724. "valid_primary_count": snapshot.valid_primary_count,
  725. "rule_config": build_rule_snapshot(),
  726. }
  727. def get_search_summaries(self, _run_id: str):
  728. if self.stage == "start":
  729. return []
  730. return [{
  731. "search_id": 1,
  732. "round_index": 1,
  733. "keyword": "测试需求",
  734. "query_reason": "建立候选池",
  735. "provider": "internal_keyword",
  736. "page_no": 1,
  737. "has_more": False,
  738. "next_cursor": None,
  739. "status": "success",
  740. "result_count": self.pending_after_search,
  741. }]
  742. def _candidate_rows(self):
  743. if self.stage == "start":
  744. return []
  745. evidence_status = "pending" if self.stage == "searched" else "success"
  746. return [{
  747. "candidate_id": index,
  748. "decision_bucket": (
  749. "rejected"
  750. if self.stage == "evaluated" or index in self.rejected_ids
  751. else "pending_evaluation"
  752. ),
  753. "detail_status": evidence_status,
  754. "portrait_status": evidence_status,
  755. "publish_at": build_rule_snapshot()["current_datetime"],
  756. "duration_seconds": 60,
  757. "share_count": 2000,
  758. "content_50_plus_ratio": 0.30,
  759. } for index in range(1, self.pending_after_search + 1)]
  760. def get_full_state(self, run_id: str, **_kwargs):
  761. return {
  762. "run": self.require_run(run_id),
  763. "searches": self.get_search_summaries(run_id),
  764. "candidates": self._candidate_rows(),
  765. }
  766. def candidate_inputs(self, _run_id: str, candidate_ids: list[int]):
  767. allowed = {int(value) for value in candidate_ids}
  768. return [
  769. item for item in self._candidate_rows()
  770. if int(item["candidate_id"]) in allowed
  771. ]
  772. def get_candidate_progress(self, _run_id: str):
  773. candidates = self._candidate_rows()
  774. pending = [
  775. item for item in candidates
  776. if item["decision_bucket"] == "pending_evaluation"
  777. ]
  778. return {
  779. "total_count": len(candidates),
  780. "pending_count": len(pending),
  781. "primary_count": 0,
  782. "rejected_count": len(candidates) - len(pending),
  783. "detail_pending_count": sum(item["detail_status"] == "pending" for item in pending),
  784. "detail_success_count": sum(item["detail_status"] == "success" for item in pending),
  785. "detail_failed_count": 0,
  786. "portrait_pending_count": sum(item["portrait_status"] == "pending" for item in pending),
  787. "portrait_success_count": sum(item["portrait_status"] == "success" for item in pending),
  788. "portrait_failed_count": 0,
  789. "evidence_completed_count": sum(
  790. item["detail_status"] != "pending" and item["portrait_status"] != "pending"
  791. for item in pending
  792. ),
  793. "evidence_success_count": sum(
  794. item["detail_status"] == "success" and item["portrait_status"] == "success"
  795. for item in pending
  796. ),
  797. }
  798. def list_pending_evidence_ids(self, _run_id: str, evidence_type: str, *, limit: int):
  799. key = f"{evidence_type}_status"
  800. return [
  801. int(item["candidate_id"]) for item in self._candidate_rows()
  802. if item["decision_bucket"] == "pending_evaluation" and item[key] == "pending"
  803. ][:limit]
  804. def list_ready_evaluation_ids(self, _run_id: str, *, limit: int):
  805. return [
  806. int(item["candidate_id"]) for item in self._candidate_rows()
  807. if item["decision_bucket"] == "pending_evaluation"
  808. and item["detail_status"] != "pending"
  809. and item["portrait_status"] != "pending"
  810. ][:limit]
  811. def count_pending_candidates(self, run_id: str) -> int:
  812. return int(self.get_candidate_progress(run_id)["pending_count"])
  813. def get_report_state(self, run_id: str):
  814. return {
  815. "run": self.require_run(run_id),
  816. "summary": self.get_candidate_progress(run_id),
  817. "primary_candidates": [],
  818. "rejection_reason_distribution": {},
  819. }
  820. def snapshot(self, _run_id: str) -> DiscoverySnapshot:
  821. base = DiscoverySnapshot("running", 0, 0, 0, 0, 0, 0)
  822. if self.stage == "searched":
  823. return replace(base, search_count=1, candidate_count=self.pending_after_search,
  824. pending_count=self.pending_after_search)
  825. if self.stage in {"evidenced", "batched"}:
  826. return replace(base, search_count=1, candidate_count=self.pending_after_search,
  827. pending_count=self.pending_after_search)
  828. if self.stage == "evaluated":
  829. return replace(base, search_count=1, candidate_count=self.pending_after_search,
  830. rejected_count=self.pending_after_search)
  831. return base
  832. def update_round(self, _run_id: str, _round_index: int, **kwargs) -> None:
  833. self.updates.append(kwargs)
  834. def recount_valid_primary(self, _run_id: str) -> int:
  835. return 0
  836. def reject_failed_gates(self, _run_id: str, failures) -> list[int]:
  837. rejected = [candidate_id for candidate_id, _gate in failures]
  838. self.rejected_ids.update(rejected)
  839. return rejected
  840. class _FakeRunner:
  841. def __init__(self, service: _FakeService) -> None:
  842. self.service = service
  843. self.calls: list[tuple[str, set[str]]] = []
  844. self.inputs: list[tuple[str, str]] = []
  845. self.supervisor_visual_outputs: list[dict] = []
  846. async def run_node(self, *, node, round_index, tools=(), user_content="", **_kwargs) -> NodeRun:
  847. self.calls.append((node, _names(tools)))
  848. self.inputs.append((node, user_content))
  849. if node == "search":
  850. self.service.stage = "searched"
  851. elif node == "evidence":
  852. self.service.stage = "evidenced"
  853. elif node == "evaluator":
  854. self.service.stage = "evaluated"
  855. if node == "supervisor":
  856. if self.service.stage == "evaluated":
  857. next_action = "finish"
  858. elif self.service.stage in {"evidenced", "batched"}:
  859. next_action = "evaluator"
  860. elif self.service.stage == "searched" and not self.service.pending_after_search:
  861. next_action = "finish"
  862. else:
  863. next_action = "search"
  864. content = json.dumps({
  865. "next_action": next_action,
  866. "worker_count": 4,
  867. "evidence_scope": "both",
  868. "plan": {"searches": []},
  869. })
  870. else:
  871. content = '{"searches": []}'
  872. return NodeRun(node, round_index, content, 1, 0)
  873. @staticmethod
  874. def execution_plan() -> ExecutionPlan:
  875. return ExecutionPlan(
  876. demand_brief=DemandBrief(
  877. core_intent="测试需求",
  878. relevance_criteria=["内容直接匹配测试需求"],
  879. ),
  880. search_tasks=[SearchTask(
  881. task_id="search-1", keyword="测试需求", query_reason="建立候选池",
  882. )],
  883. evaluation_brief=EvaluationBrief(
  884. relevance_criteria=["内容直接匹配测试需求"],
  885. ),
  886. )
  887. async def run_planning(self, *, round_index, **kwargs):
  888. run = await self.run_node(node="supervisor", round_index=round_index, **kwargs)
  889. proposal = json.loads(run.content)
  890. return run, PlanningDecision(
  891. next_action=proposal["next_action"],
  892. worker_count=proposal["worker_count"],
  893. evidence_scope=proposal["evidence_scope"],
  894. execution_plan=self.execution_plan(),
  895. )
  896. async def run_supervision(self, *, round_index, **kwargs):
  897. output_enricher = kwargs.pop("output_enricher", None)
  898. run = await self.run_node(node="supervisor", round_index=round_index, **kwargs)
  899. proposal = json.loads(run.content)
  900. decision = SupervisorDecision(
  901. next_action=proposal["next_action"],
  902. worker_count=proposal["worker_count"],
  903. evidence_scope=proposal["evidence_scope"],
  904. )
  905. if output_enricher is not None:
  906. self.supervisor_visual_outputs.append(output_enricher(decision))
  907. return run, decision
  908. class _WrongEvidenceProposalRunner(_FakeRunner):
  909. async def run_node(self, *, node, round_index, tools=(), user_content="", **kwargs):
  910. run = await super().run_node(
  911. node=node,
  912. round_index=round_index,
  913. tools=tools,
  914. user_content=user_content,
  915. **kwargs,
  916. )
  917. if node == "supervisor" and self.service.stage == "evidenced":
  918. return NodeRun(
  919. node,
  920. round_index,
  921. json.dumps({
  922. "next_action": "evidence",
  923. "worker_count": 8,
  924. "evidence_scope": "both",
  925. }),
  926. 1,
  927. 0,
  928. )
  929. return run
  930. class _OptimizedRunner(_FakeRunner):
  931. def __init__(self, service: _FakeService) -> None:
  932. super().__init__(service)
  933. self.host_calls: list[str] = []
  934. async def run_search_assignment(self, *, round_index, user_content, **_kwargs):
  935. self.host_calls.append("search")
  936. self.inputs.append(("search", user_content))
  937. self.service.stage = "searched"
  938. return NodeRun("search", round_index, "host search", 0, 1)
  939. async def run_evidence_assignment(
  940. self, *, round_index, user_content, **_kwargs,
  941. ):
  942. self.host_calls.append("evidence")
  943. self.inputs.append(("evidence", user_content))
  944. self.service.stage = "evidenced"
  945. return NodeRun("evidence", round_index, "host evidence", 0, 1)
  946. async def run_host_supervision(
  947. self, *, round_index, decision, output_enricher=None, **_kwargs,
  948. ):
  949. self.host_calls.append(f"supervisor:{decision.next_action}")
  950. if output_enricher is not None:
  951. self.supervisor_visual_outputs.append(output_enricher(decision))
  952. return NodeRun("supervisor", round_index, decision.model_dump_json(), 0, 0), decision
  953. @pytest.mark.asyncio
  954. async def test_host_executes_deterministic_stages_without_redundant_llm_calls() -> None:
  955. service = _FakeService(pending_after_search=2)
  956. runner = _OptimizedRunner(service)
  957. graph = FindAgentRoundGraph(service=service, runner=runner)
  958. result = await graph.invoke(FindAgentState(
  959. run_id="optimized-run", user_input="task", round_index=1,
  960. ))
  961. assert runner.host_calls == [
  962. "search",
  963. "supervisor:evidence",
  964. "evidence",
  965. "evidence",
  966. "supervisor:evaluator",
  967. ]
  968. assert [node for node, _tools in runner.calls] == [
  969. "supervisor", "evaluator", "supervisor",
  970. ]
  971. assert result.snapshot is not None and result.snapshot.pending_count == 0
  972. def test_supervisor_progress_counts_missing_evidence_only_for_pending_candidates() -> None:
  973. class ProgressService:
  974. @staticmethod
  975. def require_run(_run_id: str):
  976. return {}
  977. @staticmethod
  978. def get_search_summaries(_run_id: str):
  979. return []
  980. @staticmethod
  981. def get_candidate_progress(_run_id: str):
  982. return {
  983. "total_count": 4,
  984. "pending_count": 3,
  985. "primary_count": 0,
  986. "rejected_count": 1,
  987. "detail_pending_count": 0,
  988. "detail_success_count": 2,
  989. "detail_failed_count": 1,
  990. "portrait_pending_count": 1,
  991. "portrait_success_count": 2,
  992. "portrait_failed_count": 0,
  993. "evidence_completed_count": 2,
  994. "evidence_success_count": 1,
  995. }
  996. graph = FindAgentRoundGraph(service=ProgressService(), runner=object())
  997. progress = graph._supervisor_state({"run_id": "run"})["candidate_progress"]
  998. assert progress["pending_count"] == 3
  999. assert progress["detail_pending_count"] == 0
  1000. assert progress["detail_success_count"] == 2
  1001. assert progress["detail_failed_count"] == 1
  1002. assert progress["portrait_pending_count"] == 1
  1003. assert progress["portrait_success_count"] == 2
  1004. assert progress["portrait_failed_count"] == 0
  1005. assert progress["evidence_completed_count"] == 2
  1006. assert progress["evidence_success_count"] == 1
  1007. @pytest.mark.asyncio
  1008. async def test_supervisor_visualization_records_host_override() -> None:
  1009. service = _FakeService(pending_after_search=1)
  1010. runner = _WrongEvidenceProposalRunner(service)
  1011. graph = FindAgentRoundGraph(service=service, runner=runner)
  1012. await graph.invoke(FindAgentState(run_id="override-run", user_input="task", round_index=1))
  1013. validations = [
  1014. item["Supervisor决策校验"] for item in runner.supervisor_visual_outputs
  1015. ]
  1016. overridden = next(
  1017. item
  1018. for item in validations
  1019. if item["proposed_action"] == "evidence"
  1020. and item["approved_action"] == "evaluator"
  1021. )
  1022. assert overridden["proposed_action"] == "evidence"
  1023. assert overridden["approved_action"] == "evaluator"
  1024. @pytest.mark.asyncio
  1025. async def test_round_graph_supervisor_routes_with_guarded_allowlists() -> None:
  1026. service = _FakeService(pending_after_search=2)
  1027. runner = _FakeRunner(service)
  1028. graph = FindAgentRoundGraph(service=service, runner=runner)
  1029. state = FindAgentState(run_id="new-run", user_input="task", round_index=1)
  1030. result = await graph.invoke(state)
  1031. assert [name for name, _ in runner.calls] == [
  1032. "supervisor", "search", "supervisor", "evidence", "evidence",
  1033. "supervisor", "evaluator", "supervisor",
  1034. ]
  1035. assert runner.calls[0][1] == set()
  1036. assert runner.calls[1][1] == _names(SEARCH_TOOLS)
  1037. assert runner.calls[2][1] == set()
  1038. assert runner.calls[3][1] <= _names(EVIDENCE_TOOLS)
  1039. assert runner.calls[4][1] <= _names(EVIDENCE_TOOLS)
  1040. assert runner.calls[6][1] == {
  1041. "evaluate_candidates_v2", "query_pending_candidates_v2",
  1042. }
  1043. assert result.phase == "done"
  1044. assert result.snapshot is not None and result.snapshot.pending_count == 0
  1045. @pytest.mark.asyncio
  1046. async def test_raw_demand_is_only_sent_to_initial_planner() -> None:
  1047. service = _FakeService(pending_after_search=1)
  1048. runner = _FakeRunner(service)
  1049. graph = FindAgentRoundGraph(service=service, runner=runner)
  1050. marker = "RAW-DEMAND-MUST-NOT-LEAK"
  1051. await graph.invoke(FindAgentState(
  1052. run_id="assignment-run", user_input=marker, round_index=1,
  1053. ))
  1054. assert marker in runner.inputs[0][1]
  1055. assert all(marker not in content for _node, content in runner.inputs[1:])
  1056. search_payload = json.loads(next(
  1057. content for node, content in runner.inputs if node == "search"
  1058. ))
  1059. evidence_payload = json.loads(next(
  1060. content for node, content in runner.inputs if node == "evidence"
  1061. ))
  1062. supervisor_payload = json.loads([
  1063. content for node, content in runner.inputs if node == "supervisor"
  1064. ][1])
  1065. assert set(search_payload) == {"run_id", "round_index", "tasks"}
  1066. assert set(evidence_payload) == {
  1067. "run_id", "round_index", "candidate_ids", "evidence_type", "candidates",
  1068. }
  1069. assert "candidates" not in supervisor_payload["execution_state"]
  1070. assert "candidate_progress" in supervisor_payload["execution_state"]
  1071. def test_obagent_declaration_cannot_override_actual_model_input() -> None:
  1072. class Context:
  1073. declared = False
  1074. def declare(self, **_kwargs):
  1075. self.declared = True
  1076. return "SDK-rendered-input-that-must-not-reach-model"
  1077. context = Context()
  1078. handle = _ModuleHandle(context)
  1079. actual = handle.declare(
  1080. fallback='{"run_id":"r","tasks":[]}',
  1081. system_prompt="prompt",
  1082. slots=(InputSlot("阶段任务", "{}", "stage_assignment", "test", False),),
  1083. tools=(),
  1084. model="model",
  1085. )
  1086. assert context.declared is True
  1087. assert actual == '{"run_id":"r","tasks":[]}'
  1088. @pytest.mark.asyncio
  1089. async def test_round_graph_skips_evidence_and_evaluation_without_candidates() -> None:
  1090. service = _FakeService(pending_after_search=0)
  1091. runner = _FakeRunner(service)
  1092. graph = FindAgentRoundGraph(service=service, runner=runner)
  1093. await graph.invoke(FindAgentState(run_id="new-run", user_input="task", round_index=1))
  1094. assert [name for name, _ in runner.calls] == [
  1095. "supervisor", "search", "supervisor",
  1096. ]
  1097. @pytest.mark.asyncio
  1098. async def test_evaluator_model_never_receives_hard_gate_failures() -> None:
  1099. service = _FakeService(pending_after_search=1)
  1100. service.stage = "evidenced"
  1101. rejected: list[int] = []
  1102. bucket = ["pending_evaluation"]
  1103. def candidate_inputs(_run_id: str, candidate_ids: list[int]):
  1104. if 1 not in candidate_ids:
  1105. return []
  1106. return [{
  1107. "candidate_id": 1,
  1108. "decision_bucket": bucket[0],
  1109. "detail_status": "success",
  1110. "portrait_status": "success",
  1111. "publish_at": build_rule_snapshot()["current_datetime"],
  1112. "duration_seconds": 10,
  1113. "share_count": 2000,
  1114. "content_50_plus_ratio": 0.30,
  1115. "video_url": "https://example.test/video.mp4",
  1116. }]
  1117. def reject(_run_id: str, failures):
  1118. ids = [candidate_id for candidate_id, _ in failures]
  1119. rejected.extend(ids)
  1120. bucket[0] = "rejected"
  1121. return ids
  1122. service.candidate_inputs = candidate_inputs # type: ignore[method-assign]
  1123. service.list_ready_evaluation_ids = ( # type: ignore[method-assign]
  1124. lambda _run_id, limit: [1] if bucket[0] == "pending_evaluation" else []
  1125. )
  1126. service.reject_failed_gates = reject # type: ignore[method-assign]
  1127. runner = _FakeRunner(service)
  1128. graph = FindAgentRoundGraph(service=service, runner=runner)
  1129. await graph._evaluator({
  1130. "run_id": "r", "round_index": 1, "worker_count": 1,
  1131. "action_count": 0, "node_runs": [], "user_input": "需求",
  1132. })
  1133. assert rejected == [1]
  1134. assert not any(name == "evaluator" for name, _tools in runner.calls)
  1135. @pytest.mark.asyncio
  1136. async def test_evaluator_drains_database_batches_until_pending_queue_is_empty() -> None:
  1137. class BatchedService(_FakeService):
  1138. def __init__(self) -> None:
  1139. super().__init__(pending_after_search=3)
  1140. self.remaining_ids = {1, 2, 3}
  1141. def _candidate_rows(self):
  1142. if self.stage == "start":
  1143. return []
  1144. evidence_status = "pending" if self.stage == "searched" else "success"
  1145. return [{
  1146. "candidate_id": index,
  1147. "decision_bucket": (
  1148. "pending_evaluation" if index in self.remaining_ids else "rejected"
  1149. ),
  1150. "detail_status": evidence_status,
  1151. "portrait_status": evidence_status,
  1152. "publish_at": build_rule_snapshot()["current_datetime"],
  1153. "duration_seconds": 60,
  1154. "share_count": 2000,
  1155. "content_50_plus_ratio": 0.30,
  1156. } for index in range(1, 4)]
  1157. def list_ready_evaluation_ids(self, _run_id: str, *, limit: int):
  1158. del limit
  1159. return sorted(self.remaining_ids)[:1]
  1160. def snapshot(self, _run_id: str) -> DiscoverySnapshot:
  1161. if self.stage == "start":
  1162. return DiscoverySnapshot("running", 0, 0, 0, 0, 0, 0)
  1163. return DiscoverySnapshot(
  1164. "running", 1, 3, len(self.remaining_ids), 0, 0,
  1165. 3 - len(self.remaining_ids),
  1166. )
  1167. service = BatchedService()
  1168. class BatchedRunner(_FakeRunner):
  1169. def __init__(self, fake_service: _FakeService) -> None:
  1170. super().__init__(fake_service)
  1171. async def run_node(
  1172. self, *, node, round_index, tools=(), user_content="", **kwargs,
  1173. ) -> NodeRun:
  1174. if node == "evaluator":
  1175. self.calls.append((node, _names(tools)))
  1176. self.inputs.append((node, user_content))
  1177. candidate_ids = json.loads(user_content)["candidate_ids"]
  1178. self.service.remaining_ids.difference_update(candidate_ids)
  1179. self.service.stage = (
  1180. "evaluated" if not self.service.remaining_ids else "batched"
  1181. )
  1182. return NodeRun(node, round_index, "", 1, 0)
  1183. return await super().run_node(
  1184. node=node,
  1185. round_index=round_index,
  1186. tools=tools,
  1187. user_content=user_content,
  1188. **kwargs,
  1189. )
  1190. runner = BatchedRunner(service)
  1191. graph = FindAgentRoundGraph(service=service, runner=runner)
  1192. result = await graph.invoke(
  1193. FindAgentState(run_id="batched-run", user_input="task", round_index=1),
  1194. )
  1195. assert [name for name, _ in runner.calls].count("evaluator") == 3
  1196. assert result.snapshot is not None and result.snapshot.pending_count == 0
  1197. def test_supervisor_policy_overrides_unsafe_finish_and_clamps_workers() -> None:
  1198. service = _FakeService(pending_after_search=2)
  1199. service.stage = "searched"
  1200. graph = FindAgentRoundGraph(service=service, runner=_FakeRunner(service))
  1201. action, _reason, workers, scope = graph._approve_action(
  1202. {"run_id": "r", "search_actions": 1, "action_count": 1},
  1203. {"next_action": "finish", "worker_count": 99, "evidence_scope": "invalid"},
  1204. )
  1205. assert action == "evidence"
  1206. assert workers == 8
  1207. assert scope == "both"
  1208. def test_supervisor_can_choose_an_extra_search_within_budget() -> None:
  1209. service = _FakeService(pending_after_search=0)
  1210. service.stage = "searched"
  1211. graph = FindAgentRoundGraph(service=service, runner=_FakeRunner(service))
  1212. action, *_ = graph._approve_action(
  1213. {"run_id": "r", "search_actions": 1, "action_count": 1},
  1214. {"next_action": "search", "worker_count": 2},
  1215. )
  1216. assert action == "search"
  1217. @pytest.mark.asyncio
  1218. async def test_round_graph_rejects_incomplete_evaluator_batch() -> None:
  1219. service = _FakeService(pending_after_search=2)
  1220. class RetryRunner(_FakeRunner):
  1221. evaluator_calls = 0
  1222. async def run_node(self, *, node, round_index, tools=(), **kwargs) -> NodeRun:
  1223. if node == "evaluator":
  1224. self.calls.append((node, _names(tools)))
  1225. self.evaluator_calls += 1
  1226. return NodeRun(node, round_index, "", 1, 0)
  1227. return await super().run_node(
  1228. node=node, round_index=round_index, tools=tools, **kwargs,
  1229. )
  1230. runner = RetryRunner(service)
  1231. graph = FindAgentRoundGraph(service=service, runner=runner)
  1232. with pytest.raises(RuntimeError, match="评估分批未完整消费"):
  1233. await graph.invoke(
  1234. FindAgentState(run_id="retry-run", user_input="task", round_index=1),
  1235. )
  1236. assert runner.evaluator_calls == 1