test_acquisition_runner.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828
  1. from __future__ import annotations
  2. from uuid import UUID, uuid4
  3. from acquisition.classification.coarse import ClassificationResult
  4. from acquisition.domain import (
  5. AcquisitionJob,
  6. AcquisitionRun,
  7. CandidateItem,
  8. ItemClassification,
  9. MediaAsset,
  10. Query,
  11. QueryBatch,
  12. )
  13. from acquisition.media.service import StabilizedMedia
  14. from acquisition.platforms.base import PlatformCandidate, PlatformItem, PlatformSearchPage
  15. from acquisition.runner import run_batch
  16. from core.config import PgConfig, Settings
  17. def _settings() -> Settings:
  18. return Settings(
  19. pg=PgConfig(host="h", port=5432, user="u", password="p", database="d"),
  20. aiddit_crawler_base_url="http://crawler.test",
  21. crawler_timeout=30,
  22. openrouter_timeout_seconds=90,
  23. openrouter_model="m",
  24. openrouter_base_url="http://openrouter.test",
  25. openrouter_api_key="k",
  26. llm_model="m",
  27. max_cards=12,
  28. frames_dir="f",
  29. douyin_ratio="540p",
  30. data_dir="data",
  31. )
  32. class FakeRepo:
  33. def __init__(self, *, job_status: str = "pending") -> None:
  34. self.batch_id = uuid4()
  35. self.query = Query(
  36. id=uuid4(),
  37. batch_id=self.batch_id,
  38. query_text="脚本 开头 怎么做",
  39. keep=True,
  40. status="ready",
  41. )
  42. self.job_status = job_status
  43. self.runs: list[AcquisitionRun] = []
  44. self.jobs: list[AcquisitionJob] = []
  45. self.updated_jobs: list[AcquisitionJob] = []
  46. self.items: list[CandidateItem] = []
  47. self.media: list[MediaAsset] = []
  48. self.classifications: list[ItemClassification] = []
  49. self.existing_by_unique_key: dict[str, CandidateItem] = {}
  50. self.attached_items: list[CandidateItem] = []
  51. def create_query_batch(self, **kwargs):
  52. return QueryBatch(id=self.batch_id, **kwargs)
  53. def add_query(self, **kwargs):
  54. return Query(id=uuid4(), **kwargs)
  55. def list_queries_for_batch(self, batch_id: UUID, *, keep: bool | None = None):
  56. assert batch_id == self.batch_id
  57. assert keep is True
  58. return [self.query]
  59. def create_acquisition_run(self, **kwargs):
  60. run = AcquisitionRun(id=uuid4(), **kwargs)
  61. self.runs.append(run)
  62. return run
  63. def ensure_acquisition_job(self, **kwargs):
  64. kwargs["status"] = self.job_status
  65. job = AcquisitionJob(id=uuid4(), **kwargs)
  66. self.jobs.append(job)
  67. return job
  68. def update_acquisition_job(self, job_id: UUID, **kwargs):
  69. original = next(job for job in self.jobs if job.id == job_id)
  70. updated = original.model_copy(update=kwargs)
  71. self.updated_jobs.append(updated)
  72. return updated
  73. def upsert_candidate_item(self, **kwargs):
  74. item = CandidateItem(id=uuid4(), **kwargs)
  75. self.items.append(item)
  76. return item
  77. def get_candidate_item_by_unique_key(self, unique_key: str):
  78. return self.existing_by_unique_key.get(unique_key)
  79. def attach_existing_candidate_item(self, item_id: UUID, *, job_id: UUID, query_id: UUID, metadata=None):
  80. item = next(row for row in self.existing_by_unique_key.values() if row.id == item_id)
  81. updated = item.model_copy(update={"job_id": job_id, "query_id": query_id, "metadata": metadata or {}})
  82. self.attached_items.append(updated)
  83. return updated
  84. def add_media_asset(self, **kwargs):
  85. row = MediaAsset(id=uuid4(), **kwargs)
  86. self.media.append(row)
  87. return row
  88. def add_item_classification(self, **kwargs):
  89. row = ItemClassification(id=uuid4(), **kwargs)
  90. self.classifications.append(row)
  91. return row
  92. def get_run_summary(self, run_id: UUID):
  93. return {}
  94. def get_query_detail(self, *, run_id: UUID, query_id: UUID):
  95. return {}
  96. class FakeAdapter:
  97. platform = "weixin"
  98. url = (
  99. "https://mp.weixin.qq.com/s?__biz=MzUxMDg4NDY1Ng=="
  100. "&mid=2247484898&idx=2&sn=5069e65919f414b96b7aec53671dc1b5"
  101. )
  102. def __init__(self) -> None:
  103. self.search_calls = 0
  104. self.detail_calls = 0
  105. def search(self, query, *, settings, limit, rate_limiter):
  106. self.search_calls += 1
  107. assert query == "脚本 开头 怎么做"
  108. assert limit == 2
  109. return [
  110. PlatformCandidate(
  111. rank=1,
  112. platform=self.platform,
  113. source_id="wx1",
  114. url=self.url,
  115. title="公众号脚本",
  116. author="作者",
  117. cover_url="https://img.test/a.jpg",
  118. )
  119. ]
  120. def fetch_detail(self, candidate, *, settings, rate_limiter):
  121. self.detail_calls += 1
  122. return PlatformItem(
  123. platform=self.platform,
  124. source_id=candidate.source_id,
  125. url=candidate.url,
  126. content_type="图文",
  127. title="公众号脚本",
  128. author="作者",
  129. body_text="先定受众再写开头",
  130. image_urls=["https://img.test/a.jpg"],
  131. raw={"ok": True},
  132. )
  133. class PartialAdapter(FakeAdapter):
  134. def search(self, query, *, settings, limit, rate_limiter):
  135. self.search_calls += 1
  136. return [
  137. PlatformCandidate(rank=1, platform=self.platform, source_id="bad", url="https://bad"),
  138. PlatformCandidate(rank=2, platform=self.platform, source_id="good", url="https://good"),
  139. ]
  140. def fetch_detail(self, candidate, *, settings, rate_limiter):
  141. self.detail_calls += 1
  142. if candidate.source_id == "bad":
  143. raise RuntimeError("detail boom")
  144. return PlatformItem(
  145. platform=self.platform,
  146. source_id=candidate.source_id,
  147. url=candidate.url,
  148. content_type="图文",
  149. title="可用详情",
  150. author="作者",
  151. body_text="先定受众",
  152. image_urls=[],
  153. raw={},
  154. )
  155. class FailingSearchAdapter(FakeAdapter):
  156. def search(self, query, *, settings, limit, rate_limiter):
  157. self.search_calls += 1
  158. raise RuntimeError("search boom")
  159. class UnsupportedModeAdapter(FakeAdapter):
  160. def fetch_detail(self, candidate, *, settings, rate_limiter):
  161. self.detail_calls += 1
  162. return PlatformItem(
  163. platform=self.platform,
  164. source_id=candidate.source_id,
  165. url=candidate.url,
  166. content_type="mystery",
  167. content_mode="unsupported",
  168. title="未知类型",
  169. body_text="",
  170. image_urls=[],
  171. video_urls=[],
  172. raw={},
  173. )
  174. class MissingVideoAdapter(FakeAdapter):
  175. def fetch_detail(self, candidate, *, settings, rate_limiter):
  176. self.detail_calls += 1
  177. return PlatformItem(
  178. platform="douyin",
  179. source_id="dy-video",
  180. url="https://douyin.test/v",
  181. content_type="video",
  182. content_mode="video_post",
  183. title="缺视频地址",
  184. body_text="",
  185. image_urls=[],
  186. video_urls=[],
  187. raw={},
  188. )
  189. class DouyinProviderAdapter(FakeAdapter):
  190. platform = "douyin"
  191. def search(self, query, *, settings, limit, rate_limiter):
  192. self.search_calls += 1
  193. return [
  194. PlatformCandidate(
  195. rank=1,
  196. platform=self.platform,
  197. provider="aiddit",
  198. source_id="761",
  199. raw={
  200. "id": "761",
  201. "search_provider": "aiddit",
  202. "original": {"aweme_id": "761"},
  203. "request_payload": {"keyword": query},
  204. },
  205. )
  206. ]
  207. def fetch_detail(self, candidate, *, settings, rate_limiter):
  208. self.detail_calls += 1
  209. return PlatformItem(
  210. platform=self.platform,
  211. provider=candidate.provider,
  212. source_id=candidate.source_id,
  213. url="https://douyin.test/video/761",
  214. content_type="video",
  215. content_mode="video_post",
  216. title="视频标题",
  217. author="作者",
  218. body_text="视频正文",
  219. video_urls=["https://video.test/761.mp4"],
  220. raw={
  221. "detail_provider": candidate.provider,
  222. "code": 0,
  223. "data": {"data": {"channel_content_id": candidate.source_id}},
  224. },
  225. )
  226. def _paged_candidate(source_id: str, *, rank: int, page_index: int, platform: str = "douyin") -> PlatformCandidate:
  227. return PlatformCandidate(
  228. rank=rank,
  229. platform=platform,
  230. provider="aiddit" if platform == "douyin" else "",
  231. source_id=source_id,
  232. url=f"https://{platform}.test/{source_id}",
  233. title=source_id,
  234. raw={
  235. "page_index": page_index,
  236. "page_rank": rank,
  237. "source_cursor": "" if page_index == 1 else str(page_index),
  238. },
  239. )
  240. def _paged_search_page(
  241. page_index: int,
  242. source_ids: list[str],
  243. *,
  244. has_more: bool = False,
  245. next_cursor: str = "",
  246. platform: str = "douyin",
  247. ) -> PlatformSearchPage:
  248. return PlatformSearchPage(
  249. page_index=page_index,
  250. candidates=[
  251. _paged_candidate(source_id, rank=i, page_index=page_index, platform=platform)
  252. for i, source_id in enumerate(source_ids, start=1)
  253. ],
  254. raw_count=len(source_ids),
  255. cursor="" if page_index == 1 else str(page_index),
  256. next_cursor=next_cursor,
  257. has_more=has_more,
  258. provider="aiddit" if platform == "douyin" else "",
  259. )
  260. class PagedAdapter:
  261. platform = "douyin"
  262. def __init__(
  263. self,
  264. pages: list[PlatformSearchPage],
  265. details: dict[str, PlatformItem | Exception],
  266. ) -> None:
  267. self.pages = pages
  268. self.details = details
  269. self.pages_requested = 0
  270. self.detail_calls = 0
  271. def search_pages(self, query, *, settings, limit, rate_limiter, max_pages=2):
  272. for page in self.pages[:max_pages]:
  273. self.pages_requested += 1
  274. yield page
  275. def search(self, query, *, settings, limit, rate_limiter):
  276. self.pages_requested += 1
  277. return self.pages[0].candidates if self.pages else []
  278. def fetch_detail(self, candidate, *, settings, rate_limiter):
  279. self.detail_calls += 1
  280. detail = self.details[candidate.source_id]
  281. if isinstance(detail, Exception):
  282. raise detail
  283. return detail
  284. def _detail(
  285. source_id: str,
  286. *,
  287. content_mode: str = "image_post",
  288. image_urls: list[str] | None = None,
  289. video_urls: list[str] | None = None,
  290. ) -> PlatformItem:
  291. return PlatformItem(
  292. platform="douyin",
  293. provider="aiddit",
  294. source_id=source_id,
  295. url=f"https://douyin.test/{source_id}",
  296. content_type="image" if content_mode == "image_post" else "video",
  297. content_mode=content_mode,
  298. title=source_id,
  299. body_text=f"{source_id} 正文",
  300. image_urls=image_urls if image_urls is not None else ["https://img.test/a.jpg"],
  301. video_urls=video_urls or [],
  302. raw={},
  303. )
  304. def _classification_by_title(creation_titles: set[str]):
  305. def classifier(**kwargs):
  306. is_creation = kwargs["title"] in creation_titles
  307. return ClassificationResult(
  308. is_creation_knowledge=is_creation,
  309. label="creation" if is_creation else "non_creation",
  310. confidence=0.9,
  311. reason="ok",
  312. )
  313. return classifier
  314. def test_run_batch_writes_jobs_items_media_and_classification():
  315. repo = FakeRepo()
  316. adapter = FakeAdapter()
  317. def media_stabilizer(**kwargs):
  318. assert kwargs["image_urls"] == ["https://img.test/a.jpg"]
  319. return [
  320. StabilizedMedia(
  321. media_type="image",
  322. source_url="https://img.test/a.jpg",
  323. cdn_url="https://cdn.test/a.jpg",
  324. position=1,
  325. )
  326. ]
  327. def classifier(**kwargs):
  328. assert kwargs["content_mode"] == "article"
  329. assert kwargs["image_urls"] == ["https://cdn.test/a.jpg"]
  330. return ClassificationResult(
  331. is_creation_knowledge=True,
  332. label="creation",
  333. confidence=1.0,
  334. reason="是创作知识",
  335. knowledge="先定受众",
  336. prompt_version="pv1",
  337. result_payload={"knowledge": "先定受众"},
  338. )
  339. result = run_batch(
  340. repo,
  341. batch_id=repo.batch_id,
  342. settings=_settings(),
  343. platforms=("weixin",),
  344. search_limit=2,
  345. display_limit=1,
  346. adapter_factory=lambda platform: adapter,
  347. media_stabilizer=media_stabilizer,
  348. classifier=classifier,
  349. rate_limiter_factory=lambda platform: object(),
  350. )
  351. assert result.done == 1
  352. assert result.failed == 0
  353. assert result.total_jobs == 1
  354. assert adapter.search_calls == 1
  355. assert adapter.detail_calls == 1
  356. assert repo.jobs[0].query_id == repo.query.id
  357. assert repo.items[0].platform_item_id == "wx1"
  358. assert repo.items[0].content_mode == "article"
  359. assert repo.items[0].body_text == "先定受众再写开头"
  360. assert repo.items[0].raw_summary == "先定受众再写开头"
  361. assert (
  362. repo.items[0].unique_key
  363. == "wx:MzUxMDg4NDY1Ng==:2247484898:2:5069e65919f414b96b7aec53671dc1b5"
  364. )
  365. assert repo.media[0].cdn_url == "https://cdn.test/a.jpg"
  366. assert repo.classifications[0].is_creation_knowledge is True
  367. assert repo.updated_jobs[-1].status == "done"
  368. def test_run_batch_records_unsupported_item_as_skipped_without_processing():
  369. repo = FakeRepo()
  370. adapter = UnsupportedModeAdapter()
  371. result = run_batch(
  372. repo,
  373. batch_id=repo.batch_id,
  374. settings=_settings(),
  375. platforms=("weixin",),
  376. search_limit=2,
  377. display_limit=1,
  378. adapter_factory=lambda platform: adapter,
  379. media_stabilizer=lambda **kwargs: (_ for _ in ()).throw(AssertionError("media should be skipped")),
  380. classifier=lambda **kwargs: (_ for _ in ()).throw(AssertionError("classifier should be skipped")),
  381. rate_limiter_factory=lambda platform: object(),
  382. )
  383. assert result.done == 1
  384. assert repo.items[0].status == "skipped"
  385. assert repo.items[0].content_mode == "unsupported"
  386. assert repo.items[0].metadata["skip_reason"] == "unsupported_content_mode"
  387. assert repo.media == []
  388. assert repo.classifications[0].status == "skipped"
  389. assert repo.classifications[0].label == "unsupported_content_mode"
  390. assert repo.updated_jobs[-1].metadata["skipped_item_count"] == 1
  391. def test_run_batch_records_video_missing_as_skipped_without_processing():
  392. repo = FakeRepo()
  393. adapter = MissingVideoAdapter()
  394. result = run_batch(
  395. repo,
  396. batch_id=repo.batch_id,
  397. settings=_settings(),
  398. platforms=("douyin",),
  399. search_limit=2,
  400. display_limit=1,
  401. adapter_factory=lambda platform: adapter,
  402. media_stabilizer=lambda **kwargs: (_ for _ in ()).throw(AssertionError("media should be skipped")),
  403. classifier=lambda **kwargs: (_ for _ in ()).throw(AssertionError("classifier should be skipped")),
  404. rate_limiter_factory=lambda platform: object(),
  405. )
  406. assert result.done == 1
  407. assert repo.items[0].content_mode == "video_post"
  408. assert repo.items[0].metadata["video_url_missing"] is True
  409. assert repo.classifications[0].label == "video_missing"
  410. def test_run_batch_attaches_existing_unique_key_without_fetching_detail_or_classifying():
  411. repo = FakeRepo()
  412. adapter = FakeAdapter()
  413. unique_key = "wx:MzUxMDg4NDY1Ng==:2247484898:2:5069e65919f414b96b7aec53671dc1b5"
  414. repo.existing_by_unique_key[unique_key] = CandidateItem(
  415. id=uuid4(),
  416. platform="weixin",
  417. platform_item_id="wx1",
  418. unique_key=unique_key,
  419. title="历史帖子",
  420. status="candidate",
  421. )
  422. result = run_batch(
  423. repo,
  424. batch_id=repo.batch_id,
  425. settings=_settings(),
  426. platforms=("weixin",),
  427. search_limit=2,
  428. display_limit=1,
  429. adapter_factory=lambda platform: adapter,
  430. media_stabilizer=lambda **kwargs: (_ for _ in ()).throw(AssertionError("media should be skipped")),
  431. classifier=lambda **kwargs: (_ for _ in ()).throw(AssertionError("classifier should be skipped")),
  432. rate_limiter_factory=lambda platform: object(),
  433. )
  434. assert result.done == 1
  435. assert adapter.search_calls == 1
  436. assert adapter.detail_calls == 0
  437. assert repo.items == []
  438. assert repo.media == []
  439. assert repo.classifications == []
  440. assert repo.attached_items[0].job_id == repo.jobs[0].id
  441. assert repo.attached_items[0].query_id == repo.query.id
  442. assert repo.attached_items[0].metadata["acquisition_match_status"] == "existing"
  443. assert repo.attached_items[0].metadata["matched_unique_key"] == unique_key
  444. def test_run_batch_records_douyin_provider_in_metadata_and_source_payload():
  445. repo = FakeRepo()
  446. adapter = DouyinProviderAdapter()
  447. result = run_batch(
  448. repo,
  449. batch_id=repo.batch_id,
  450. settings=_settings(),
  451. platforms=("douyin",),
  452. search_limit=1,
  453. display_limit=1,
  454. adapter_factory=lambda platform: adapter,
  455. media_stabilizer=lambda **kwargs: [
  456. StabilizedMedia(
  457. media_type="video",
  458. source_url="https://video.test/761.mp4",
  459. cdn_url="https://cdn.test/761.mp4",
  460. position=1,
  461. )
  462. ],
  463. classifier=lambda **kwargs: ClassificationResult(
  464. is_creation_knowledge=True,
  465. label="creation",
  466. confidence=0.9,
  467. reason="ok",
  468. ),
  469. rate_limiter_factory=lambda platform: object(),
  470. )
  471. assert result.done == 1
  472. item = repo.items[0]
  473. assert item.metadata["search_provider"] == "aiddit"
  474. assert item.metadata["detail_provider"] == "aiddit"
  475. assert item.source_payload["candidate"]["provider"] == "aiddit"
  476. assert item.source_payload["candidate"]["raw"]["original"] == {"aweme_id": "761"}
  477. assert item.source_payload["detail"]["detail_provider"] == "aiddit"
  478. def test_run_batch_records_duplicate_search_provider_without_fetching_detail():
  479. repo = FakeRepo()
  480. adapter = DouyinProviderAdapter()
  481. unique_key = "dy:761"
  482. repo.existing_by_unique_key[unique_key] = CandidateItem(
  483. id=uuid4(),
  484. platform="douyin",
  485. platform_item_id="761",
  486. unique_key=unique_key,
  487. title="历史视频",
  488. status="candidate",
  489. )
  490. result = run_batch(
  491. repo,
  492. batch_id=repo.batch_id,
  493. settings=_settings(),
  494. platforms=("douyin",),
  495. search_limit=1,
  496. display_limit=1,
  497. adapter_factory=lambda platform: adapter,
  498. media_stabilizer=lambda **kwargs: (_ for _ in ()).throw(AssertionError("media should be skipped")),
  499. classifier=lambda **kwargs: (_ for _ in ()).throw(AssertionError("classifier should be skipped")),
  500. rate_limiter_factory=lambda platform: object(),
  501. )
  502. assert result.done == 1
  503. assert adapter.detail_calls == 0
  504. assert repo.attached_items[0].metadata["search_provider"] == "aiddit"
  505. assert repo.attached_items[0].metadata["matched_candidate"]["provider"] == "aiddit"
  506. def test_run_batch_requests_second_page_when_first_page_creation_ratio_reaches_threshold():
  507. repo = FakeRepo()
  508. adapter = PagedAdapter(
  509. pages=[
  510. _paged_search_page(1, ["a", "b"], has_more=True, next_cursor="2"),
  511. _paged_search_page(2, ["c"], has_more=True, next_cursor="3"),
  512. _paged_search_page(3, ["d"]),
  513. ],
  514. details={source_id: _detail(source_id) for source_id in ["a", "b", "c", "d"]},
  515. )
  516. result = run_batch(
  517. repo,
  518. batch_id=repo.batch_id,
  519. settings=_settings(),
  520. platforms=("douyin",),
  521. search_limit=10,
  522. display_limit=1,
  523. adapter_factory=lambda platform: adapter,
  524. media_stabilizer=lambda **kwargs: [],
  525. classifier=_classification_by_title({"a", "c"}),
  526. rate_limiter_factory=lambda platform: object(),
  527. )
  528. pagination = repo.updated_jobs[-1].metadata["pagination"]
  529. assert result.done == 1
  530. assert adapter.pages_requested == 2
  531. assert pagination["first_page_classified_count"] == 2
  532. assert pagination["first_page_creation_count"] == 1
  533. assert pagination["first_page_creation_ratio"] == 0.5
  534. assert pagination["requested_second_page"] is True
  535. assert pagination["stop_reason"] == "max_pages_reached"
  536. assert len(pagination["pages"]) == 2
  537. def test_run_batch_does_not_request_second_page_below_threshold():
  538. repo = FakeRepo()
  539. adapter = PagedAdapter(
  540. pages=[
  541. _paged_search_page(1, ["a", "b"], has_more=True, next_cursor="2"),
  542. _paged_search_page(2, ["c"]),
  543. ],
  544. details={source_id: _detail(source_id) for source_id in ["a", "b", "c"]},
  545. )
  546. run_batch(
  547. repo,
  548. batch_id=repo.batch_id,
  549. settings=_settings(),
  550. platforms=("douyin",),
  551. search_limit=10,
  552. display_limit=1,
  553. adapter_factory=lambda platform: adapter,
  554. media_stabilizer=lambda **kwargs: [],
  555. classifier=_classification_by_title(set()),
  556. rate_limiter_factory=lambda platform: object(),
  557. )
  558. pagination = repo.updated_jobs[-1].metadata["pagination"]
  559. assert adapter.pages_requested == 1
  560. assert pagination["first_page_creation_ratio"] == 0.0
  561. assert pagination["requested_second_page"] is False
  562. assert pagination["stop_reason"] == "below_threshold"
  563. def test_run_batch_does_not_request_second_page_when_no_classified_candidates():
  564. repo = FakeRepo()
  565. adapter = PagedAdapter(
  566. pages=[
  567. _paged_search_page(1, ["bad"], has_more=True, next_cursor="2"),
  568. _paged_search_page(2, ["c"]),
  569. ],
  570. details={"bad": RuntimeError("detail failed"), "c": _detail("c")},
  571. )
  572. run_batch(
  573. repo,
  574. batch_id=repo.batch_id,
  575. settings=_settings(),
  576. platforms=("douyin",),
  577. search_limit=10,
  578. display_limit=1,
  579. adapter_factory=lambda platform: adapter,
  580. media_stabilizer=lambda **kwargs: [],
  581. classifier=_classification_by_title({"c"}),
  582. rate_limiter_factory=lambda platform: object(),
  583. )
  584. pagination = repo.updated_jobs[-1].metadata["pagination"]
  585. assert adapter.pages_requested == 1
  586. assert pagination["first_page_classified_count"] == 0
  587. assert pagination["requested_second_page"] is False
  588. assert pagination["stop_reason"] == "no_classified_candidates"
  589. def test_run_batch_excludes_duplicate_and_skipped_items_from_pagination_denominator():
  590. repo = FakeRepo()
  591. repo.existing_by_unique_key["dy:dup"] = CandidateItem(
  592. id=uuid4(),
  593. platform="douyin",
  594. platform_item_id="dup",
  595. unique_key="dy:dup",
  596. status="candidate",
  597. )
  598. adapter = PagedAdapter(
  599. pages=[
  600. _paged_search_page(
  601. 1,
  602. ["dup", "unsupported", "missing_video", "good"],
  603. has_more=True,
  604. next_cursor="2",
  605. ),
  606. _paged_search_page(2, ["next"]),
  607. ],
  608. details={
  609. "unsupported": _detail("unsupported", content_mode="unsupported", image_urls=[]),
  610. "missing_video": _detail("missing_video", content_mode="video_post", image_urls=[]),
  611. "good": _detail("good"),
  612. "next": _detail("next"),
  613. },
  614. )
  615. run_batch(
  616. repo,
  617. batch_id=repo.batch_id,
  618. settings=_settings(),
  619. platforms=("douyin",),
  620. search_limit=10,
  621. display_limit=1,
  622. adapter_factory=lambda platform: adapter,
  623. media_stabilizer=lambda **kwargs: [],
  624. classifier=_classification_by_title({"good", "next"}),
  625. rate_limiter_factory=lambda platform: object(),
  626. )
  627. pagination = repo.updated_jobs[-1].metadata["pagination"]
  628. assert adapter.pages_requested == 2
  629. assert pagination["first_page_classified_count"] == 1
  630. assert pagination["first_page_creation_count"] == 1
  631. assert pagination["first_page_creation_ratio"] == 1.0
  632. assert pagination["requested_second_page"] is True
  633. def test_run_batch_threshold_is_configurable():
  634. repo = FakeRepo()
  635. adapter = PagedAdapter(
  636. pages=[
  637. _paged_search_page(1, ["a", "b"], has_more=True, next_cursor="2"),
  638. _paged_search_page(2, ["c"]),
  639. ],
  640. details={source_id: _detail(source_id) for source_id in ["a", "b", "c"]},
  641. )
  642. run_batch(
  643. repo,
  644. batch_id=repo.batch_id,
  645. settings=_settings(),
  646. platforms=("douyin",),
  647. search_limit=10,
  648. display_limit=1,
  649. adapter_factory=lambda platform: adapter,
  650. media_stabilizer=lambda **kwargs: [],
  651. classifier=_classification_by_title({"a"}),
  652. rate_limiter_factory=lambda platform: object(),
  653. pagination_creation_threshold=0.75,
  654. )
  655. pagination = repo.updated_jobs[-1].metadata["pagination"]
  656. assert adapter.pages_requested == 1
  657. assert pagination["creation_threshold"] == 0.75
  658. assert pagination["stop_reason"] == "below_threshold"
  659. def test_run_batch_skip_done_does_not_call_platform():
  660. repo = FakeRepo(job_status="done")
  661. adapter = FakeAdapter()
  662. result = run_batch(
  663. repo,
  664. batch_id=repo.batch_id,
  665. settings=_settings(),
  666. platforms=("weixin",),
  667. adapter_factory=lambda platform: adapter,
  668. )
  669. assert result.skipped == 1
  670. assert adapter.search_calls == 0
  671. assert repo.updated_jobs == []
  672. def test_run_batch_marks_partial_when_some_details_fail():
  673. repo = FakeRepo()
  674. adapter = PartialAdapter()
  675. result = run_batch(
  676. repo,
  677. batch_id=repo.batch_id,
  678. settings=_settings(),
  679. platforms=("weixin",),
  680. search_limit=2,
  681. display_limit=2,
  682. adapter_factory=lambda platform: adapter,
  683. media_stabilizer=lambda **kwargs: [],
  684. classifier=lambda **kwargs: ClassificationResult(
  685. is_creation_knowledge=True,
  686. label="creation",
  687. confidence=1.0,
  688. reason="ok",
  689. ),
  690. rate_limiter_factory=lambda platform: object(),
  691. )
  692. assert result.partial == 1
  693. assert result.done == 0
  694. assert result.failed == 0
  695. assert repo.updated_jobs[-1].status == "partial"
  696. assert repo.updated_jobs[-1].metadata["display_count"] == 1
  697. assert repo.updated_jobs[-1].metadata["errors"] == ["detail boom"]
  698. def test_run_batch_marks_failed_when_search_fails_before_any_item():
  699. repo = FakeRepo()
  700. adapter = FailingSearchAdapter()
  701. result = run_batch(
  702. repo,
  703. batch_id=repo.batch_id,
  704. settings=_settings(),
  705. platforms=("weixin",),
  706. adapter_factory=lambda platform: adapter,
  707. rate_limiter_factory=lambda platform: object(),
  708. )
  709. assert result.failed == 1
  710. assert result.done == 0
  711. assert repo.updated_jobs[-1].status == "failed"
  712. assert "search boom" in (repo.updated_jobs[-1].error_message or "")
  713. def test_formal_runner_does_not_import_legacy_store():
  714. source = __import__("pathlib").Path("acquisition/runner.py").read_text(encoding="utf-8")
  715. assert "acquisition.store" not in source
  716. assert "creation_demo.json" not in source