search.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673
  1. """关键词搜索:query → content_id 列表(翻页 + 去重 + 截断)。
  2. 接口(实测 2026-06-26,host = settings.aiddit_crawler_base_url):
  3. 小红书 POST /crawler/xiao_hong_shu/keyword
  4. body{keyword, sort_type:"综合", publish_time:"", cursor:""}
  5. → {code:0, data:{has_more, next_cursor, data:[{id:<content_id>, note_card:{...}}]}}
  6. 抖音 POST /crawler/dou_yin/keyword
  7. body{keyword, content_type:"视频", sort_type:"综合排序", publish_time:"不限",
  8. cursor:"0", account_id:"771431222"} # ← 缺 account_id 会 code 10000
  9. → {code:0, data:{has_more, next_cursor, data:[{aweme_id:<content_id>, ...}]}}
  10. 各平台请求参数 / id 字段不同,集中在 PLATFORM_SEARCH 表里。
  11. 搜索只给预览;全文/全图/视频仍需 crawler.fetch_post_detail 逐 id 取详情(detail 只需 content_id)。
  12. parse_search_response 纯函数可离线测;search_keyword 负责 HTTP + 翻页。
  13. """
  14. from __future__ import annotations
  15. from dataclasses import dataclass, field
  16. import re
  17. from typing import Any, Optional
  18. from urllib.parse import urljoin
  19. import httpx
  20. from core.config import Settings
  21. from core.text_limits import TITLE_FALLBACK_MAX_CHARS, clip_text
  22. from acquisition.crawler import (
  23. CRAWLER_MAX_INTERVAL_SECONDS,
  24. CRAWLER_MIN_INTERVAL_SECONDS,
  25. RateLimiter,
  26. )
  27. # 每平台的搜索参数差异(path / 条目 id 字段 / 排序 / 发布时间 / 起始 cursor / account_id)
  28. PLATFORM_SEARCH = {
  29. "xiaohongshu": {
  30. "path": "/crawler/xiao_hong_shu/keyword", "id_key": "id",
  31. "sort_type": "综合", "publish_time": "", "cursor0": "", "account_id": None,
  32. },
  33. "douyin": {
  34. "path": "/crawler/dou_yin/keyword", "id_key": "aweme_id",
  35. "sort_type": "综合排序", "publish_time": "不限", "cursor0": "0",
  36. # account_id 改由 settings.piaoquantv_douyin_account_id 提供(抖音走 piaoquantv 后端);缺它搜索报错
  37. "account_id": None,
  38. },
  39. }
  40. class SearchError(RuntimeError):
  41. pass
  42. @dataclass(frozen=True)
  43. class SearchHit:
  44. content_id: str
  45. provider: str = ""
  46. raw: dict[str, Any] = field(default_factory=dict)
  47. request_payload: dict[str, Any] = field(default_factory=dict)
  48. @dataclass(frozen=True)
  49. class SearchPage:
  50. rows: list[Any]
  51. raw_count: int
  52. cursor: str = ""
  53. next_cursor: str = ""
  54. has_more: bool = False
  55. provider: str = ""
  56. request_payload: dict[str, Any] = field(default_factory=dict)
  57. raw_metadata: dict[str, Any] = field(default_factory=dict)
  58. DOUYIN_PROVIDERS = ("piaoquantv", "aiddit")
  59. def _douyin_cursor0(provider: str) -> str:
  60. return "" if provider == "aiddit" else "0"
  61. def _douyin_base_url(settings: Settings, provider: str) -> str:
  62. if provider == "piaoquantv":
  63. return settings.piaoquantv_douyin_base_url
  64. if provider == "aiddit":
  65. return settings.aiddit_crawler_base_url
  66. raise SearchError(f"unsupported douyin provider: {provider}")
  67. def _douyin_search_body(
  68. keyword: str,
  69. *,
  70. provider: str,
  71. cursor: str,
  72. settings: Settings,
  73. content_type: str | None,
  74. sort_type: str | None,
  75. publish_time: str | None,
  76. account_id: str | None,
  77. ) -> dict[str, Any]:
  78. if provider == "piaoquantv":
  79. body: dict[str, Any] = {
  80. "keyword": keyword,
  81. "sort_type": sort_type or "综合排序",
  82. "publish_time": publish_time or "不限",
  83. "cursor": cursor,
  84. }
  85. if content_type:
  86. body["content_type"] = content_type
  87. body["account_id"] = account_id or settings.piaoquantv_douyin_account_id
  88. body["cookie_batch"] = settings.piaoquantv_douyin_cookie_batch
  89. return body
  90. if provider == "aiddit":
  91. body = {
  92. "keyword": keyword,
  93. "cursor": cursor,
  94. "sort_type": sort_type or "最多点赞",
  95. }
  96. if content_type:
  97. body["content_type"] = content_type
  98. return body
  99. raise SearchError(f"unsupported douyin provider: {provider}")
  100. def _parse_search_items(response: Any, *, platform: str = "xiaohongshu") -> tuple[list[dict[str, Any]], int, bool, str]:
  101. if not isinstance(response, dict):
  102. raise SearchError("bad_response: not a dict")
  103. code = response.get("code")
  104. if code not in (0, "0"):
  105. raise SearchError(f"business_error: code={code} msg={response.get('msg')}")
  106. data = response.get("data") or {}
  107. items = data.get("data") or []
  108. if not isinstance(items, list):
  109. raise SearchError("bad_response: data.data is not a list")
  110. id_key = (PLATFORM_SEARCH.get(platform) or {}).get("id_key", "id")
  111. valid = [it for it in items if isinstance(it, dict) and it.get(id_key)]
  112. return valid, len(items), bool(data.get("has_more")), str(data.get("next_cursor") or "")
  113. def parse_search_response(response: Any, *, platform: str = "xiaohongshu") -> tuple[list[str], bool, str]:
  114. """信封 {code,msg,data:{has_more,next_cursor,data:[...]}} → (content_ids, has_more, next_cursor)。
  115. 条目 id 字段按平台取(小红书 id / 抖音 aweme_id)。code!=0 抛 SearchError。"""
  116. id_key = (PLATFORM_SEARCH.get(platform) or {}).get("id_key", "id")
  117. items, _, has_more, next_cursor = _parse_search_items(response, platform=platform)
  118. ids = [it[id_key] for it in items]
  119. return ids, has_more, next_cursor
  120. def search_douyin_page(
  121. keyword: str,
  122. *,
  123. provider: str = "piaoquantv",
  124. cursors: dict[str, str] | None = None,
  125. content_type: str | None = "视频",
  126. sort_type: Optional[str] = None,
  127. publish_time: Optional[str] = None,
  128. account_id: Optional[str] = None,
  129. limit: int = 10,
  130. settings: Optional[Settings] = None,
  131. http_client: Any = None,
  132. rate_limiter: Optional[RateLimiter] = None,
  133. env_file: str = ".env",
  134. ) -> SearchPage:
  135. """Fetch one Douyin page, using piaoquantv first and AIDDIT as fallback."""
  136. settings = settings or Settings.from_env(env_file)
  137. rate_limiter = rate_limiter or RateLimiter(
  138. min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
  139. max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
  140. )
  141. owns_client = http_client is None
  142. client = http_client or httpx.Client()
  143. cursors = dict(cursors or {p: _douyin_cursor0(p) for p in DOUYIN_PROVIDERS})
  144. provider_order = [provider] + [p for p in DOUYIN_PROVIDERS if p != provider]
  145. errors: list[str] = []
  146. try:
  147. for attempt_provider in provider_order:
  148. cursor = cursors.get(attempt_provider, _douyin_cursor0(attempt_provider))
  149. body = _douyin_search_body(
  150. keyword,
  151. provider=attempt_provider,
  152. cursor=cursor,
  153. settings=settings,
  154. content_type=content_type,
  155. sort_type=sort_type,
  156. publish_time=publish_time,
  157. account_id=account_id,
  158. )
  159. rate_limiter.wait("douyin")
  160. url = urljoin(_douyin_base_url(settings, attempt_provider), PLATFORM_SEARCH["douyin"]["path"])
  161. try:
  162. resp = client.post(
  163. url,
  164. json=body,
  165. headers={"Content-Type": "application/json"},
  166. timeout=settings.crawler_timeout,
  167. )
  168. resp.raise_for_status()
  169. data = resp.json()
  170. items, raw_count, has_more, next_cursor = _parse_search_items(data, platform="douyin")
  171. except httpx.HTTPError as exc:
  172. errors.append(f"{attempt_provider}: http_error: {exc}")
  173. continue
  174. except ValueError:
  175. errors.append(f"{attempt_provider}: bad_json")
  176. continue
  177. except SearchError as exc:
  178. errors.append(f"{attempt_provider}: {exc}")
  179. continue
  180. if not items:
  181. errors.append(f"{attempt_provider}: empty_aweme_id")
  182. continue
  183. id_key = PLATFORM_SEARCH["douyin"]["id_key"]
  184. hits = [
  185. SearchHit(
  186. content_id=str(item[id_key]),
  187. provider=attempt_provider,
  188. raw=item,
  189. request_payload=dict(body),
  190. )
  191. for item in items[:limit]
  192. ]
  193. next_cursors = dict(cursors)
  194. if has_more and next_cursor and next_cursor != cursor:
  195. next_cursors[attempt_provider] = next_cursor
  196. return SearchPage(
  197. rows=hits,
  198. raw_count=raw_count,
  199. cursor=cursor,
  200. next_cursor=next_cursor,
  201. has_more=has_more,
  202. provider=attempt_provider,
  203. request_payload=dict(body),
  204. raw_metadata={"cursors": next_cursors, "errors": errors},
  205. )
  206. finally:
  207. if owns_client:
  208. client.close()
  209. raise SearchError("; ".join(errors) or "douyin_search_failed")
  210. def search_douyin_hits(
  211. keyword: str,
  212. *,
  213. content_type: str | None = "视频",
  214. sort_type: Optional[str] = None,
  215. publish_time: Optional[str] = None,
  216. account_id: Optional[str] = None,
  217. limit: int = 5,
  218. settings: Optional[Settings] = None,
  219. http_client: Any = None,
  220. rate_limiter: Optional[RateLimiter] = None,
  221. env_file: str = ".env",
  222. max_pages: int = 10,
  223. ) -> list[SearchHit]:
  224. """Douyin search with piaoquantv primary and AIDDIT fallback per cursor."""
  225. settings = settings or Settings.from_env(env_file)
  226. rate_limiter = rate_limiter or RateLimiter(
  227. min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
  228. max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
  229. )
  230. owns_client = http_client is None
  231. client = http_client or httpx.Client()
  232. out: list[SearchHit] = []
  233. seen: set[str] = set()
  234. provider = "piaoquantv"
  235. cursors = {p: _douyin_cursor0(p) for p in DOUYIN_PROVIDERS}
  236. try:
  237. for _ in range(max_pages):
  238. page = search_douyin_page(
  239. keyword,
  240. provider=provider,
  241. cursors=cursors,
  242. content_type=content_type,
  243. sort_type=sort_type,
  244. publish_time=publish_time,
  245. account_id=account_id,
  246. limit=limit,
  247. settings=settings,
  248. http_client=client,
  249. rate_limiter=rate_limiter,
  250. env_file=env_file,
  251. )
  252. page_hits = list(page.rows)
  253. provider = page.provider
  254. current_cursor = cursors.get(provider, _douyin_cursor0(provider))
  255. for hit in page_hits:
  256. if hit.content_id not in seen:
  257. seen.add(hit.content_id)
  258. out.append(hit)
  259. if len(out) >= limit:
  260. return out
  261. if not page.has_more or not page.next_cursor or page.next_cursor == current_cursor:
  262. break
  263. cursors = dict(page.raw_metadata.get("cursors") or cursors)
  264. finally:
  265. if owns_client:
  266. client.close()
  267. return out[:limit]
  268. def search_keyword(
  269. keyword: str,
  270. *,
  271. platform: str = "xiaohongshu",
  272. content_type: str | None = "图文",
  273. sort_type: Optional[str] = None, # None → 平台默认(小红书:综合 / 抖音:综合排序)
  274. publish_time: Optional[str] = None, # None → 平台默认(抖音:不限)
  275. account_id: Optional[str] = None, # None → 平台默认(抖音:771431222;小红书无)
  276. limit: int = 5,
  277. settings: Optional[Settings] = None,
  278. http_client: Any = None,
  279. rate_limiter: Optional[RateLimiter] = None,
  280. env_file: str = ".env",
  281. max_pages: int = 10,
  282. ) -> list[str]:
  283. """搜索 → content_id 列表(翻页直到够 limit 或 has_more=False;按 id 去重、截断 limit)。
  284. 平台专属参数(path/sort_type/publish_time/起始cursor/account_id)取自 PLATFORM_SEARCH,
  285. 显式入参可覆盖。复用 crawler.RateLimiter('{platform}_keyword' bucket)。失败抛 SearchError。"""
  286. cfg = PLATFORM_SEARCH.get(platform)
  287. if not cfg:
  288. raise SearchError(f"unsupported platform: {platform}")
  289. settings = settings or Settings.from_env(env_file)
  290. is_douyin = platform == "douyin"
  291. if account_id is None:
  292. account_id = settings.piaoquantv_douyin_account_id if is_douyin else cfg["account_id"]
  293. if is_douyin:
  294. return [
  295. hit.content_id
  296. for hit in search_douyin_hits(
  297. keyword,
  298. content_type=content_type,
  299. sort_type=sort_type,
  300. publish_time=publish_time,
  301. account_id=account_id,
  302. limit=limit,
  303. settings=settings,
  304. http_client=http_client,
  305. rate_limiter=rate_limiter,
  306. env_file=env_file,
  307. max_pages=max_pages,
  308. )
  309. ]
  310. sort_type = cfg["sort_type"] if sort_type is None else sort_type
  311. publish_time = cfg["publish_time"] if publish_time is None else publish_time
  312. base_url = settings.piaoquantv_douyin_base_url if is_douyin else settings.aiddit_crawler_base_url
  313. rate_limiter = rate_limiter or RateLimiter(
  314. min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
  315. max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
  316. )
  317. owns_client = http_client is None
  318. client = http_client or httpx.Client()
  319. out: list[str] = []
  320. seen: set[str] = set()
  321. cursor = cfg["cursor0"]
  322. try:
  323. for _ in range(max_pages):
  324. rate_limiter.wait(f"{platform}_keyword")
  325. url = urljoin(base_url, cfg["path"])
  326. body = {"keyword": keyword,
  327. "sort_type": sort_type, "publish_time": publish_time, "cursor": cursor}
  328. if content_type:
  329. body["content_type"] = content_type
  330. if account_id:
  331. body["account_id"] = account_id # 抖音必带;小红书不带
  332. if is_douyin:
  333. body["cookie_batch"] = settings.piaoquantv_douyin_cookie_batch # piaoquantv 抖音必带
  334. try:
  335. resp = client.post(url, json=body,
  336. headers={"Content-Type": "application/json"},
  337. timeout=settings.crawler_timeout)
  338. resp.raise_for_status()
  339. data = resp.json()
  340. except httpx.HTTPError as exc:
  341. raise SearchError(f"http_error: {exc}") from exc
  342. except ValueError as exc:
  343. raise SearchError("bad_json") from exc
  344. ids, has_more, next_cursor = parse_search_response(data, platform=platform)
  345. for cid in ids:
  346. if cid not in seen:
  347. seen.add(cid)
  348. out.append(cid)
  349. if len(out) >= limit:
  350. return out
  351. if not has_more or not next_cursor or next_cursor == cursor:
  352. break
  353. cursor = next_cursor
  354. finally:
  355. if owns_client:
  356. client.close()
  357. return out[:limit]
  358. # ---------- 微信公众号搜索(回的是带 url/封面的文章,不是纯 id,故单列)----------
  359. _EM = re.compile(r"<[^>]+>")
  360. def parse_weixin(response: Any, *, limit: int = 5) -> list[dict]:
  361. """微信搜索回包 {code,data:{data:[{title,url,cover_url,nick_name,time}]}} → 文章列表(title 去 <em> 标记)。"""
  362. if not isinstance(response, dict):
  363. raise SearchError("bad_response: not a dict")
  364. if response.get("code") not in (0, "0"):
  365. raise SearchError(f"business_error: code={response.get('code')} msg={response.get('msg')}")
  366. items = ((response.get("data") or {}).get("data")) or []
  367. out: list[dict] = []
  368. for it in items:
  369. if not isinstance(it, dict) or not it.get("url"):
  370. continue
  371. out.append({"title": _EM.sub("", it.get("title", "")).strip(),
  372. "url": it.get("url", ""), "cover_url": it.get("cover_url") or "",
  373. "nick_name": it.get("nick_name") or "", "time": it.get("time") or ""})
  374. if len(out) >= limit:
  375. break
  376. return out
  377. def search_weixin(
  378. keyword: str,
  379. *,
  380. limit: int = 5,
  381. settings: Optional[Settings] = None,
  382. http_client: Any = None,
  383. rate_limiter: Optional[RateLimiter] = None,
  384. env_file: str = ".env",
  385. ) -> list[dict]:
  386. """微信公众号搜索:query → 文章列表(带 url/封面,无需 detail)。失败抛 SearchError。"""
  387. settings = settings or Settings.from_env(env_file)
  388. rate_limiter = rate_limiter or RateLimiter(
  389. min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
  390. max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
  391. )
  392. owns_client = http_client is None
  393. client = http_client or httpx.Client()
  394. try:
  395. rate_limiter.wait("weixin_keyword")
  396. url = urljoin(settings.aiddit_crawler_base_url, "/crawler/wei_xin/keyword")
  397. try:
  398. resp = client.post(url, json={"keyword": keyword, "cursor": "0"},
  399. headers={"Content-Type": "application/json"},
  400. timeout=settings.crawler_timeout)
  401. resp.raise_for_status()
  402. data = resp.json()
  403. except httpx.HTTPError as exc:
  404. raise SearchError(f"http_error: {exc}") from exc
  405. except ValueError as exc:
  406. raise SearchError("bad_json") from exc
  407. return parse_weixin(data, limit=limit)
  408. finally:
  409. if owns_client:
  410. client.close()
  411. def search_weixin_page(
  412. keyword: str,
  413. *,
  414. cursor: str = "0",
  415. limit: int = 10,
  416. settings: Optional[Settings] = None,
  417. http_client: Any = None,
  418. rate_limiter: Optional[RateLimiter] = None,
  419. env_file: str = ".env",
  420. ) -> SearchPage:
  421. """微信公众号搜索单页:query + cursor → SearchPage。"""
  422. settings = settings or Settings.from_env(env_file)
  423. rate_limiter = rate_limiter or RateLimiter(
  424. min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
  425. max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
  426. )
  427. owns_client = http_client is None
  428. client = http_client or httpx.Client()
  429. body = {"keyword": keyword, "cursor": cursor}
  430. try:
  431. rate_limiter.wait("weixin_keyword")
  432. url = urljoin(settings.aiddit_crawler_base_url, "/crawler/wei_xin/keyword")
  433. try:
  434. resp = client.post(url, json=body,
  435. headers={"Content-Type": "application/json"},
  436. timeout=settings.crawler_timeout)
  437. resp.raise_for_status()
  438. data = resp.json()
  439. except httpx.HTTPError as exc:
  440. raise SearchError(f"http_error: {exc}") from exc
  441. except ValueError as exc:
  442. raise SearchError("bad_json") from exc
  443. if not isinstance(data, dict):
  444. raise SearchError("bad_response: not a dict")
  445. if data.get("code") not in (0, "0"):
  446. raise SearchError(f"business_error: code={data.get('code')} msg={data.get('msg')}")
  447. envelope = data.get("data") or {}
  448. raw_rows = envelope.get("data") or []
  449. if not isinstance(raw_rows, list):
  450. raise SearchError("bad_response: data.data is not a list")
  451. return SearchPage(
  452. rows=parse_weixin(data, limit=limit),
  453. raw_count=len(raw_rows),
  454. cursor=cursor,
  455. next_cursor=str(envelope.get("next_cursor") or ""),
  456. has_more=bool(envelope.get("has_more")),
  457. provider="aiddit",
  458. request_payload=body,
  459. )
  460. finally:
  461. if owns_client:
  462. client.close()
  463. def fetch_weixin_detail(
  464. url: str,
  465. *,
  466. settings: Optional[Settings] = None,
  467. http_client: Any = None,
  468. rate_limiter: Optional[RateLimiter] = None,
  469. env_file: str = ".env",
  470. ) -> tuple[str, list[str]]:
  471. """微信公众号文章详情:content_link → (body_text, image_urls)。实测不需 token。失败抛 SearchError。"""
  472. settings = settings or Settings.from_env(env_file)
  473. rate_limiter = rate_limiter or RateLimiter(
  474. min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
  475. max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
  476. )
  477. owns_client = http_client is None
  478. client = http_client or httpx.Client()
  479. try:
  480. rate_limiter.wait("weixin_detail")
  481. api = urljoin(settings.aiddit_crawler_base_url, "/crawler/wei_xin/detail")
  482. try:
  483. resp = client.post(api, json={"content_link": url},
  484. headers={"Content-Type": "application/json"},
  485. timeout=settings.crawler_timeout)
  486. resp.raise_for_status()
  487. data = resp.json()
  488. except httpx.HTTPError as exc:
  489. raise SearchError(f"http_error: {exc}") from exc
  490. except ValueError as exc:
  491. raise SearchError("bad_json") from exc
  492. if data.get("code") not in (0, "0"):
  493. raise SearchError(f"business_error: code={data.get('code')} msg={data.get('msg')}")
  494. inner = ((data.get("data") or {}).get("data")) or {}
  495. imgs = []
  496. for im in inner.get("image_url_list") or []:
  497. u = im.get("image_url") if isinstance(im, dict) else im
  498. if u:
  499. imgs.append(u)
  500. return inner.get("body_text") or "", imgs
  501. finally:
  502. if owns_client:
  503. client.close()
  504. # ---------- 小红书搜索(封面/标题/作者已在搜索回包的 note_card 里,无需 detail)----------
  505. def parse_xiaohongshu(response: Any, *, limit: int = 5) -> list[dict]:
  506. """小红书搜索回包 {code,data:{data:[{id, note_card:{image_list,display_title,desc,user,type}}]}}
  507. → 帖子列表,直接取封面(image_list[0].image_url)+标题+作者,省去逐条 detail。无视频直链。"""
  508. if not isinstance(response, dict):
  509. raise SearchError("bad_response: not a dict")
  510. if response.get("code") not in (0, "0"):
  511. raise SearchError(f"business_error: code={response.get('code')} msg={response.get('msg')}")
  512. items = ((response.get("data") or {}).get("data")) or []
  513. out: list[dict] = []
  514. for it in items:
  515. if not isinstance(it, dict):
  516. continue
  517. cid = it.get("id")
  518. nc = it.get("note_card") or {}
  519. img_list = nc.get("image_list") or []
  520. cover = (img_list[0] or {}).get("image_url") if img_list else ""
  521. if not cid or not cover:
  522. continue
  523. title = (nc.get("display_title") or "").strip()
  524. if not title:
  525. title = clip_text((nc.get("desc") or "").strip().split("\n")[0], TITLE_FALLBACK_MAX_CHARS)
  526. out.append({"id": cid, "title": title, "cover_url": cover,
  527. "nick_name": (nc.get("user") or {}).get("nickname") or "",
  528. "type": nc.get("type") or "",
  529. "url": f"https://www.xiaohongshu.com/explore/{cid}"})
  530. if len(out) >= limit:
  531. break
  532. return out
  533. def search_xiaohongshu(
  534. keyword: str,
  535. *,
  536. content_type: str | None = None,
  537. limit: int = 5,
  538. settings: Optional[Settings] = None,
  539. http_client: Any = None,
  540. rate_limiter: Optional[RateLimiter] = None,
  541. env_file: str = ".env",
  542. ) -> list[dict]:
  543. """小红书搜索:query → 帖子列表(带 url/封面/标题,无需 detail;视频帖无直链)。失败抛 SearchError。"""
  544. settings = settings or Settings.from_env(env_file)
  545. rate_limiter = rate_limiter or RateLimiter(
  546. min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
  547. max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
  548. )
  549. owns_client = http_client is None
  550. client = http_client or httpx.Client()
  551. try:
  552. rate_limiter.wait("xiaohongshu_keyword")
  553. url = urljoin(settings.aiddit_crawler_base_url, "/crawler/xiao_hong_shu/keyword")
  554. try:
  555. body = {"keyword": keyword, "sort_type": "综合", "publish_time": "", "cursor": ""}
  556. if content_type:
  557. body["content_type"] = content_type
  558. resp = client.post(url, json=body,
  559. headers={"Content-Type": "application/json"},
  560. timeout=settings.crawler_timeout)
  561. resp.raise_for_status()
  562. data = resp.json()
  563. except httpx.HTTPError as exc:
  564. raise SearchError(f"http_error: {exc}") from exc
  565. except ValueError as exc:
  566. raise SearchError("bad_json") from exc
  567. return parse_xiaohongshu(data, limit=limit)
  568. finally:
  569. if owns_client:
  570. client.close()
  571. def search_xiaohongshu_page(
  572. keyword: str,
  573. *,
  574. cursor: str = "",
  575. content_type: str | None = None,
  576. limit: int = 10,
  577. settings: Optional[Settings] = None,
  578. http_client: Any = None,
  579. rate_limiter: Optional[RateLimiter] = None,
  580. env_file: str = ".env",
  581. ) -> SearchPage:
  582. """小红书搜索单页:query + cursor → SearchPage。"""
  583. settings = settings or Settings.from_env(env_file)
  584. rate_limiter = rate_limiter or RateLimiter(
  585. min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
  586. max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
  587. )
  588. owns_client = http_client is None
  589. client = http_client or httpx.Client()
  590. body = {"keyword": keyword, "sort_type": "综合", "publish_time": "", "cursor": cursor}
  591. if content_type:
  592. body["content_type"] = content_type
  593. try:
  594. rate_limiter.wait("xiaohongshu_keyword")
  595. url = urljoin(settings.aiddit_crawler_base_url, "/crawler/xiao_hong_shu/keyword")
  596. try:
  597. resp = client.post(url, json=body,
  598. headers={"Content-Type": "application/json"},
  599. timeout=settings.crawler_timeout)
  600. resp.raise_for_status()
  601. data = resp.json()
  602. except httpx.HTTPError as exc:
  603. raise SearchError(f"http_error: {exc}") from exc
  604. except ValueError as exc:
  605. raise SearchError("bad_json") from exc
  606. if not isinstance(data, dict):
  607. raise SearchError("bad_response: not a dict")
  608. if data.get("code") not in (0, "0"):
  609. raise SearchError(f"business_error: code={data.get('code')} msg={data.get('msg')}")
  610. envelope = data.get("data") or {}
  611. raw_rows = envelope.get("data") or []
  612. if not isinstance(raw_rows, list):
  613. raise SearchError("bad_response: data.data is not a list")
  614. return SearchPage(
  615. rows=parse_xiaohongshu(data, limit=limit),
  616. raw_count=len(raw_rows),
  617. cursor=cursor,
  618. next_cursor=str(envelope.get("next_cursor") or ""),
  619. has_more=bool(envelope.get("has_more")),
  620. provider="aiddit",
  621. request_payload=body,
  622. )
  623. finally:
  624. if owns_client:
  625. client.close()