|
|
@@ -2,7 +2,7 @@
|
|
|
|
|
|
接口(实测 2026-06-26,host = settings.aiddit_crawler_base_url):
|
|
|
小红书 POST /crawler/xiao_hong_shu/keyword
|
|
|
- body{keyword, content_type:"图文"|"视频", sort_type:"综合", publish_time:"", cursor:""}
|
|
|
+ body{keyword, sort_type:"综合", publish_time:"", cursor:""}
|
|
|
→ {code:0, data:{has_more, next_cursor, data:[{id:<content_id>, note_card:{...}}]}}
|
|
|
抖音 POST /crawler/dou_yin/keyword
|
|
|
body{keyword, content_type:"视频", sort_type:"综合排序", publish_time:"不限",
|
|
|
@@ -15,6 +15,7 @@ parse_search_response 纯函数可离线测;search_keyword 负责 HTTP + 翻
|
|
|
"""
|
|
|
from __future__ import annotations
|
|
|
|
|
|
+from dataclasses import dataclass, field
|
|
|
import re
|
|
|
from typing import Any, Optional
|
|
|
from urllib.parse import urljoin
|
|
|
@@ -22,7 +23,12 @@ from urllib.parse import urljoin
|
|
|
import httpx
|
|
|
|
|
|
from core.config import Settings
|
|
|
-from acquisition.crawler import RateLimiter
|
|
|
+from core.text_limits import TITLE_FALLBACK_MAX_CHARS, clip_text
|
|
|
+from acquisition.crawler import (
|
|
|
+ CRAWLER_MAX_INTERVAL_SECONDS,
|
|
|
+ CRAWLER_MIN_INTERVAL_SECONDS,
|
|
|
+ RateLimiter,
|
|
|
+)
|
|
|
|
|
|
# 每平台的搜索参数差异(path / 条目 id 字段 / 排序 / 发布时间 / 起始 cursor / account_id)
|
|
|
PLATFORM_SEARCH = {
|
|
|
@@ -43,9 +49,77 @@ class SearchError(RuntimeError):
|
|
|
pass
|
|
|
|
|
|
|
|
|
-def parse_search_response(response: Any, *, platform: str = "xiaohongshu") -> tuple[list[str], bool, str]:
|
|
|
- """信封 {code,msg,data:{has_more,next_cursor,data:[...]}} → (content_ids, has_more, next_cursor)。
|
|
|
- 条目 id 字段按平台取(小红书 id / 抖音 aweme_id)。code!=0 抛 SearchError。"""
|
|
|
+@dataclass(frozen=True)
|
|
|
+class SearchHit:
|
|
|
+ content_id: str
|
|
|
+ provider: str = ""
|
|
|
+ raw: dict[str, Any] = field(default_factory=dict)
|
|
|
+ request_payload: dict[str, Any] = field(default_factory=dict)
|
|
|
+
|
|
|
+
|
|
|
+@dataclass(frozen=True)
|
|
|
+class SearchPage:
|
|
|
+ rows: list[Any]
|
|
|
+ raw_count: int
|
|
|
+ cursor: str = ""
|
|
|
+ next_cursor: str = ""
|
|
|
+ has_more: bool = False
|
|
|
+ provider: str = ""
|
|
|
+ request_payload: dict[str, Any] = field(default_factory=dict)
|
|
|
+ raw_metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
+
|
|
|
+
|
|
|
+DOUYIN_PROVIDERS = ("piaoquantv", "aiddit")
|
|
|
+
|
|
|
+
|
|
|
+def _douyin_cursor0(provider: str) -> str:
|
|
|
+ return "" if provider == "aiddit" else "0"
|
|
|
+
|
|
|
+
|
|
|
+def _douyin_base_url(settings: Settings, provider: str) -> str:
|
|
|
+ if provider == "piaoquantv":
|
|
|
+ return settings.piaoquantv_douyin_base_url
|
|
|
+ if provider == "aiddit":
|
|
|
+ return settings.aiddit_crawler_base_url
|
|
|
+ raise SearchError(f"unsupported douyin provider: {provider}")
|
|
|
+
|
|
|
+
|
|
|
+def _douyin_search_body(
|
|
|
+ keyword: str,
|
|
|
+ *,
|
|
|
+ provider: str,
|
|
|
+ cursor: str,
|
|
|
+ settings: Settings,
|
|
|
+ content_type: str | None,
|
|
|
+ sort_type: str | None,
|
|
|
+ publish_time: str | None,
|
|
|
+ account_id: str | None,
|
|
|
+) -> dict[str, Any]:
|
|
|
+ if provider == "piaoquantv":
|
|
|
+ body: dict[str, Any] = {
|
|
|
+ "keyword": keyword,
|
|
|
+ "sort_type": sort_type or "综合排序",
|
|
|
+ "publish_time": publish_time or "不限",
|
|
|
+ "cursor": cursor,
|
|
|
+ }
|
|
|
+ if content_type:
|
|
|
+ body["content_type"] = content_type
|
|
|
+ body["account_id"] = account_id or settings.piaoquantv_douyin_account_id
|
|
|
+ body["cookie_batch"] = settings.piaoquantv_douyin_cookie_batch
|
|
|
+ return body
|
|
|
+ if provider == "aiddit":
|
|
|
+ body = {
|
|
|
+ "keyword": keyword,
|
|
|
+ "cursor": cursor,
|
|
|
+ "sort_type": sort_type or "最多点赞",
|
|
|
+ }
|
|
|
+ if content_type:
|
|
|
+ body["content_type"] = content_type
|
|
|
+ return body
|
|
|
+ raise SearchError(f"unsupported douyin provider: {provider}")
|
|
|
+
|
|
|
+
|
|
|
+def _parse_search_items(response: Any, *, platform: str = "xiaohongshu") -> tuple[list[dict[str, Any]], int, bool, str]:
|
|
|
if not isinstance(response, dict):
|
|
|
raise SearchError("bad_response: not a dict")
|
|
|
code = response.get("code")
|
|
|
@@ -53,16 +127,180 @@ def parse_search_response(response: Any, *, platform: str = "xiaohongshu") -> tu
|
|
|
raise SearchError(f"business_error: code={code} msg={response.get('msg')}")
|
|
|
data = response.get("data") or {}
|
|
|
items = data.get("data") or []
|
|
|
+ if not isinstance(items, list):
|
|
|
+ raise SearchError("bad_response: data.data is not a list")
|
|
|
+ id_key = (PLATFORM_SEARCH.get(platform) or {}).get("id_key", "id")
|
|
|
+ valid = [it for it in items if isinstance(it, dict) and it.get(id_key)]
|
|
|
+ return valid, len(items), bool(data.get("has_more")), str(data.get("next_cursor") or "")
|
|
|
+
|
|
|
+
|
|
|
+def parse_search_response(response: Any, *, platform: str = "xiaohongshu") -> tuple[list[str], bool, str]:
|
|
|
+ """信封 {code,msg,data:{has_more,next_cursor,data:[...]}} → (content_ids, has_more, next_cursor)。
|
|
|
+ 条目 id 字段按平台取(小红书 id / 抖音 aweme_id)。code!=0 抛 SearchError。"""
|
|
|
id_key = (PLATFORM_SEARCH.get(platform) or {}).get("id_key", "id")
|
|
|
- ids = [it.get(id_key) for it in items if isinstance(it, dict) and it.get(id_key)]
|
|
|
- return ids, bool(data.get("has_more")), str(data.get("next_cursor") or "")
|
|
|
+ items, _, has_more, next_cursor = _parse_search_items(response, platform=platform)
|
|
|
+ ids = [it[id_key] for it in items]
|
|
|
+ return ids, has_more, next_cursor
|
|
|
+
|
|
|
+
|
|
|
+def search_douyin_page(
|
|
|
+ keyword: str,
|
|
|
+ *,
|
|
|
+ provider: str = "piaoquantv",
|
|
|
+ cursors: dict[str, str] | None = None,
|
|
|
+ content_type: str | None = "视频",
|
|
|
+ sort_type: Optional[str] = None,
|
|
|
+ publish_time: Optional[str] = None,
|
|
|
+ account_id: Optional[str] = None,
|
|
|
+ limit: int = 10,
|
|
|
+ settings: Optional[Settings] = None,
|
|
|
+ http_client: Any = None,
|
|
|
+ rate_limiter: Optional[RateLimiter] = None,
|
|
|
+ env_file: str = ".env",
|
|
|
+) -> SearchPage:
|
|
|
+ """Fetch one Douyin page, using piaoquantv first and AIDDIT as fallback."""
|
|
|
+ settings = settings or Settings.from_env(env_file)
|
|
|
+ rate_limiter = rate_limiter or RateLimiter(
|
|
|
+ min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
|
|
|
+ max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
|
|
|
+ )
|
|
|
+ owns_client = http_client is None
|
|
|
+ client = http_client or httpx.Client()
|
|
|
+ cursors = dict(cursors or {p: _douyin_cursor0(p) for p in DOUYIN_PROVIDERS})
|
|
|
+ provider_order = [provider] + [p for p in DOUYIN_PROVIDERS if p != provider]
|
|
|
+ errors: list[str] = []
|
|
|
+ try:
|
|
|
+ for attempt_provider in provider_order:
|
|
|
+ cursor = cursors.get(attempt_provider, _douyin_cursor0(attempt_provider))
|
|
|
+ body = _douyin_search_body(
|
|
|
+ keyword,
|
|
|
+ provider=attempt_provider,
|
|
|
+ cursor=cursor,
|
|
|
+ settings=settings,
|
|
|
+ content_type=content_type,
|
|
|
+ sort_type=sort_type,
|
|
|
+ publish_time=publish_time,
|
|
|
+ account_id=account_id,
|
|
|
+ )
|
|
|
+ rate_limiter.wait("douyin")
|
|
|
+ url = urljoin(_douyin_base_url(settings, attempt_provider), PLATFORM_SEARCH["douyin"]["path"])
|
|
|
+ try:
|
|
|
+ resp = client.post(
|
|
|
+ url,
|
|
|
+ json=body,
|
|
|
+ headers={"Content-Type": "application/json"},
|
|
|
+ timeout=settings.crawler_timeout,
|
|
|
+ )
|
|
|
+ resp.raise_for_status()
|
|
|
+ data = resp.json()
|
|
|
+ items, raw_count, has_more, next_cursor = _parse_search_items(data, platform="douyin")
|
|
|
+ except httpx.HTTPError as exc:
|
|
|
+ errors.append(f"{attempt_provider}: http_error: {exc}")
|
|
|
+ continue
|
|
|
+ except ValueError:
|
|
|
+ errors.append(f"{attempt_provider}: bad_json")
|
|
|
+ continue
|
|
|
+ except SearchError as exc:
|
|
|
+ errors.append(f"{attempt_provider}: {exc}")
|
|
|
+ continue
|
|
|
+ if not items:
|
|
|
+ errors.append(f"{attempt_provider}: empty_aweme_id")
|
|
|
+ continue
|
|
|
+
|
|
|
+ id_key = PLATFORM_SEARCH["douyin"]["id_key"]
|
|
|
+ hits = [
|
|
|
+ SearchHit(
|
|
|
+ content_id=str(item[id_key]),
|
|
|
+ provider=attempt_provider,
|
|
|
+ raw=item,
|
|
|
+ request_payload=dict(body),
|
|
|
+ )
|
|
|
+ for item in items[:limit]
|
|
|
+ ]
|
|
|
+ next_cursors = dict(cursors)
|
|
|
+ if has_more and next_cursor and next_cursor != cursor:
|
|
|
+ next_cursors[attempt_provider] = next_cursor
|
|
|
+ return SearchPage(
|
|
|
+ rows=hits,
|
|
|
+ raw_count=raw_count,
|
|
|
+ cursor=cursor,
|
|
|
+ next_cursor=next_cursor,
|
|
|
+ has_more=has_more,
|
|
|
+ provider=attempt_provider,
|
|
|
+ request_payload=dict(body),
|
|
|
+ raw_metadata={"cursors": next_cursors, "errors": errors},
|
|
|
+ )
|
|
|
+ finally:
|
|
|
+ if owns_client:
|
|
|
+ client.close()
|
|
|
+ raise SearchError("; ".join(errors) or "douyin_search_failed")
|
|
|
+
|
|
|
+
|
|
|
+def search_douyin_hits(
|
|
|
+ keyword: str,
|
|
|
+ *,
|
|
|
+ content_type: str | None = "视频",
|
|
|
+ sort_type: Optional[str] = None,
|
|
|
+ publish_time: Optional[str] = None,
|
|
|
+ account_id: Optional[str] = None,
|
|
|
+ limit: int = 5,
|
|
|
+ settings: Optional[Settings] = None,
|
|
|
+ http_client: Any = None,
|
|
|
+ rate_limiter: Optional[RateLimiter] = None,
|
|
|
+ env_file: str = ".env",
|
|
|
+ max_pages: int = 10,
|
|
|
+) -> list[SearchHit]:
|
|
|
+ """Douyin search with piaoquantv primary and AIDDIT fallback per cursor."""
|
|
|
+ settings = settings or Settings.from_env(env_file)
|
|
|
+ rate_limiter = rate_limiter or RateLimiter(
|
|
|
+ min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
|
|
|
+ max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
|
|
|
+ )
|
|
|
+ owns_client = http_client is None
|
|
|
+ client = http_client or httpx.Client()
|
|
|
+ out: list[SearchHit] = []
|
|
|
+ seen: set[str] = set()
|
|
|
+ provider = "piaoquantv"
|
|
|
+ cursors = {p: _douyin_cursor0(p) for p in DOUYIN_PROVIDERS}
|
|
|
+ try:
|
|
|
+ for _ in range(max_pages):
|
|
|
+ page = search_douyin_page(
|
|
|
+ keyword,
|
|
|
+ provider=provider,
|
|
|
+ cursors=cursors,
|
|
|
+ content_type=content_type,
|
|
|
+ sort_type=sort_type,
|
|
|
+ publish_time=publish_time,
|
|
|
+ account_id=account_id,
|
|
|
+ limit=limit,
|
|
|
+ settings=settings,
|
|
|
+ http_client=client,
|
|
|
+ rate_limiter=rate_limiter,
|
|
|
+ env_file=env_file,
|
|
|
+ )
|
|
|
+ page_hits = list(page.rows)
|
|
|
+ provider = page.provider
|
|
|
+ current_cursor = cursors.get(provider, _douyin_cursor0(provider))
|
|
|
+ for hit in page_hits:
|
|
|
+ if hit.content_id not in seen:
|
|
|
+ seen.add(hit.content_id)
|
|
|
+ out.append(hit)
|
|
|
+ if len(out) >= limit:
|
|
|
+ return out
|
|
|
+ if not page.has_more or not page.next_cursor or page.next_cursor == current_cursor:
|
|
|
+ break
|
|
|
+ cursors = dict(page.raw_metadata.get("cursors") or cursors)
|
|
|
+ finally:
|
|
|
+ if owns_client:
|
|
|
+ client.close()
|
|
|
+ return out[:limit]
|
|
|
|
|
|
|
|
|
def search_keyword(
|
|
|
keyword: str,
|
|
|
*,
|
|
|
platform: str = "xiaohongshu",
|
|
|
- content_type: str = "图文",
|
|
|
+ content_type: str | None = "图文",
|
|
|
sort_type: Optional[str] = None, # None → 平台默认(小红书:综合 / 抖音:综合排序)
|
|
|
publish_time: Optional[str] = None, # None → 平台默认(抖音:不限)
|
|
|
account_id: Optional[str] = None, # None → 平台默认(抖音:771431222;小红书无)
|
|
|
@@ -80,13 +318,33 @@ def search_keyword(
|
|
|
if not cfg:
|
|
|
raise SearchError(f"unsupported platform: {platform}")
|
|
|
settings = settings or Settings.from_env(env_file)
|
|
|
- sort_type = cfg["sort_type"] if sort_type is None else sort_type
|
|
|
- publish_time = cfg["publish_time"] if publish_time is None else publish_time
|
|
|
is_douyin = platform == "douyin"
|
|
|
if account_id is None:
|
|
|
account_id = settings.piaoquantv_douyin_account_id if is_douyin else cfg["account_id"]
|
|
|
+ if is_douyin:
|
|
|
+ return [
|
|
|
+ hit.content_id
|
|
|
+ for hit in search_douyin_hits(
|
|
|
+ keyword,
|
|
|
+ content_type=content_type,
|
|
|
+ sort_type=sort_type,
|
|
|
+ publish_time=publish_time,
|
|
|
+ account_id=account_id,
|
|
|
+ limit=limit,
|
|
|
+ settings=settings,
|
|
|
+ http_client=http_client,
|
|
|
+ rate_limiter=rate_limiter,
|
|
|
+ env_file=env_file,
|
|
|
+ max_pages=max_pages,
|
|
|
+ )
|
|
|
+ ]
|
|
|
+ sort_type = cfg["sort_type"] if sort_type is None else sort_type
|
|
|
+ publish_time = cfg["publish_time"] if publish_time is None else publish_time
|
|
|
base_url = settings.piaoquantv_douyin_base_url if is_douyin else settings.aiddit_crawler_base_url
|
|
|
- rate_limiter = rate_limiter or RateLimiter()
|
|
|
+ rate_limiter = rate_limiter or RateLimiter(
|
|
|
+ min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
|
|
|
+ max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
|
|
|
+ )
|
|
|
|
|
|
owns_client = http_client is None
|
|
|
client = http_client or httpx.Client()
|
|
|
@@ -97,8 +355,10 @@ def search_keyword(
|
|
|
for _ in range(max_pages):
|
|
|
rate_limiter.wait(f"{platform}_keyword")
|
|
|
url = urljoin(base_url, cfg["path"])
|
|
|
- body = {"keyword": keyword, "content_type": content_type,
|
|
|
+ body = {"keyword": keyword,
|
|
|
"sort_type": sort_type, "publish_time": publish_time, "cursor": cursor}
|
|
|
+ if content_type:
|
|
|
+ body["content_type"] = content_type
|
|
|
if account_id:
|
|
|
body["account_id"] = account_id # 抖音必带;小红书不带
|
|
|
if is_douyin:
|
|
|
@@ -163,7 +423,10 @@ def search_weixin(
|
|
|
) -> list[dict]:
|
|
|
"""微信公众号搜索:query → 文章列表(带 url/封面,无需 detail)。失败抛 SearchError。"""
|
|
|
settings = settings or Settings.from_env(env_file)
|
|
|
- rate_limiter = rate_limiter or RateLimiter()
|
|
|
+ rate_limiter = rate_limiter or RateLimiter(
|
|
|
+ min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
|
|
|
+ max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
|
|
|
+ )
|
|
|
owns_client = http_client is None
|
|
|
client = http_client or httpx.Client()
|
|
|
try:
|
|
|
@@ -185,6 +448,60 @@ def search_weixin(
|
|
|
client.close()
|
|
|
|
|
|
|
|
|
+def search_weixin_page(
|
|
|
+ keyword: str,
|
|
|
+ *,
|
|
|
+ cursor: str = "0",
|
|
|
+ limit: int = 10,
|
|
|
+ settings: Optional[Settings] = None,
|
|
|
+ http_client: Any = None,
|
|
|
+ rate_limiter: Optional[RateLimiter] = None,
|
|
|
+ env_file: str = ".env",
|
|
|
+) -> SearchPage:
|
|
|
+ """微信公众号搜索单页:query + cursor → SearchPage。"""
|
|
|
+ settings = settings or Settings.from_env(env_file)
|
|
|
+ rate_limiter = rate_limiter or RateLimiter(
|
|
|
+ min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
|
|
|
+ max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
|
|
|
+ )
|
|
|
+ owns_client = http_client is None
|
|
|
+ client = http_client or httpx.Client()
|
|
|
+ body = {"keyword": keyword, "cursor": cursor}
|
|
|
+ try:
|
|
|
+ rate_limiter.wait("weixin_keyword")
|
|
|
+ url = urljoin(settings.aiddit_crawler_base_url, "/crawler/wei_xin/keyword")
|
|
|
+ try:
|
|
|
+ resp = client.post(url, json=body,
|
|
|
+ headers={"Content-Type": "application/json"},
|
|
|
+ timeout=settings.crawler_timeout)
|
|
|
+ resp.raise_for_status()
|
|
|
+ data = resp.json()
|
|
|
+ except httpx.HTTPError as exc:
|
|
|
+ raise SearchError(f"http_error: {exc}") from exc
|
|
|
+ except ValueError as exc:
|
|
|
+ raise SearchError("bad_json") from exc
|
|
|
+ if not isinstance(data, dict):
|
|
|
+ raise SearchError("bad_response: not a dict")
|
|
|
+ if data.get("code") not in (0, "0"):
|
|
|
+ raise SearchError(f"business_error: code={data.get('code')} msg={data.get('msg')}")
|
|
|
+ envelope = data.get("data") or {}
|
|
|
+ raw_rows = envelope.get("data") or []
|
|
|
+ if not isinstance(raw_rows, list):
|
|
|
+ raise SearchError("bad_response: data.data is not a list")
|
|
|
+ return SearchPage(
|
|
|
+ rows=parse_weixin(data, limit=limit),
|
|
|
+ raw_count=len(raw_rows),
|
|
|
+ cursor=cursor,
|
|
|
+ next_cursor=str(envelope.get("next_cursor") or ""),
|
|
|
+ has_more=bool(envelope.get("has_more")),
|
|
|
+ provider="aiddit",
|
|
|
+ request_payload=body,
|
|
|
+ )
|
|
|
+ finally:
|
|
|
+ if owns_client:
|
|
|
+ client.close()
|
|
|
+
|
|
|
+
|
|
|
def fetch_weixin_detail(
|
|
|
url: str,
|
|
|
*,
|
|
|
@@ -195,7 +512,10 @@ def fetch_weixin_detail(
|
|
|
) -> tuple[str, list[str]]:
|
|
|
"""微信公众号文章详情:content_link → (body_text, image_urls)。实测不需 token。失败抛 SearchError。"""
|
|
|
settings = settings or Settings.from_env(env_file)
|
|
|
- rate_limiter = rate_limiter or RateLimiter()
|
|
|
+ rate_limiter = rate_limiter or RateLimiter(
|
|
|
+ min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
|
|
|
+ max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
|
|
|
+ )
|
|
|
owns_client = http_client is None
|
|
|
client = http_client or httpx.Client()
|
|
|
try:
|
|
|
@@ -246,7 +566,7 @@ def parse_xiaohongshu(response: Any, *, limit: int = 5) -> list[dict]:
|
|
|
continue
|
|
|
title = (nc.get("display_title") or "").strip()
|
|
|
if not title:
|
|
|
- title = (nc.get("desc") or "").strip().split("\n")[0][:50]
|
|
|
+ title = clip_text((nc.get("desc") or "").strip().split("\n")[0], TITLE_FALLBACK_MAX_CHARS)
|
|
|
out.append({"id": cid, "title": title, "cover_url": cover,
|
|
|
"nick_name": (nc.get("user") or {}).get("nickname") or "",
|
|
|
"type": nc.get("type") or "",
|
|
|
@@ -259,7 +579,7 @@ def parse_xiaohongshu(response: Any, *, limit: int = 5) -> list[dict]:
|
|
|
def search_xiaohongshu(
|
|
|
keyword: str,
|
|
|
*,
|
|
|
- content_type: str = "图文",
|
|
|
+ content_type: str | None = None,
|
|
|
limit: int = 5,
|
|
|
settings: Optional[Settings] = None,
|
|
|
http_client: Any = None,
|
|
|
@@ -268,15 +588,20 @@ def search_xiaohongshu(
|
|
|
) -> list[dict]:
|
|
|
"""小红书搜索:query → 帖子列表(带 url/封面/标题,无需 detail;视频帖无直链)。失败抛 SearchError。"""
|
|
|
settings = settings or Settings.from_env(env_file)
|
|
|
- rate_limiter = rate_limiter or RateLimiter()
|
|
|
+ rate_limiter = rate_limiter or RateLimiter(
|
|
|
+ min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
|
|
|
+ max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
|
|
|
+ )
|
|
|
owns_client = http_client is None
|
|
|
client = http_client or httpx.Client()
|
|
|
try:
|
|
|
rate_limiter.wait("xiaohongshu_keyword")
|
|
|
url = urljoin(settings.aiddit_crawler_base_url, "/crawler/xiao_hong_shu/keyword")
|
|
|
try:
|
|
|
- resp = client.post(url, json={"keyword": keyword, "content_type": content_type,
|
|
|
- "sort_type": "综合", "publish_time": "", "cursor": ""},
|
|
|
+ body = {"keyword": keyword, "sort_type": "综合", "publish_time": "", "cursor": ""}
|
|
|
+ if content_type:
|
|
|
+ body["content_type"] = content_type
|
|
|
+ resp = client.post(url, json=body,
|
|
|
headers={"Content-Type": "application/json"},
|
|
|
timeout=settings.crawler_timeout)
|
|
|
resp.raise_for_status()
|
|
|
@@ -289,3 +614,60 @@ def search_xiaohongshu(
|
|
|
finally:
|
|
|
if owns_client:
|
|
|
client.close()
|
|
|
+
|
|
|
+
|
|
|
+def search_xiaohongshu_page(
|
|
|
+ keyword: str,
|
|
|
+ *,
|
|
|
+ cursor: str = "",
|
|
|
+ content_type: str | None = None,
|
|
|
+ limit: int = 10,
|
|
|
+ settings: Optional[Settings] = None,
|
|
|
+ http_client: Any = None,
|
|
|
+ rate_limiter: Optional[RateLimiter] = None,
|
|
|
+ env_file: str = ".env",
|
|
|
+) -> SearchPage:
|
|
|
+ """小红书搜索单页:query + cursor → SearchPage。"""
|
|
|
+ settings = settings or Settings.from_env(env_file)
|
|
|
+ rate_limiter = rate_limiter or RateLimiter(
|
|
|
+ min_interval_seconds=CRAWLER_MIN_INTERVAL_SECONDS,
|
|
|
+ max_interval_seconds=CRAWLER_MAX_INTERVAL_SECONDS,
|
|
|
+ )
|
|
|
+ owns_client = http_client is None
|
|
|
+ client = http_client or httpx.Client()
|
|
|
+ body = {"keyword": keyword, "sort_type": "综合", "publish_time": "", "cursor": cursor}
|
|
|
+ if content_type:
|
|
|
+ body["content_type"] = content_type
|
|
|
+ try:
|
|
|
+ rate_limiter.wait("xiaohongshu_keyword")
|
|
|
+ url = urljoin(settings.aiddit_crawler_base_url, "/crawler/xiao_hong_shu/keyword")
|
|
|
+ try:
|
|
|
+ resp = client.post(url, json=body,
|
|
|
+ headers={"Content-Type": "application/json"},
|
|
|
+ timeout=settings.crawler_timeout)
|
|
|
+ resp.raise_for_status()
|
|
|
+ data = resp.json()
|
|
|
+ except httpx.HTTPError as exc:
|
|
|
+ raise SearchError(f"http_error: {exc}") from exc
|
|
|
+ except ValueError as exc:
|
|
|
+ raise SearchError("bad_json") from exc
|
|
|
+ if not isinstance(data, dict):
|
|
|
+ raise SearchError("bad_response: not a dict")
|
|
|
+ if data.get("code") not in (0, "0"):
|
|
|
+ raise SearchError(f"business_error: code={data.get('code')} msg={data.get('msg')}")
|
|
|
+ envelope = data.get("data") or {}
|
|
|
+ raw_rows = envelope.get("data") or []
|
|
|
+ if not isinstance(raw_rows, list):
|
|
|
+ raise SearchError("bad_response: data.data is not a list")
|
|
|
+ return SearchPage(
|
|
|
+ rows=parse_xiaohongshu(data, limit=limit),
|
|
|
+ raw_count=len(raw_rows),
|
|
|
+ cursor=cursor,
|
|
|
+ next_cursor=str(envelope.get("next_cursor") or ""),
|
|
|
+ has_more=bool(envelope.get("has_more")),
|
|
|
+ provider="aiddit",
|
|
|
+ request_payload=body,
|
|
|
+ )
|
|
|
+ finally:
|
|
|
+ if owns_client:
|
|
|
+ client.close()
|