|
|
@@ -1,13 +1,17 @@
|
|
|
"""关键词搜索:query → content_id 列表(翻页 + 去重 + 截断)。
|
|
|
|
|
|
接口(实测 2026-06-26,host = settings.crawler_base_url):
|
|
|
- POST /crawler/xiao_hong_shu/keyword
|
|
|
+ 小红书 POST /crawler/xiao_hong_shu/keyword
|
|
|
body{keyword, content_type:"图文"|"视频", sort_type:"综合", publish_time:"", cursor:""}
|
|
|
- → {code:0, data:{has_more, next_cursor, data:[{id:<content_id>, note_card:{...预览...}}]}}
|
|
|
+ → {code:0, data:{has_more, next_cursor, data:[{id:<content_id>, note_card:{...}}]}}
|
|
|
+ 抖音 POST /crawler/dou_yin/keyword
|
|
|
+ body{keyword, content_type:"视频", sort_type:"综合排序", publish_time:"不限",
|
|
|
+ cursor:"0", account_id:"771431222"} # ← 缺 account_id 会 code 10000
|
|
|
+ → {code:0, data:{has_more, next_cursor, data:[{aweme_id:<content_id>, ...}]}}
|
|
|
|
|
|
-搜索只给预览;全文/全图/视频仍需 crawler.fetch_post_detail 逐 id 取详情。
|
|
|
+各平台请求参数 / id 字段不同,集中在 PLATFORM_SEARCH 表里。
|
|
|
+搜索只给预览;全文/全图/视频仍需 crawler.fetch_post_detail 逐 id 取详情(detail 只需 content_id)。
|
|
|
parse_search_response 纯函数可离线测;search_keyword 负责 HTTP + 翻页。
|
|
|
-抖音 /keyword 实测 code:10000(疑需 cookie/鉴权)→ 抛 SearchError,由调用方跳过。
|
|
|
"""
|
|
|
from __future__ import annotations
|
|
|
|
|
|
@@ -19,9 +23,18 @@ import httpx
|
|
|
from creation_knowledge.config import Settings
|
|
|
from creation_knowledge.integrations.crawler import RateLimiter
|
|
|
|
|
|
-SEARCH_PATHS = {
|
|
|
- "xiaohongshu": "/crawler/xiao_hong_shu/keyword",
|
|
|
- "douyin": "/crawler/dou_yin/keyword", # TODO: 实测 code 10000,需 cookie/鉴权
|
|
|
+# 每平台的搜索参数差异(path / 条目 id 字段 / 排序 / 发布时间 / 起始 cursor / account_id)
|
|
|
+PLATFORM_SEARCH = {
|
|
|
+ "xiaohongshu": {
|
|
|
+ "path": "/crawler/xiao_hong_shu/keyword", "id_key": "id",
|
|
|
+ "sort_type": "综合", "publish_time": "", "cursor0": "", "account_id": None,
|
|
|
+ },
|
|
|
+ "douyin": {
|
|
|
+ "path": "/crawler/dou_yin/keyword", "id_key": "aweme_id",
|
|
|
+ "sort_type": "综合排序", "publish_time": "不限", "cursor0": "0",
|
|
|
+ # 对齐 ContentFindAgentNew CONTENTFIND_DOUYIN_DEFAULT_ACCOUNT_ID;缺它抖音搜索 code 10000
|
|
|
+ "account_id": "771431222",
|
|
|
+ },
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -31,7 +44,7 @@ class SearchError(RuntimeError):
|
|
|
|
|
|
def parse_search_response(response: Any, *, platform: str = "xiaohongshu") -> tuple[list[str], bool, str]:
|
|
|
"""信封 {code,msg,data:{has_more,next_cursor,data:[...]}} → (content_ids, has_more, next_cursor)。
|
|
|
- code!=0 抛 SearchError(如抖音 10000)。"""
|
|
|
+ 条目 id 字段按平台取(小红书 id / 抖音 aweme_id)。code!=0 抛 SearchError。"""
|
|
|
if not isinstance(response, dict):
|
|
|
raise SearchError("bad_response: not a dict")
|
|
|
code = response.get("code")
|
|
|
@@ -39,7 +52,8 @@ def parse_search_response(response: Any, *, platform: str = "xiaohongshu") -> tu
|
|
|
raise SearchError(f"business_error: code={code} msg={response.get('msg')}")
|
|
|
data = response.get("data") or {}
|
|
|
items = data.get("data") or []
|
|
|
- ids = [it.get("id") for it in items if isinstance(it, dict) and it.get("id")]
|
|
|
+ id_key = (PLATFORM_SEARCH.get(platform) or {}).get("id_key", "id")
|
|
|
+ ids = [it.get(id_key) for it in items if isinstance(it, dict) and it.get(id_key)]
|
|
|
return ids, bool(data.get("has_more")), str(data.get("next_cursor") or "")
|
|
|
|
|
|
|
|
|
@@ -48,9 +62,10 @@ def search_keyword(
|
|
|
*,
|
|
|
platform: str = "xiaohongshu",
|
|
|
content_type: str = "图文",
|
|
|
- sort_type: str = "综合",
|
|
|
+ sort_type: Optional[str] = None, # None → 平台默认(小红书:综合 / 抖音:综合排序)
|
|
|
+ publish_time: Optional[str] = None, # None → 平台默认(抖音:不限)
|
|
|
+ account_id: Optional[str] = None, # None → 平台默认(抖音:771431222;小红书无)
|
|
|
limit: int = 5,
|
|
|
- publish_time: str = "",
|
|
|
settings: Optional[Settings] = None,
|
|
|
http_client: Any = None,
|
|
|
rate_limiter: Optional[RateLimiter] = None,
|
|
|
@@ -58,24 +73,30 @@ def search_keyword(
|
|
|
max_pages: int = 10,
|
|
|
) -> list[str]:
|
|
|
"""搜索 → content_id 列表(翻页直到够 limit 或 has_more=False;按 id 去重、截断 limit)。
|
|
|
- 复用 crawler.RateLimiter('{platform}_keyword' bucket)。失败抛 SearchError。"""
|
|
|
- settings = settings or Settings.from_env(env_file)
|
|
|
- path = SEARCH_PATHS.get(platform)
|
|
|
- if not path:
|
|
|
+ 平台专属参数(path/sort_type/publish_time/起始cursor/account_id)取自 PLATFORM_SEARCH,
|
|
|
+ 显式入参可覆盖。复用 crawler.RateLimiter('{platform}_keyword' bucket)。失败抛 SearchError。"""
|
|
|
+ cfg = PLATFORM_SEARCH.get(platform)
|
|
|
+ if not cfg:
|
|
|
raise SearchError(f"unsupported platform: {platform}")
|
|
|
+ settings = settings or Settings.from_env(env_file)
|
|
|
+ sort_type = cfg["sort_type"] if sort_type is None else sort_type
|
|
|
+ publish_time = cfg["publish_time"] if publish_time is None else publish_time
|
|
|
+ account_id = cfg["account_id"] if account_id is None else account_id
|
|
|
rate_limiter = rate_limiter or RateLimiter()
|
|
|
|
|
|
owns_client = http_client is None
|
|
|
client = http_client or httpx.Client()
|
|
|
out: list[str] = []
|
|
|
seen: set[str] = set()
|
|
|
- cursor = ""
|
|
|
+ cursor = cfg["cursor0"]
|
|
|
try:
|
|
|
for _ in range(max_pages):
|
|
|
rate_limiter.wait(f"{platform}_keyword")
|
|
|
- url = urljoin(settings.crawler_base_url, path)
|
|
|
+ url = urljoin(settings.crawler_base_url, cfg["path"])
|
|
|
body = {"keyword": keyword, "content_type": content_type,
|
|
|
"sort_type": sort_type, "publish_time": publish_time, "cursor": cursor}
|
|
|
+ if account_id:
|
|
|
+ body["account_id"] = account_id # 抖音必带;小红书不带
|
|
|
try:
|
|
|
resp = client.post(url, json=body,
|
|
|
headers={"Content-Type": "application/json"},
|