Browse Source

抖音搜索修复:补 account_id + 平台专属参数 + 解析改用 aweme_id

实测根因:抖音 /keyword 缺 account_id 必填字段 → code 10000(小红书无此问题)。
- search.py:加 PLATFORM_SEARCH 表按平台分参数(path/id字段/sort_type/publish_time/起始cursor/account_id)
  · 抖音必带 account_id=771431222、sort_type=综合排序、publish_time=不限、cursor 起始"0"
  · 抖音条目 id 取 aweme_id(小红书取 id)——否则即使接口通也解析出 0 条
- decompose.py expand_queries:sort_type/account_id 交给平台默认(不再把小红书的"综合"套到抖音)
- test_search.py:+3(抖音 aweme_id 解析 / 抖音 body 带 account_id+综合排序+cursor0 / 小红书 body 不带 account_id)
- 实测:两平台 search_keyword 均 code 0;抖音翻页 cursor 0/10/20 无重复

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
SamLee 3 weeks ago
parent
commit
a200aad2e1
3 changed files with 72 additions and 18 deletions
  1. 38 17
      creation_knowledge/integrations/search.py
  2. 2 1
      scripts/decompose.py
  3. 32 0
      tests/test_search.py

+ 38 - 17
creation_knowledge/integrations/search.py

@@ -1,13 +1,17 @@
 """关键词搜索:query → content_id 列表(翻页 + 去重 + 截断)。
 
 接口(实测 2026-06-26,host = settings.crawler_base_url):
-    POST /crawler/xiao_hong_shu/keyword
+    小红书 POST /crawler/xiao_hong_shu/keyword
       body{keyword, content_type:"图文"|"视频", sort_type:"综合", publish_time:"", cursor:""}
-      → {code:0, data:{has_more, next_cursor, data:[{id:<content_id>, note_card:{...预览...}}]}}
+      → {code:0, data:{has_more, next_cursor, data:[{id:<content_id>, note_card:{...}}]}}
+    抖音   POST /crawler/dou_yin/keyword
+      body{keyword, content_type:"视频", sort_type:"综合排序", publish_time:"不限",
+           cursor:"0", account_id:"771431222"}   # ← 缺 account_id 会 code 10000
+      → {code:0, data:{has_more, next_cursor, data:[{aweme_id:<content_id>, ...}]}}
 
-搜索只给预览;全文/全图/视频仍需 crawler.fetch_post_detail 逐 id 取详情。
+各平台请求参数 / id 字段不同,集中在 PLATFORM_SEARCH 表里。
+搜索只给预览;全文/全图/视频仍需 crawler.fetch_post_detail 逐 id 取详情(detail 只需 content_id)。
 parse_search_response 纯函数可离线测;search_keyword 负责 HTTP + 翻页。
-抖音 /keyword 实测 code:10000(疑需 cookie/鉴权)→ 抛 SearchError,由调用方跳过。
 """
 from __future__ import annotations
 
@@ -19,9 +23,18 @@ import httpx
 from creation_knowledge.config import Settings
 from creation_knowledge.integrations.crawler import RateLimiter
 
-SEARCH_PATHS = {
-    "xiaohongshu": "/crawler/xiao_hong_shu/keyword",
-    "douyin": "/crawler/dou_yin/keyword",   # TODO: 实测 code 10000,需 cookie/鉴权
+# 每平台的搜索参数差异(path / 条目 id 字段 / 排序 / 发布时间 / 起始 cursor / account_id)
+PLATFORM_SEARCH = {
+    "xiaohongshu": {
+        "path": "/crawler/xiao_hong_shu/keyword", "id_key": "id",
+        "sort_type": "综合", "publish_time": "", "cursor0": "", "account_id": None,
+    },
+    "douyin": {
+        "path": "/crawler/dou_yin/keyword", "id_key": "aweme_id",
+        "sort_type": "综合排序", "publish_time": "不限", "cursor0": "0",
+        # 对齐 ContentFindAgentNew CONTENTFIND_DOUYIN_DEFAULT_ACCOUNT_ID;缺它抖音搜索 code 10000
+        "account_id": "771431222",
+    },
 }
 
 
@@ -31,7 +44,7 @@ class SearchError(RuntimeError):
 
 def parse_search_response(response: Any, *, platform: str = "xiaohongshu") -> tuple[list[str], bool, str]:
     """信封 {code,msg,data:{has_more,next_cursor,data:[...]}} → (content_ids, has_more, next_cursor)。
-    code!=0 抛 SearchError(如抖音 10000)。"""
+    条目 id 字段按平台取(小红书 id / 抖音 aweme_id)。code!=0 抛 SearchError。"""
     if not isinstance(response, dict):
         raise SearchError("bad_response: not a dict")
     code = response.get("code")
@@ -39,7 +52,8 @@ def parse_search_response(response: Any, *, platform: str = "xiaohongshu") -> tu
         raise SearchError(f"business_error: code={code} msg={response.get('msg')}")
     data = response.get("data") or {}
     items = data.get("data") or []
-    ids = [it.get("id") for it in items if isinstance(it, dict) and it.get("id")]
+    id_key = (PLATFORM_SEARCH.get(platform) or {}).get("id_key", "id")
+    ids = [it.get(id_key) for it in items if isinstance(it, dict) and it.get(id_key)]
     return ids, bool(data.get("has_more")), str(data.get("next_cursor") or "")
 
 
@@ -48,9 +62,10 @@ def search_keyword(
     *,
     platform: str = "xiaohongshu",
     content_type: str = "图文",
-    sort_type: str = "综合",
+    sort_type: Optional[str] = None,       # None → 平台默认(小红书:综合 / 抖音:综合排序)
+    publish_time: Optional[str] = None,     # None → 平台默认(抖音:不限)
+    account_id: Optional[str] = None,       # None → 平台默认(抖音:771431222;小红书无)
     limit: int = 5,
-    publish_time: str = "",
     settings: Optional[Settings] = None,
     http_client: Any = None,
     rate_limiter: Optional[RateLimiter] = None,
@@ -58,24 +73,30 @@ def search_keyword(
     max_pages: int = 10,
 ) -> list[str]:
     """搜索 → content_id 列表(翻页直到够 limit 或 has_more=False;按 id 去重、截断 limit)。
-    复用 crawler.RateLimiter('{platform}_keyword' bucket)。失败抛 SearchError。"""
-    settings = settings or Settings.from_env(env_file)
-    path = SEARCH_PATHS.get(platform)
-    if not path:
+    平台专属参数(path/sort_type/publish_time/起始cursor/account_id)取自 PLATFORM_SEARCH,
+    显式入参可覆盖。复用 crawler.RateLimiter('{platform}_keyword' bucket)。失败抛 SearchError。"""
+    cfg = PLATFORM_SEARCH.get(platform)
+    if not cfg:
         raise SearchError(f"unsupported platform: {platform}")
+    settings = settings or Settings.from_env(env_file)
+    sort_type = cfg["sort_type"] if sort_type is None else sort_type
+    publish_time = cfg["publish_time"] if publish_time is None else publish_time
+    account_id = cfg["account_id"] if account_id is None else account_id
     rate_limiter = rate_limiter or RateLimiter()
 
     owns_client = http_client is None
     client = http_client or httpx.Client()
     out: list[str] = []
     seen: set[str] = set()
-    cursor = ""
+    cursor = cfg["cursor0"]
     try:
         for _ in range(max_pages):
             rate_limiter.wait(f"{platform}_keyword")
-            url = urljoin(settings.crawler_base_url, path)
+            url = urljoin(settings.crawler_base_url, cfg["path"])
             body = {"keyword": keyword, "content_type": content_type,
                     "sort_type": sort_type, "publish_time": publish_time, "cursor": cursor}
+            if account_id:
+                body["account_id"] = account_id   # 抖音必带;小红书不带
             try:
                 resp = client.post(url, json=body,
                                    headers={"Content-Type": "application/json"},

+ 2 - 1
scripts/decompose.py

@@ -495,7 +495,8 @@ def expand_queries(queries: list[dict], settings: Settings) -> list[dict]:
             ids = search_keyword(
                 q["query"], platform=platform,
                 content_type=q.get("content_type") or settings.search_content_type,
-                sort_type=q.get("sort_type") or settings.search_sort_type,
+                sort_type=q.get("sort_type"),       # None → 平台默认(小红书:综合 / 抖音:综合排序)
+                account_id=q.get("account_id"),     # None → 平台默认(抖音:771431222)
                 limit=q.get("limit") or settings.search_default_limit,
                 settings=settings)
         except Exception as exc:

+ 32 - 0
tests/test_search.py

@@ -88,3 +88,35 @@ def test_search_unsupported_platform():
     with pytest.raises(SearchError):
         search_keyword("x", platform="bilibili", settings=_settings(),
                        rate_limiter=_no_wait())
+
+
+def test_parse_douyin_uses_aweme_id():
+    resp = {"code": 0, "data": {"has_more": True, "next_cursor": "10",
+            "data": [{"aweme_id": "7618138722512424246"}, {"aweme_id": "7600000000000000000"}]}}
+    ids, has_more, cursor = parse_search_response(resp, platform="douyin")
+    assert ids == ["7618138722512424246", "7600000000000000000"]
+    assert has_more is True and cursor == "10"
+
+
+def test_douyin_body_has_account_id_and_params():
+    page = {"code": 0, "data": {"has_more": False, "next_cursor": "",
+            "data": [{"aweme_id": "a1"}, {"aweme_id": "a2"}]}}
+    client = _FakeClient([page])
+    out = search_keyword("科普", platform="douyin", content_type="视频",
+                         settings=_settings(), http_client=client, rate_limiter=_no_wait(), limit=10)
+    assert out == ["a1", "a2"]                         # 用 aweme_id 解析
+    body = client.calls[0]
+    assert body["account_id"] == "771431222"           # 抖音必带
+    assert body["sort_type"] == "综合排序"             # 平台默认(非小红书的"综合")
+    assert body["publish_time"] == "不限"
+    assert body["cursor"] == "0"                       # 抖音起始 cursor
+
+
+def test_xiaohongshu_body_no_account_id():
+    page = {"code": 0, "data": {"has_more": False, "next_cursor": "", "data": [{"id": "x1"}]}}
+    client = _FakeClient([page])
+    search_keyword("科普", platform="xiaohongshu", settings=_settings(),
+                   http_client=client, rate_limiter=_no_wait())
+    body = client.calls[0]
+    assert "account_id" not in body                    # 小红书不带
+    assert body["sort_type"] == "综合" and body["cursor"] == ""