fix douyin keyword provider fallback

.env

@@ -35,6 +35,11 @@ CONTENTFIND_DOUYIN_DEFAULT_CONTENT_TYPE=视频
 CONTENTFIND_DOUYIN_DEFAULT_SORT_TYPE=综合排序
 CONTENTFIND_DOUYIN_DEFAULT_PUBLISH_TIME=不限
 CONTENTFIND_DOUYIN_DEFAULT_CURSOR=0
+CONTENTFIND_DOUYIN_PIAOQUANTV_ACCOUNT_ID=7450041106378522636
+CONTENTFIND_DOUYIN_PIAOQUANTV_DURATION=不限
+CONTENTFIND_DOUYIN_PIAOQUANTV_COOKIE_BATCH=default
+CONTENTFIND_DOUYIN_CRAWLER_SORT_TYPE=最多点赞
+CONTENTFIND_DOUYIN_CRAWLER_CURSOR_DEFAULT=
 CONTENTFIND_DOUYIN_ACCOUNT_WORKS_DEFAULT_SORT_TYPE=最新
 CONTENTFIND_DOUYIN_MAX_RESULTS_PER_QUERY=3
 

content_agent/integrations/crawapi_http.py

@@ -249,7 +249,15 @@ def _extract_trace_refs(
 def _request_payload_summary(payload: dict[str, Any]) -> dict[str, Any]:
     summary: dict[str, Any] = {}
     for key, value in payload.items():
-        if key in {"keyword", "content_type", "sort_type", "publish_time", "cursor"}:
+        if key in {
+            "keyword",
+            "content_type",
+            "sort_type",
+            "publish_time",
+            "duration",
+            "cursor",
+            "cookie_batch",
+        }:
             summary[key] = value
         elif key.endswith("_id") or key in {"account_id", "sec_uid"}:
             summary[key] = "<redacted>"

content_agent/integrations/douyin.py

@@ -57,6 +57,11 @@ class CrawapiDouyinClient:
         default_sort_type: str = "综合排序",
         default_publish_time: str = "不限",
         default_cursor: str = "0",
+        piaoquantv_account_id: str = "",
+        piaoquantv_duration: str = "不限",
+        piaoquantv_cookie_batch: str = "default",
+        crawler_sort_type: str = "最多点赞",
+        crawler_cursor_default: str = "",
         default_account_works_sort_type: str = "最新",
         max_results_per_query: int | None = 5,
         http_client: Any | None = None,
@@ -68,6 +73,7 @@ class CrawapiDouyinClient:
         self._crawapi_base_urls = _dedupe_crawapi_base_urls(
             [("primary", self.base_url), ("candidate", self.fallback_base_url)]
         )
+        self._legacy_crawapi_base_urls = _legacy_crawapi_base_urls(self._crawapi_base_urls)
         self.keyword_path = keyword_path.lstrip("/")
         self.blogger_path = blogger_path.lstrip("/")
         self.detail_path = detail_path.lstrip("/")
@@ -78,6 +84,11 @@ class CrawapiDouyinClient:
         self.default_sort_type = default_sort_type
         self.default_publish_time = default_publish_time
         self.default_cursor = default_cursor
+        self.piaoquantv_account_id = piaoquantv_account_id or default_crawapi_account_ref
+        self.piaoquantv_duration = piaoquantv_duration
+        self.piaoquantv_cookie_batch = piaoquantv_cookie_batch
+        self.crawler_sort_type = crawler_sort_type
+        self.crawler_cursor_default = crawler_cursor_default
         self.default_account_works_sort_type = default_account_works_sort_type
         self.max_results_per_query = max_results_per_query
         self.http_client = http_client or httpx.Client(timeout=timeout_seconds)
@@ -108,6 +119,17 @@ class CrawapiDouyinClient:
             default_sort_type=_env("CONTENTFIND_DOUYIN_DEFAULT_SORT_TYPE", env, default="综合排序"),
             default_publish_time=_env("CONTENTFIND_DOUYIN_DEFAULT_PUBLISH_TIME", env, default="不限"),
             default_cursor=_env("CONTENTFIND_DOUYIN_DEFAULT_CURSOR", env, default="0"),
+            piaoquantv_account_id=_env(
+                "CONTENTFIND_DOUYIN_PIAOQUANTV_ACCOUNT_ID",
+                env,
+                default=_env("CONTENTFIND_DOUYIN_DEFAULT_ACCOUNT_ID", env, default=""),
+            ),
+            piaoquantv_duration=_env("CONTENTFIND_DOUYIN_PIAOQUANTV_DURATION", env, default="不限"),
+            piaoquantv_cookie_batch=_env(
+                "CONTENTFIND_DOUYIN_PIAOQUANTV_COOKIE_BATCH", env, default="default"
+            ),
+            crawler_sort_type=_env("CONTENTFIND_DOUYIN_CRAWLER_SORT_TYPE", env, default="最多点赞"),
+            crawler_cursor_default=_env("CONTENTFIND_DOUYIN_CRAWLER_CURSOR_DEFAULT", env, default=""),
             default_account_works_sort_type=_env(
                 "CONTENTFIND_DOUYIN_ACCOUNT_WORKS_DEFAULT_SORT_TYPE", env, default="最新"
             ),
@@ -131,18 +153,7 @@ class CrawapiDouyinClient:
         query: dict[str, Any],
         max_results_per_query: int | None,
     ) -> list[dict[str, Any]]:
-        payload = {
-            "keyword": query["search_query"],
-            "content_type": self.default_content_type,
-            "sort_type": self.default_sort_type,
-            "publish_time": self.default_publish_time,
-            "cursor": str(query.get("page_cursor") or self.default_cursor),
-            RAW_AUTHOR_ACCOUNT_KEY: self.default_crawapi_account_ref,
-        }
-        data = self._post_json(
-            self.keyword_path, payload, operation="keyword_search",
-            rate_limit_bucket=SEARCH_RATE_LIMIT_BUCKET,
-        )
+        data = self._post_keyword_json(query)
         data_block = data.get("data", {}) if isinstance(data.get("data"), dict) else {}
         items = data_block.get("data", []) if isinstance(data_block.get("data"), list) else []
         has_more = bool(data_block.get("has_more", False))
@@ -351,6 +362,52 @@ class CrawapiDouyinClient:
             http_client=self.http_client,
         )
 
+    def _post_keyword_json(self, query: dict[str, Any]) -> dict[str, Any]:
+        attempts: list[dict[str, Any]] = []
+        for index, (role, base_url) in enumerate(self._crawapi_base_urls):
+            provider = _keyword_provider_for_base_url(role, base_url)
+            payload = self._keyword_payload_for_provider(provider, query)
+            try:
+                return post_crawapi_json(
+                    http_client=self.http_client,
+                    base_url=base_url,
+                    path=self.keyword_path,
+                    payload=payload,
+                    operation="keyword_search",
+                    timeout_seconds=self.timeout_seconds,
+                    rate_limiter=self.rate_limiter,
+                    rate_limit_bucket=SEARCH_RATE_LIMIT_BUCKET,
+                    business_codes=RATE_LIMIT_BUSINESS_CODES,
+                )
+            except ContentAgentError as exc:
+                attempt = _crawapi_host_attempt_summary(role, base_url, exc, provider=provider)
+                self._raise_crawapi_error(exc, attempts + [attempt])
+            except (CrawapiBusinessError, CrawapiTransientError, RuntimeError) as exc:
+                attempt = _crawapi_host_attempt_summary(role, base_url, exc, provider=provider)
+                if index == len(self._crawapi_base_urls) - 1 or not _should_try_crawapi_candidate(exc):
+                    self._raise_crawapi_error(exc, attempts + [attempt])
+                attempts.append(attempt)
+        raise RuntimeError("crawapi keyword_search failed: no base_url configured")
+
+    def _keyword_payload_for_provider(self, provider: str, query: dict[str, Any]) -> dict[str, Any]:
+        if provider == "crawler":
+            return {
+                "content_type": self.default_content_type,
+                "keyword": query["search_query"],
+                "cursor": _query_cursor(query, self.crawler_cursor_default),
+                "sort_type": self.crawler_sort_type,
+            }
+        return {
+            RAW_AUTHOR_ACCOUNT_KEY: self.piaoquantv_account_id,
+            "keyword": query["search_query"],
+            "content_type": self.default_content_type,
+            "sort_type": self.default_sort_type,
+            "publish_time": self.default_publish_time,
+            "duration": self.piaoquantv_duration,
+            "cursor": _query_cursor(query, self.default_cursor),
+            "cookie_batch": self.piaoquantv_cookie_batch,
+        }
+
     def _post_json(
         self,
         path: str,
@@ -359,7 +416,8 @@ class CrawapiDouyinClient:
         rate_limit_bucket: str | None = None,
     ) -> dict[str, Any]:
         attempts: list[dict[str, Any]] = []
-        for index, (role, base_url) in enumerate(self._crawapi_base_urls):
+        base_urls = self._legacy_crawapi_base_urls
+        for index, (role, base_url) in enumerate(base_urls):
             try:
                 return post_crawapi_json(
                     http_client=self.http_client,
@@ -377,7 +435,7 @@ class CrawapiDouyinClient:
                 self._raise_crawapi_error(exc, attempts + [attempt])
             except (CrawapiBusinessError, CrawapiTransientError, RuntimeError) as exc:
                 attempt = _crawapi_host_attempt_summary(role, base_url, exc)
-                if index == len(self._crawapi_base_urls) - 1 or not _should_try_crawapi_candidate(exc):
+                if index == len(base_urls) - 1 or not _should_try_crawapi_candidate(exc):
                     self._raise_crawapi_error(exc, attempts + [attempt])
                 attempts.append(attempt)
         raise RuntimeError(f"crawapi {operation} failed: no base_url configured")
@@ -406,6 +464,32 @@ def _dedupe_crawapi_base_urls(candidates: list[tuple[str, str]]) -> list[tuple[s
     return result
 
 
+def _legacy_crawapi_base_urls(candidates: list[tuple[str, str]]) -> list[tuple[str, str]]:
+    aiddit_candidates = [
+        (role, base_url) for role, base_url in candidates if _is_aiddit_crawapi_base_url(base_url)
+    ]
+    return aiddit_candidates or candidates
+
+
+def _is_aiddit_crawapi_base_url(base_url: str) -> bool:
+    return "crawler.aiddit.com" in base_url.lower()
+
+
+def _keyword_provider_for_base_url(role: str, base_url: str) -> str:
+    if _is_aiddit_crawapi_base_url(base_url):
+        return "crawler"
+    normalized = base_url.lower()
+    if "piaoquantv" in normalized:
+        return "piaoquantv"
+    return "crawler" if role == "candidate" else "piaoquantv"
+
+
+def _query_cursor(query: dict[str, Any], default: str) -> str:
+    if "page_cursor" in query:
+        return str(query.get("page_cursor") or "")
+    return str(default)
+
+
 def _should_try_crawapi_candidate(exc: Exception) -> bool:
     if not isinstance(exc, CrawapiBusinessError):
         return True
@@ -426,9 +510,11 @@ def _crawapi_host_attempt_summary(
     role: str,
     base_url: str,
     exc: Exception,
+    provider: str | None = None,
 ) -> dict[str, Any]:
     summary: dict[str, Any] = {
         "role": role,
+        "provider": provider or _keyword_provider_for_base_url(role, base_url),
         "base_url": base_url.rstrip("/"),
         "exception_type": type(exc).__name__,
     }
@@ -450,6 +536,9 @@ def _crawapi_host_attempt_summary(
             for key in ("status_code", "content_type"):
                 if key in response_summary:
                     summary[key] = response_summary.get(key)
+        request_payload_summary = detail.get("request_payload_summary")
+        if isinstance(request_payload_summary, dict):
+            summary["request_payload_summary"] = request_payload_summary
     return summary
 
 

tests/test_douyin_client.py

@@ -34,16 +34,18 @@ def _response(status_code, data):
     )
 
 
-def _client(responses, rate_limiter=None, fallback_base_url=""):
-    return CrawapiDouyinClient(
-        base_url="http://crawapi.test",
-        fallback_base_url=fallback_base_url,
-        keyword_path="/crawler/dou_yin/keyword",
-        blogger_path="/crawler/dou_yin/blogger",
-        default_crawapi_account_ref="771431222",
-        http_client=FakeHttpClient(responses),
-        rate_limiter=rate_limiter,
-    )
+def _client(responses, rate_limiter=None, fallback_base_url="", **overrides):
+    options = {
+        "base_url": "http://crawapi.test",
+        "fallback_base_url": fallback_base_url,
+        "keyword_path": "/crawler/dou_yin/keyword",
+        "blogger_path": "/crawler/dou_yin/blogger",
+        "default_crawapi_account_ref": "771431222",
+        "http_client": FakeHttpClient(responses),
+        "rate_limiter": rate_limiter,
+    }
+    options.update(overrides)
+    return CrawapiDouyinClient(**options)
 
 
 def _search_query(text="早上好祝福视频"):
@@ -105,6 +107,23 @@ def test_douyin_keyword_search_maps_content_fields():
     assert client.http_client.requests[0]["url"].endswith("/crawler/dou_yin/keyword")
 
 
+def test_douyin_keyword_search_posts_piaoquantv_apifox_payload():
+    client = _client([_response(200, {"data": {"data": [], "has_more": False}})])
+
+    client.search(_search_query("宁艺卓"))
+
+    assert client.http_client.requests[0]["json"] == {
+        RAW_AUTHOR_ACCOUNT_KEY: "771431222",
+        "keyword": "宁艺卓",
+        "content_type": "视频",
+        "sort_type": "综合排序",
+        "publish_time": "不限",
+        "duration": "不限",
+        "cursor": "0",
+        "cookie_batch": "default",
+    }
+
+
 def test_douyin_keyword_search_returns_empty_list():
     client = _client([_response(200, {"data": {"data": [], "has_more": False}})])
 
@@ -159,6 +178,42 @@ def test_douyin_fetch_author_works_maps_fake_response():
     assert results[0]["search_query_id"] == "author_001"
     assert results[0]["previous_discovery_step"] == "author_works"
     assert client.http_client.requests[0]["json"][RAW_AUTHOR_ACCOUNT_KEY] == "MS4wLjABAAAA001"
+
+
+def test_douyin_non_keyword_requests_keep_aiddit_legacy_base_url():
+    client = _client(
+        [
+            _response(
+                200,
+                {
+                    "data": {
+                        "data": [
+                            {
+                                RAW_CONTENT_ID_KEY: "7615247738577423001",
+                                "desc": "作者作品",
+                                "author": {RAW_AUTHOR_ID_KEY: "MS4wLjABAAAA001"},
+                            }
+                        ],
+                        "has_more": False,
+                    }
+                },
+            ),
+        ],
+        base_url="http://crawapi.piaoquantv.com",
+        fallback_base_url="http://crawler.aiddit.com",
+    )
+
+    client.fetch_author_works(
+        {
+            "search_query_id": "author_001",
+            "search_query": "作者作品",
+            "platform_author_id": "MS4wLjABAAAA001",
+            "discovery_start_source": "pattern_itemset",
+        }
+    )
+
+    assert client.http_client.requests[0]["url"].startswith("http://crawler.aiddit.com/")
+    assert client.http_client.requests[0]["url"].endswith("/crawler/dou_yin/blogger")
     assert len(client.http_client.requests) == 1
 
 
@@ -192,6 +247,65 @@ def test_douyin_keyword_search_falls_back_to_candidate_on_unknown_business_error
     ]
 
 
+def test_douyin_keyword_search_falls_back_to_crawler_payload_on_force_login():
+    client = _client(
+        [
+            _response(200, {"code": 22001, "msg": "抖音搜索异常: 强制登录", "data": None}),
+            _response(
+                200,
+                {
+                    "code": 0,
+                    "data": {
+                        "data": [
+                            {
+                                RAW_CONTENT_ID_KEY: "7615247738577423622",
+                                "desc": "国风传统",
+                                "author": {
+                                    "nickname": "作者",
+                                    RAW_AUTHOR_ID_KEY: "MS4wLjABAAAAdYc",
+                                },
+                                "statistics": {
+                                    "digg_count": 10,
+                                    "comment_count": 2,
+                                    "share_count": 3,
+                                    "collect_count": 4,
+                                },
+                            }
+                        ],
+                        "has_more": True,
+                        "next_cursor": "12",
+                    },
+                },
+            ),
+        ],
+        fallback_base_url="http://crawler.aiddit.com",
+    )
+
+    results = client.search(_search_query("国风传统"))
+
+    assert client.http_client.requests[0]["json"] == {
+        RAW_AUTHOR_ACCOUNT_KEY: "771431222",
+        "keyword": "国风传统",
+        "content_type": "视频",
+        "sort_type": "综合排序",
+        "publish_time": "不限",
+        "duration": "不限",
+        "cursor": "0",
+        "cookie_batch": "default",
+    }
+    assert client.http_client.requests[1]["json"] == {
+        "content_type": "视频",
+        "keyword": "国风传统",
+        "cursor": "",
+        "sort_type": "最多点赞",
+    }
+    assert results[0]["platform_content_id"] == "7615247738577423622"
+    assert results[0]["platform_author_id"] == "MS4wLjABAAAAdYc"
+    assert results[0]["statistics"]["digg_count"] == 10
+    assert results[0]["has_more"] is True
+    assert results[0]["next_cursor"] == "12"
+
+
 def test_douyin_keyword_search_does_not_fallback_on_parameter_error():
     client = _client(
         [
@@ -251,7 +365,10 @@ def test_douyin_keyword_search_keeps_host_attempts_when_candidate_fails():
         client.search(_search_query("主备都失败"))
     attempts = exc_info.value.detail["crawapi_host_attempts"]
     assert [attempt["role"] for attempt in attempts] == ["primary", "candidate"]
+    assert [attempt["provider"] for attempt in attempts] == ["piaoquantv", "crawler"]
     assert attempts[0]["business_code"] == "10000"
+    assert attempts[0]["request_payload_summary"]["duration"] == "不限"
+    assert attempts[0]["request_payload_summary"]["cookie_batch"] == "default"
     assert attempts[1]["status_code"] == 500
 
 
@@ -453,12 +570,22 @@ def test_from_env_reads_blogger_path_and_sort_type(monkeypatch, tmp_path):
     monkeypatch.setenv("CONTENTFIND_DOUYIN_KEYWORD_PATH", "/crawler/dou_yin/keyword")
     monkeypatch.setenv("CONTENTFIND_DOUYIN_BLOGGER_PATH", "/crawler/dou_yin/blogger")
     monkeypatch.setenv("CONTENTFIND_DOUYIN_ACCOUNT_WORKS_DEFAULT_SORT_TYPE", "最热")
+    monkeypatch.setenv("CONTENTFIND_DOUYIN_PIAOQUANTV_ACCOUNT_ID", "7450041106378522636")
+    monkeypatch.setenv("CONTENTFIND_DOUYIN_PIAOQUANTV_DURATION", "不限")
+    monkeypatch.setenv("CONTENTFIND_DOUYIN_PIAOQUANTV_COOKIE_BATCH", "default")
+    monkeypatch.setenv("CONTENTFIND_DOUYIN_CRAWLER_SORT_TYPE", "最多点赞")
+    monkeypatch.setenv("CONTENTFIND_DOUYIN_CRAWLER_CURSOR_DEFAULT", "")
 
     client = CrawapiDouyinClient.from_env(env_path=tmp_path / "missing.env")
 
     assert client.base_url == "http://crawapi.test/"
     assert client.fallback_base_url == "http://candidate.test/"
     assert client.blogger_path == "crawler/dou_yin/blogger"
+    assert client.piaoquantv_account_id == "7450041106378522636"
+    assert client.piaoquantv_duration == "不限"
+    assert client.piaoquantv_cookie_batch == "default"
+    assert client.crawler_sort_type == "最多点赞"
+    assert client.crawler_cursor_default == ""
     assert client.default_account_works_sort_type == "最热"
     assert client.max_results_per_query == 5
     assert isinstance(client.rate_limiter, RateLimiter)