"""帖子详情拉取:小红书 detail 接口。 POST {base}/crawler/xiao_hong_shu/detail body={"content_id": ...} 无 token,限流 ≥15s。 逻辑(限流 + code==0 信封校验)对齐 ContentFindAgentNew 的 crawapi_http.post_crawapi_json, 但不依赖其错误体系,保持本项目自包含。 拆成两层:parse_detail_response 纯函数(可离线用 fixture 测)+ fetch_post_detail 负责 HTTP。 """ from __future__ import annotations import time from typing import Any, Callable, Optional from urllib.parse import urljoin import httpx from creation_knowledge.config import Settings from creation_knowledge.models import Post DETAIL_PATH = "/crawler/xiao_hong_shu/detail" RATE_LIMIT_SECONDS = 15.0 class CrawlerError(RuntimeError): pass class RateLimiter: """同一 bucket 两次调用间隔 ≥ min_interval。对齐 CFA RateLimiter。""" def __init__( self, min_interval_seconds: float = RATE_LIMIT_SECONDS, now_fn: Callable[[], float] = time.monotonic, sleep_fn: Callable[[float], None] = time.sleep, ) -> None: self.min_interval_seconds = min_interval_seconds self.now_fn = now_fn self.sleep_fn = sleep_fn self._last: dict[str, float] = {} def wait(self, bucket: str) -> None: last = self._last.get(bucket) if last is not None: remaining = self.min_interval_seconds - (self.now_fn() - last) if remaining > 0: self.sleep_fn(remaining) self._last[bucket] = self.now_fn() def parse_content_id(content_id_or_url: str) -> str: """接受裸 content_id 或 https://www.xiaohongshu.com/explore/?... 链接。""" s = content_id_or_url.strip() if "/explore/" in s: s = s.split("/explore/", 1)[1] if "/discovery/item/" in content_id_or_url: s = content_id_or_url.split("/discovery/item/", 1)[1] return s.split("?", 1)[0].strip("/").strip() def _image_urls(inner: dict) -> list[str]: out: list[str] = [] seen: set[str] = set() for x in inner.get("image_url_list") or []: u = x.get("image_url") if isinstance(x, dict) else x if u and u not in seen: seen.add(u) out.append(u) return out def _video_urls(inner: dict) -> list[str]: out: list[str] = [] for x in inner.get("video_url_list") or []: u = x.get("video_url") if isinstance(x, dict) else x if u: out.append(u) return out def parse_detail_response(response: dict, fallback_content_id: str = "") -> Post: """把 detail 接口返回(信封 {code,msg,data:{data:{...}}})解析成 Post。""" if not isinstance(response, dict): raise CrawlerError("bad_response: not a dict") code = response.get("code") if code not in (0, "0"): raise CrawlerError(f"business_error: code={code} msg={response.get('msg')}") inner = ((response.get("data") or {}).get("data")) or {} if not inner: raise CrawlerError("empty_detail: data.data is empty") content_id = inner.get("channel_content_id") or fallback_content_id link = inner.get("content_link") or ( f"https://www.xiaohongshu.com/explore/{content_id}" if content_id else "" ) return Post( id=f"xhs_{content_id}", platform="xiaohongshu", url=link, content_id=content_id, title=inner.get("title") or "", content_type=inner.get("content_type") or "", body_text=inner.get("body_text") or "", topic_list=list(inner.get("topic_list") or []), image_urls=_image_urls(inner), video_urls=_video_urls(inner), author_id=inner.get("channel_account_id"), author_name=inner.get("channel_account_name"), raw=response, ) def fetch_post_detail( content_id_or_url: str, *, settings: Optional[Settings] = None, http_client: Any = None, rate_limiter: Optional[RateLimiter] = None, env_file: str = ".env", ) -> Post: """真实拉取一条小红书帖子详情,返回 Post。""" settings = settings or Settings.from_env(env_file) content_id = parse_content_id(content_id_or_url) if not content_id: raise CrawlerError(f"cannot parse content_id from: {content_id_or_url}") rate_limiter = rate_limiter or RateLimiter() rate_limiter.wait("xhs_detail") owns_client = http_client is None client = http_client or httpx.Client() try: url = urljoin(settings.crawler_base_url, DETAIL_PATH) resp = client.post( url, json={"content_id": content_id}, headers={"Content-Type": "application/json"}, timeout=settings.crawler_timeout, ) resp.raise_for_status() data = resp.json() except httpx.HTTPError as exc: raise CrawlerError(f"http_error: {exc}") from exc except ValueError as exc: raise CrawlerError("bad_json") from exc finally: if owns_client: client.close() return parse_detail_response(data, fallback_content_id=content_id)