Create-knowledge-find-decode/acquisition/backfill_weixin.py

"""补下载微信公众号帖子的【完整正文 + 内文图】到本地，写回 app.db。

微信搜索只回标题+封面，正文没下；分类只看标题会判不准。本脚本对库里所有微信帖逐个拉详情
（wei_xin/detail，实测不需 token）→ 下全部内文图到 data/search/weixin/<hash>/img_*.jpg
→ 把 body_text + images 写回 search_results.extra。之后微信就和小红书一样是「完整内容」。
限流：详情接口 10~12s 随机一次（图片下载走 CDN，不计入闸）。已补过的(extra 有 body_text)跳过，可重跑。
用法：PYTHONPATH=. CK_ENV_FILE=.env python -m acquisition.backfill_weixin
"""
from __future__ import annotations

import hashlib
import time
from pathlib import Path

from acquisition import store
from acquisition.crawler import RateLimiter
from acquisition.search import fetch_weixin_detail
from core.config import Settings
from creation_knowledge.integrations.video_extract import _default_download

ROOT = Path(__file__).resolve().parent.parent
DATA = ROOT / "data"
GATE_MIN, GATE_MAX = 10.0, 12.0     # 微信详情接口两次调用随机间隔（秒）


def _dl(url: str, dst: Path, public: str):
    try:
        dst.parent.mkdir(parents=True, exist_ok=True)
        dst.write_bytes(_default_download(url, "weixin"))
        return public
    except Exception:
        return None


def main() -> None:
    settings = Settings.from_env()
    conn = store.connect()
    urls = store.weixin_urls_missing_body(conn)
    if not urls:
        print("微信帖正文都已补过。"); conn.close(); return
    gate = RateLimiter(min_interval_seconds=GATE_MIN, max_interval_seconds=GATE_MAX)
    total, ok, fail, img_total = len(urls), 0, 0, 0
    print(f"待补正文的微信帖 {total} 个（详情 {GATE_MIN}~{GATE_MAX}s 一次）...")
    for i, url in enumerate(urls, 1):
        try:
            body, img_urls = fetch_weixin_detail(url, settings=settings, rate_limiter=gate)
        except Exception as exc:
            fail += 1
            print(f"  [{i}/{total}] 详情失败: {str(exc)[:50]}")
            continue
        h = hashlib.md5(url.encode()).hexdigest()[:16]
        base, pub = DATA / "search" / "weixin" / h, f"/data/search/weixin/{h}"
        imgs = []
        for j, u in enumerate(img_urls):
            local = _dl(u, base / f"img_{j}.jpg", f"{pub}/img_{j}.jpg")
            if local:
                imgs.append(local)
        store.update_post_content(conn, url, body, imgs)
        ok += 1
        img_total += len(imgs)
        if i % 10 == 0 or i == total:
            print(f"  [{i}/{total}] 正文 {len(body)} 字 / 图 {len(imgs)} 张（累计 {ok} 成 / {fail} 败 / {img_total} 图）")
    conn.close()
    print(f"完成：{ok} 帖补到正文，{fail} 失败，共下 {img_total} 张内文图。")


if __name__ == "__main__":
    main()