Create-knowledge-find-decode/scripts/decompose.py

"""创作知识解构引擎 v2：一帖 → N 颗（how/what/why，含组件颗）→ frameworks.json + payloads.json。

编排全流程，把 skill 的 phase 文档当 prompt 喂给 LLM（skill 是唯一真源）：
  ① 读懂：图文帖→extractor 读图；视频帖→video_extract 下载 mp4+原生整段提炼（base64→Gemini）
  ②  判颗+类型闸+三lane成形+轻标签：system = phase1-frame.md
  ③  作用域：system = phase2-scope.md → 候选 → scope_link 回扣（火山）
  ⑤  组装：代码 → 每颗一个 ingest payload（按类型分拼）
① 判 is_empty=true（无可提取知识）→ 短路，跳过 ②③⑤。
数据源：fixture（已有 5 帖）或实时 crawler 取数（新帖）。须在云端跑。用法：PYTHONPATH=. python scripts/decompose.py
"""
from __future__ import annotations

import json
from pathlib import Path

from creation_knowledge.config import Settings
from creation_knowledge.integrations import video_extract
from creation_knowledge.integrations.crawler import fetch_post_detail, parse_detail_response
from creation_knowledge.integrations.extractor import GeminiExtractor
from creation_knowledge.integrations.llm import chat_json
from scripts.scope_link import ScopeLinker

ROOT = Path(__file__).resolve().parent.parent
FIX = ROOT / "tests" / "fixtures"
DATA = ROOT / "data" / "demo"
SKILL = ROOT / "创作知识提取-skill"
PHASE1 = (SKILL / "extraction" / "phase1-frame.md").read_text(encoding="utf-8")
PHASE2 = (SKILL / "extraction" / "phase2-scope.md").read_text(encoding="utf-8")

# from: fixture（读 tests/fixtures）/ live（实时 crawler 取数）
SOURCES = [
    {"cid": "699308fa0000000016009697", "platform": "xiaohongshu", "from": "fixture"},
    {"cid": "698481e1000000000a02a7c1", "platform": "xiaohongshu", "from": "fixture"},
    {"cid": "67e2e39b0000000003028ff0", "platform": "xiaohongshu", "from": "fixture"},
    {"cid": "680659e8000000001a007a11", "platform": "xiaohongshu", "from": "fixture"},
    {"cid": "67e4bdf50000000006028a59", "platform": "xiaohongshu", "from": "fixture"},
    {"cid": "7589257893544165455", "platform": "douyin", "from": "live"},        # 抖音视频
    {"cid": "6a33655e000000000f0055af", "platform": "xiaohongshu", "from": "live"},  # 无知识
]
SRC2CN = {"substance": "实质", "form": "形式", "feeling": "感受", "effect": "作用", "intent": "意图"}
TYPE2ATTR = {"how": "how工序", "what": "what构成", "why": "why原理"}
CSTAGE = {"定向", "构思", "结构", "成文", "打磨"}  # 创作阶段受控 5 值
REUSE_THRESHOLD = 0.90


# ---------- 取数 ----------
def load_post(src: dict, settings: Settings):
    if src["from"] == "fixture":
        resp = json.loads((FIX / f"xhs_case_{src['cid']}.json").read_text("utf-8"))
        post = parse_detail_response(resp, fallback_content_id=src["cid"])
    else:
        post = fetch_post_detail(src["cid"], settings=settings)
    if not post.url:
        post.url = f"https://www.xiaohongshu.com/explore/{src['cid']}"
    return post


# ---------- ① 读懂（图文/视频分流） ----------
def read_one(src: dict, post, settings: Settings, extractor: GeminiExtractor):
    cid = src["cid"]
    if post.video_urls:  # 视频帖：下载 mp4 + 原生整段提炼
        save = DATA / "douyin" / f"dy_{cid}" / "video.mp4"
        pub = f"/data/demo/douyin/dy_{cid}/video.mp4"
        ec = video_extract.extract_video(post, settings=settings, save_path=save, public_url=pub)
        media = {"type": "video", "video_url": pub, "images": []}
        cmap = {c.index: c.content for c in ec.cards}
        cards = [{"index": c.index, "content": cmap.get(c.index, ""), "video_url": pub,
                  "start": c.start, "end": c.end} for c in post.cards]  # 段卡：时间戳 + 读到的内容
    else:                # 图文帖：读图
        ec = extractor.extract(post)
        if src["from"] == "fixture":
            imgs = [f"/data/demo/xiaohongshu/{post.id}/image_{n}.webp" for n in range(1, len(post.image_urls) + 1)]
        else:
            imgs = list(post.image_urls)  # 新帖未落盘 → 直接用 CDN url
        media = {"type": "image", "video_url": None, "images": imgs}
        cmap = {c.index: c.content for c in ec.cards}
        cards = [{"index": n, "content": cmap.get(n, ""), "image_url": imgs[n - 1] if n <= len(imgs) else None}
                 for n in range(1, len(imgs) + 1)]  # 每图一张卡：图 url + 读到的内容
    parts = [ec.text]
    if ec.from_image:
        parts.append("【图片要点】\n" + ec.from_image)
    parts += [f"【卡片{c.index}】{c.content}" for c in ec.cards if c.content]
    return "\n\n".join(p for p in parts if p), bool(ec.is_empty), media, cards


# ---------- ② 判颗+成形+轻标签 ----------
def shape(post, read: str) -> list[dict]:
    user = (f"原帖标题：{post.title or '(无)'}\n\n读懂后的完整内容：\n{read}\n\n"
            "按上面规则拆颗+判类型+成形+轻标签。作用域字段一律留空 []。"
            "只输出 JSON：{\"knowledges\":[ ... 见模板 ... ]}")
    return chat_json(PHASE1, user, timeout=120).get("knowledges") or []


# ---------- ③ 作用域候选 + 回扣 ----------
def _slim(knowledges: list[dict]) -> list[dict]:
    slim = []
    for k in knowledges:
        e = {"id": k.get("id"), "type": k.get("type"), "title": k.get("title")}
        if k.get("type") == "how":
            e["steps"] = [{"id": s.get("id"), "intent": s.get("intent"),
                           "directive": (s.get("directive") or "")[:500], "output": s.get("output")}
                          for s in k.get("steps", [])]
        else:
            e["内容"] = {x: k.get(x) for x in ("界定", "主体", "主张", "支撑") if k.get(x)}
        slim.append(e)
    return slim


def scope_candidates(knowledges: list[dict]) -> list:
    user = ("给下面每颗知识标作用域候选（how 逐步：每个 step 一组；what/why 颗级：整颗一组）。\n"
            "只输出 JSON：{\"scopes\":[{\"knowledge_id\":\"k1\",\"step_id\":\"s1\",\"items\":[{\"scope_type\":\"substance\",\"value\":\"…\"}]},"
            "{\"knowledge_id\":\"k2\",\"step_id\":null,\"items\":[...]}]}\n\n"
            + json.dumps(_slim(knowledges), ensure_ascii=False))
    return chat_json(PHASE2, user, timeout=120).get("scopes") or []


def link_scope(linker: ScopeLinker, scope_type: str, value: str) -> dict:
    try:
        hits = linker.link(value, source_type=SRC2CN.get(scope_type, scope_type), top_k=3)
    except Exception:
        hits = []
    top = hits[0] if hits else {}
    score = float(top.get("score", 0.0))
    reuse = score >= REUSE_THRESHOLD and top.get("name")
    return {"scope_type": scope_type, "value": top["name"] if reuse else value,
            "candidate": value, "link": "复用" if reuse else "新建", "score": round(score, 4),
            "top": [{"name": h["name"], "score": h["score"], "path": h.get("path", "")} for h in hits]}


def apply_scopes(knowledges: list[dict], scopes: list, linker: ScopeLinker) -> None:
    by_k = {k.get("id"): k for k in knowledges}
    for sc in scopes:
        k = by_k.get(sc.get("knowledge_id"))
        if not k:
            continue
        linked = [link_scope(linker, it["scope_type"], it["value"])
                  for it in (sc.get("items") or []) if it.get("scope_type") and it.get("value")]
        if k.get("type") == "how" and sc.get("step_id"):
            for s in k.get("steps", []):
                if s.get("id") == sc["step_id"]:
                    s["作用域"] = linked
        else:
            k["作用域"] = (k.get("作用域") or []) + linked


# ---------- ⑤ 组装 ----------
def build_content(k: dict) -> str:
    t = k.get("type")
    if t == "how":
        lines = [f"目标：{k.get('purpose','')}"]
        for i, s in enumerate(k.get("steps", []), 1):
            lines += [f"步骤{i}（目的：{s.get('intent','')}）",
                      f"  指引：{s.get('directive','')}", f"  产出：{s.get('output','')}"]
        return "\n".join(lines)
    if t == "what":
        return "\n".join([f"界定：{k.get('界定','')}"] + _sections(k.get("主体")))
    return "\n".join([f"主张：{k.get('主张','')}"] + _sections(k.get("支撑")))


def _sections(blocks) -> list[str]:
    """把 what.主体 / why.支撑 的自由小节拼成文本行。"""
    out = []
    for b in blocks or []:
        head = b.get("小标题") or ""
        form = b.get("形式")
        out.append(f"【{head}】" + (f"（{form}）" if form else ""))
        if b.get("内容"):
            out.append(f"  {b['内容']}")
        for it in b.get("条目") or []:
            word = it.get("词") or it.get("要素") or ""
            out.append(f"  - {word}：{it.get('说明','')}" if word else f"  - {it.get('说明','')}")
    return out


def build_payload(post, k: dict, how_titles: dict | None = None) -> dict:
    how_titles = how_titles or {}
    t = k.get("type")
    scopes, seen = [], set()
    def add(lst):
        for sc in lst:
            key = (sc["scope_type"], sc["value"])
            if key not in seen:
                seen.add(key); scopes.append({"scope_type": sc["scope_type"], "value": sc["value"]})
    if t == "how":
        for s in k.get("steps", []):
            add(s.get("作用域", []))
    else:
        add(k.get("作用域", []))
    ext = [{"key": "业务阶段", "type": "str", "value": v} for v in (k.get("业务阶段") or [])]
    if t == "how":
        cs, cseen = [], set()
        for s in k.get("steps", []):
            c = s.get("创作阶段")
            if c and c not in cseen:
                cseen.add(c); cs.append(c)
        ext += [{"key": "创作阶段", "type": "str", "value": v} for v in cs]
        ext += [{"key": "动作", "type": "str", "value": s["动作"]} for s in k.get("steps", []) if s.get("动作")]
    if k.get("role") == "组件" and k.get("parent"):
        p = k["parent"]
        ext.append({"key": "出自", "type": "str",
                    "value": f"{how_titles.get(p.get('how_id'), p.get('how_id'))} 第{p.get('step')}步"})
    return {"source": {"id": post.id, "source_type": "post", "title": post.title or "",
                       "author": post.author_name or "", "source_metadata": {"platform": post.platform, "url": post.url}},
            "title": k.get("title"), "content": build_content(k),
            "dim_creations": ["创作"], "dim_attributes": [TYPE2ATTR.get(t, "how工序")],
            "scopes": scopes, "custom_ext": ext}


def main() -> None:
    settings = Settings.from_env()
    extractor = GeminiExtractor.from_env()
    linker = ScopeLinker()
    posts_out, payloads = [], []
    for src in SOURCES:
        cid = src["cid"]
        print(f"\n=== {src['platform']} {cid[:10]} ({src['from']}) ===")
        try:
            post = load_post(src, settings)
            read, is_empty, media, cards = read_one(src, post, settings, extractor)
        except Exception as exc:
            print(f"  ✗ 取数/读懂失败：{exc}")
            posts_out.append({"post_id": cid, "source_id": cid, "title": f"(取数失败 {cid})",
                              "platform": src["platform"], "url": "", "media": {"type": "image", "images": []},
                              "cards": [], "error": str(exc)[:200], "knowledges": []})
            continue
        meta = {"post_id": cid, "source_id": post.id, "title": post.title or "",
                "platform": post.platform, "url": post.url, "media": media, "cards": cards}
        if is_empty:  # ① 总闸
            print(f"  ① 读懂 {len(read)} 字 → 判定无可提取的创作知识，跳过拆颗")
            posts_out.append({**meta, "no_knowledge": True, "knowledges": []})
            continue
        print(f"  ① 读懂 {len(read)} 字（{media['type']}）")
        knowledges = shape(post, read)
        how_ids = {k.get("id") for k in knowledges if k.get("type") == "how"}
        how_titles = {k.get("id"): k.get("title") for k in knowledges if k.get("type") == "how"}
        for k in knowledges:
            k["业务阶段"] = [b for b in (k.get("业务阶段") or []) if b in ("灵感", "选题", "脚本")]  # 守卫：只留合法业务阶段
            for s in k.get("steps", []):  # 守卫：创作阶段只留合法 5 值，非法（如"定稿/输出"）丢弃
                if s.get("创作阶段") not in CSTAGE:
                    s["创作阶段"] = None
            if k.get("role") == "组件" and (k.get("parent") or {}).get("how_id") not in how_ids:  # 守卫：组件 parent 必指向同帖 how
                k["role"] = "主"; k["parent"] = None
        print(f"  ② {len(knowledges)} 颗：" + ", ".join(f"{k.get('type')}/{k.get('role')}" for k in knowledges))
        apply_scopes(knowledges, scope_candidates(knowledges), linker)
        print("  ③⑤ 作用域回扣 + 组装")
        posts_out.append({**meta, "knowledges": knowledges})
        payloads += [build_payload(post, k, how_titles) for k in knowledges]
    (ROOT / "web/frameworks.json").write_text(
        json.dumps({"count": len(posts_out), "posts": posts_out}, ensure_ascii=False, indent=1), encoding="utf-8")
    (ROOT / "web/payloads.json").write_text(
        json.dumps(payloads, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"\nwrote {len(posts_out)} posts, {len(payloads)} payloads")


if __name__ == "__main__":
    main()