Create-knowledge-find-decode/scripts/decompose.py

"""创作知识解构引擎 v2：一帖 → N 颗（how/what/why，含组件颗）→ frameworks.json + payloads.json。

编排全流程，把 skill 的 phase 文档当 prompt 喂给 LLM（skill 是唯一真源）：
  ① 读懂：图文帖→extractor 读图；视频帖→video_extract 下载 mp4+原生整段提炼（base64→Gemini）
  ②  判颗+类型闸+三lane成形+轻标签：system = phase1-frame.md
  ③  作用域：system = phase2-scope.md → 候选 → scope_link 回扣（火山）
  ⑤  组装：代码 → 每颗一个 ingest payload（按类型分拼）
① 判 is_empty=true（无可提取知识）→ 短路，跳过 ②③⑤。
数据源：fixture（已有 5 帖）或实时 crawler 取数（新帖）。须在云端跑。用法：PYTHONPATH=. python scripts/decompose.py
"""
from __future__ import annotations

import json
import os
from pathlib import Path

from creation_knowledge.config import Settings
from creation_knowledge.integrations import video_extract
from creation_knowledge.integrations.crawler import fetch_post_detail, parse_detail_response
from creation_knowledge.integrations.extractor import GeminiExtractor
from creation_knowledge.integrations.llm import chat_json
from creation_knowledge.prompts import load_prompt
from scripts.scope_link import ScopeLinker

ROOT = Path(__file__).resolve().parent.parent
FIX = ROOT / "tests" / "fixtures"
DATA = ROOT / "data" / "demo"
SKILL = ROOT / "创作知识提取-skill"
PHASE1 = (SKILL / "extraction" / "phase1-frame.md").read_text(encoding="utf-8")
PHASE2 = (SKILL / "extraction" / "phase2-scope.md").read_text(encoding="utf-8")
GATE_ADMIT = load_prompt("gate_admit")        # ①.5 创作判定闸：判"是创作"
GATE_REFUTE = load_prompt("gate_refute")      # ①.5：挑刺"是制作/越界"
GATE_TIEBREAK = load_prompt("gate_tiebreak")  # ①.5：分歧裁决（边界倾向排除）
NORMALIZE = load_prompt("normalize_scope")    # ③前：作用域值名词化（③LLM）

# ① 窄表确定性兜底：只放无歧义裸动作动词（绝不放 营造/引导/表达 等名词语素）
_STRIP_VERBS = ("寻找", "定位", "推导", "核验", "提取", "挖掘", "捕捉",
                "识别", "梳理", "归纳", "判断", "验证", "确认", "复盘")
_PROTECT = ("营造", "引导", "表达", "塑造", "叙述", "呈现", "刻画",
            "升华", "控制", "推进", "转化", "传达")  # 名词语素，永不砍

# from: fixture（读 tests/fixtures）/ live（实时 crawler 取数）
SOURCES = [
    {"cid": "699308fa0000000016009697", "platform": "xiaohongshu", "from": "fixture"},
    {"cid": "698481e1000000000a02a7c1", "platform": "xiaohongshu", "from": "fixture"},
    {"cid": "67e2e39b0000000003028ff0", "platform": "xiaohongshu", "from": "fixture"},
    {"cid": "680659e8000000001a007a11", "platform": "xiaohongshu", "from": "fixture"},
    {"cid": "67e4bdf50000000006028a59", "platform": "xiaohongshu", "from": "fixture"},
    {"cid": "7589257893544165455", "platform": "douyin", "from": "live"},        # 抖音视频
    {"cid": "6a33655e000000000f0055af", "platform": "xiaohongshu", "from": "live"},  # 无知识
]
SRC2CN = {"substance": "实质", "form": "形式", "feeling": "感受", "effect": "作用", "intent": "意图"}
TYPE2ATTR = {"how": "how工序", "what": "what构成", "why": "why原理"}
CSTAGE = {"定向", "构思", "结构", "成文", "打磨"}  # 创作阶段受控 5 值
REUSE_THRESHOLD = 0.90


# ---------- 取数 ----------
def load_post(src: dict, settings: Settings):
    if src["from"] == "fixture":
        resp = json.loads((FIX / f"xhs_case_{src['cid']}.json").read_text("utf-8"))
        post = parse_detail_response(resp, fallback_content_id=src["cid"])
    else:
        post = fetch_post_detail(src["cid"], settings=settings)
    if not post.url:
        post.url = f"https://www.xiaohongshu.com/explore/{src['cid']}"
    return post


# ---------- ① 读懂（图文/视频分流） ----------
def read_one(src: dict, post, settings: Settings, extractor: GeminiExtractor):
    cid = src["cid"]
    if post.video_urls:  # 视频帖：下载 mp4 + 原生整段提炼
        save = DATA / "douyin" / f"dy_{cid}" / "video.mp4"
        pub = f"/data/demo/douyin/dy_{cid}/video.mp4"
        ec = video_extract.extract_video(post, settings=settings, save_path=save, public_url=pub)
        media = {"type": "video", "video_url": pub, "images": []}
        cmap = {c.index: c.content for c in ec.cards}
        cards = [{"index": c.index, "content": cmap.get(c.index, ""), "video_url": pub,
                  "start": c.start, "end": c.end} for c in post.cards]  # 段卡：时间戳 + 读到的内容
    else:                # 图文帖：读图
        ec = extractor.extract(post)
        if src["from"] == "fixture":
            imgs = [f"/data/demo/xiaohongshu/{post.id}/image_{n}.webp" for n in range(1, len(post.image_urls) + 1)]
        else:
            imgs = list(post.image_urls)  # 新帖未落盘 → 直接用 CDN url
        media = {"type": "image", "video_url": None, "images": imgs}
        cmap = {c.index: c.content for c in ec.cards}
        cards = [{"index": n, "content": cmap.get(n, ""), "image_url": imgs[n - 1] if n <= len(imgs) else None}
                 for n in range(1, len(imgs) + 1)]  # 每图一张卡：图 url + 读到的内容
    parts = [ec.text]
    if ec.from_image:
        parts.append("【图片要点】\n" + ec.from_image)
    parts += [f"【卡片{c.index}】{c.content}" for c in ec.cards if c.content]
    return "\n\n".join(p for p in parts if p), bool(ec.is_empty), media, cards


# ---------- ② 判颗+成形+轻标签 ----------
def shape(post, read: str) -> list[dict]:
    user = (f"原帖标题：{post.title or '(无)'}\n\n读懂后的完整内容：\n{read}\n\n"
            "按上面规则拆颗+判类型+成形+轻标签。作用域字段一律留空 []。"
            "只输出 JSON：{\"knowledges\":[ ... 见模板 ... ]}")
    return chat_json(PHASE1, user, timeout=120).get("knowledges") or []


# ---------- ①.5 创作判定闸（voting：admit + refute，分歧上 tiebreak；边界倾向排除）----------
def _vote(system: str, read: str, key: str, on_fail: bool) -> bool:
    """跑一次判定，取 key 字段为 bool；出错按 on_fail 兜底（避免 API 抖动误杀）。"""
    try:
        return bool(chat_json(system, read, timeout=90).get(key))
    except Exception:
        return on_fail


def creation_gate(read: str) -> tuple[bool, str]:
    """判这帖是不是【图文/视频内容创作知识】。返回 (in_scope, 说明)。
    甲方案：admit + refute 两票；一致即定；分歧→tiebreak 裁决（边界倾向排除）。
    判定只看 ① 读懂后的内容，不看标题、不数关键词。"""
    v_admit = _vote(GATE_ADMIT, read, "in_scope", on_fail=True)            # 判"是创作"
    v_refute = not _vote(GATE_REFUTE, read, "out_of_scope", on_fail=False)  # 挑刺"是制作/越界"→归一成 in_scope
    if v_admit == v_refute:
        return v_admit, f"admit={v_admit}/refute={v_refute} 一致"
    v_tie = _vote(GATE_TIEBREAK, read, "in_scope", on_fail=False)           # 分歧裁决，失败也倾向排除
    return v_tie, f"admit={v_admit}/refute={v_refute} 分歧→裁决={v_tie}"


# ---------- ③ 作用域候选 + 回扣 ----------
def _slim(knowledges: list[dict]) -> list[dict]:
    slim = []
    for k in knowledges:
        e = {"id": k.get("id"), "type": k.get("type"), "title": k.get("title")}
        if k.get("type") == "how":
            e["steps"] = [{"id": s.get("id"), "input": s.get("input"),
                           "directive": (s.get("directive") or "")[:500], "output": s.get("output")}
                          for s in k.get("steps", [])]
        else:
            e["内容"] = {x: k.get(x) for x in ("界定", "主体", "主张", "支撑") if k.get(x)}
        slim.append(e)
    return slim


def scope_candidates(knowledges: list[dict]) -> list:
    user = ("给下面每颗知识标作用域候选（how 逐步：每个 step 一组；what/why 颗级：整颗一组）。\n"
            "只输出 JSON：{\"scopes\":[{\"knowledge_id\":\"k1\",\"step_id\":\"s1\",\"items\":[{\"scope_type\":\"substance\",\"value\":\"…\"}]},"
            "{\"knowledge_id\":\"k2\",\"step_id\":null,\"items\":[...]}]}\n\n"
            + json.dumps(_slim(knowledges), ensure_ascii=False))
    return chat_json(PHASE2, user, timeout=120).get("scopes") or []


def strip_verb_tail(v: str) -> str:
    """① 窄表确定性兜底：砍掉值开头/结尾的无歧义裸动词；砍到 <2 字则回退原值。"""
    if not v or len(v) < 3:
        return v
    for verb in _STRIP_VERBS:                       # 开头裸动词
        if v.startswith(verb) and len(v) - len(verb) >= 2:
            v = v[len(verb):]
            break
    for verb in _STRIP_VERBS:                        # 结尾裸动词（_PROTECT 与之不相交，名词语素天然不在表里）
        if v.endswith(verb) and len(v) - len(verb) >= 2:
            v = v[:-len(verb)]
            break
    return v


def nounify_scopes(scopes: list) -> list:
    """作用域值名词化：③ 一次批量 LLM 名词化（语义，不误砍"氛围营造"）→ ① 窄表兜底。在回扣前做。"""
    vals = sorted({it["value"] for sc in scopes for it in (sc.get("items") or []) if it.get("value")})
    if not vals:
        return scopes
    try:
        mp = chat_json(NORMALIZE, json.dumps(vals, ensure_ascii=False), timeout=90).get("映射") or {}
    except Exception:
        mp = {}
    for sc in scopes:
        for it in sc.get("items") or []:
            v = it.get("value")
            if not v:
                continue
            it["value"] = strip_verb_tail(mp.get(v) or v)   # ③ 映射优先，再 ① 兜底
    return scopes


def link_scope(linker: ScopeLinker, scope_type: str, value: str) -> dict:
    try:
        hits = linker.link(value, source_type=SRC2CN.get(scope_type, scope_type), top_k=3)
    except Exception:
        hits = []
    top = hits[0] if hits else {}
    score = float(top.get("score", 0.0))
    reuse = score >= REUSE_THRESHOLD and top.get("name")
    return {"scope_type": scope_type, "value": top["name"] if reuse else value,
            "candidate": value, "link": "复用" if reuse else "新建", "score": round(score, 4),
            "top": [{"name": h["name"], "score": h["score"], "path": h.get("path", "")} for h in hits]}


def apply_scopes(knowledges: list[dict], scopes: list, linker: ScopeLinker) -> None:
    by_k = {k.get("id"): k for k in knowledges}
    for sc in scopes:
        k = by_k.get(sc.get("knowledge_id"))
        if not k:
            continue
        linked = [link_scope(linker, it["scope_type"], it["value"])
                  for it in (sc.get("items") or []) if it.get("scope_type") and it.get("value")]
        if k.get("type") == "how" and sc.get("step_id"):
            for s in k.get("steps", []):
                if s.get("id") == sc["step_id"]:
                    s["作用域"] = linked
        else:
            k["作用域"] = (k.get("作用域") or []) + linked


# ---------- ⑤ 组装 ----------
def build_content(k: dict) -> str:
    t = k.get("type")
    if t == "how":
        lines = [f"目标：{k.get('purpose','')}"]
        for i, s in enumerate(k.get("steps", []), 1):
            lines += [f"步骤{i}",
                      f"  输入：{s.get('input','')}", f"  方法：{s.get('directive','')}", f"  产出：{s.get('output','')}"]
        return "\n".join(lines)
    if t == "what":
        head = f"界定：{k.get('界定','')}"
        if k.get("kind"):
            head += f"（{k['kind']}型）"
        return "\n".join([head] + _sections(k.get("主体")))
    return "\n".join([f"主张：{k.get('主张','')}"] + _sections(k.get("支撑")))


def _sections(blocks) -> list[str]:
    """把 what.主体 / why.支撑 的自由小节拼成文本行。"""
    out = []
    for b in blocks or []:
        head = b.get("小标题") or ""
        form = b.get("形式")
        out.append(f"【{head}】" + (f"（{form}）" if form else ""))
        if b.get("内容"):
            out.append(f"  {b['内容']}")
        for it in b.get("条目") or []:
            word = it.get("词") or it.get("要素") or ""
            cue = it.get("选择线索")
            line = f"  - {word}：{it.get('说明','')}" if word else f"  - {it.get('说明','')}"
            if cue:
                line += f"（选用：{cue}）"
            out.append(line)
    return out


def build_payload(post, k: dict, how_titles: dict | None = None) -> dict:
    how_titles = how_titles or {}
    t = k.get("type")
    scopes, seen = [], set()
    def add(lst):
        for sc in lst:
            key = (sc["scope_type"], sc["value"])
            if key not in seen:
                seen.add(key); scopes.append({"scope_type": sc["scope_type"], "value": sc["value"]})
    if t == "how":
        for s in k.get("steps", []):
            add(s.get("作用域", []))
    else:
        add(k.get("作用域", []))
    ext = [{"key": "业务阶段", "type": "str", "value": v} for v in (k.get("业务阶段") or [])]
    if t == "how":
        cs, cseen = [], set()
        for s in k.get("steps", []):
            c = s.get("创作阶段")
            if c and c not in cseen:
                cseen.add(c); cs.append(c)
        ext += [{"key": "创作阶段", "type": "str", "value": v} for v in cs]
        ext += [{"key": "动作", "type": "str", "value": s["动作"]} for s in k.get("steps", []) if s.get("动作")]
    if k.get("role") == "组件" and k.get("parent"):
        p = k["parent"]
        ext.append({"key": "出自", "type": "str",
                    "value": f"{how_titles.get(p.get('how_id'), p.get('how_id'))} 第{p.get('step')}步"})
    return {"source": {"id": post.id, "source_type": "post", "title": post.title or "",
                       "author": post.author_name or "", "source_metadata": {"platform": post.platform, "url": post.url}},
            "title": k.get("title"), "content": build_content(k),
            "dim_creations": ["创作"], "dim_attributes": [TYPE2ATTR.get(t, "how工序")],
            "scopes": scopes, "custom_ext": ext}


def main() -> None:
    settings = Settings.from_env()
    extractor = GeminiExtractor.from_env()
    linker = ScopeLinker()
    posts_out, payloads = [], []
    for src in SOURCES:
        cid = src["cid"]
        print(f"\n=== {src['platform']} {cid[:10]} ({src['from']}) ===")
        try:
            post = load_post(src, settings)
            read, is_empty, media, cards = read_one(src, post, settings, extractor)
        except Exception as exc:
            print(f"  ✗ 取数/读懂失败：{exc}")
            posts_out.append({"post_id": cid, "source_id": cid, "title": f"(取数失败 {cid})",
                              "platform": src["platform"], "url": "", "media": {"type": "image", "images": []},
                              "cards": [], "error": str(exc)[:200], "knowledges": []})
            continue
        meta = {"post_id": cid, "source_id": post.id, "title": post.title or "",
                "platform": post.platform, "url": post.url, "media": media, "cards": cards}
        if is_empty:  # ① 总闸：纯展示/无可提取
            print(f"  ① 读懂 {len(read)} 字 → ① 判纯展示/无知识，跳过")
            posts_out.append({**meta, "no_knowledge": True, "knowledges": []})
            continue
        in_scope, gate_why = creation_gate(read)  # ①.5 创作判定闸（创作 vs 制作 vs 越界）
        if not in_scope:
            print(f"  ① 读懂 {len(read)} 字 → ①.5 闸判【非创作】（{gate_why}）→ 整帖排除")
            posts_out.append({**meta, "no_knowledge": True, "knowledges": []})
            continue
        print(f"  ① 读懂 {len(read)} 字（{media['type']}）· ①.5 闸：创作（{gate_why}）")
        knowledges = shape(post, read)
        how_ids = {k.get("id") for k in knowledges if k.get("type") == "how"}
        how_titles = {k.get("id"): k.get("title") for k in knowledges if k.get("type") == "how"}
        for k in knowledges:
            k["业务阶段"] = [b for b in (k.get("业务阶段") or []) if b in ("灵感", "选题", "脚本")]  # 守卫：只留合法业务阶段
            for s in k.get("steps", []):  # 守卫：创作阶段只留合法 5 值，非法（如"定稿/输出"）丢弃
                if s.get("创作阶段") not in CSTAGE:
                    s["创作阶段"] = None
            if k.get("role") == "组件" and (k.get("parent") or {}).get("how_id") not in how_ids:  # 守卫：组件 parent 必指向同帖 how
                k["role"] = "主"; k["parent"] = None
        print(f"  ② {len(knowledges)} 颗：" + ", ".join(f"{k.get('type')}/{k.get('role')}" for k in knowledges))
        apply_scopes(knowledges, nounify_scopes(scope_candidates(knowledges)), linker)  # ③名词化+①兜底 → 回扣
        print("  ③⑤ 作用域回扣 + 组装")
        posts_out.append({**meta, "knowledges": knowledges})
        payloads += [build_payload(post, k, how_titles) for k in knowledges]
    suffix = os.environ.get("OUT", "")  # OUT=_v2 → 写 frameworks_v2.json，不覆盖原版
    (ROOT / f"web/frameworks{suffix}.json").write_text(
        json.dumps({"count": len(posts_out), "posts": posts_out}, ensure_ascii=False, indent=1), encoding="utf-8")
    (ROOT / f"web/payloads{suffix}.json").write_text(
        json.dumps(payloads, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"\nwrote {len(posts_out)} posts, {len(payloads)} payloads → web/frameworks{suffix}.json")


if __name__ == "__main__":
    main()