|
@@ -0,0 +1,720 @@
|
|
|
|
|
+"""Adapters for the input services used by the legacy Script Build runtime.
|
|
|
|
|
+
|
|
|
|
|
+The code intentionally reproduces only the old tools that were present in the
|
|
|
|
|
+Script Agent allowlist. It has no dependency on legacy logging or output tables.
|
|
|
|
|
+"""
|
|
|
|
|
+# ruff: noqa: RUF001
|
|
|
|
|
+
|
|
|
|
|
+from __future__ import annotations
|
|
|
|
|
+
|
|
|
|
|
+import asyncio
|
|
|
|
|
+import html
|
|
|
|
|
+import json
|
|
|
|
|
+from collections.abc import Mapping
|
|
|
|
|
+from hashlib import sha256
|
|
|
|
|
+from pathlib import Path
|
|
|
|
|
+from typing import Any
|
|
|
|
|
+
|
|
|
|
|
+from script_build_host.adapters.retrieval import SafeHttpClient
|
|
|
|
|
+from script_build_host.domain.errors import ProtocolViolation
|
|
|
|
|
+from script_build_host.infrastructure.canonical_json import canonical_sha256
|
|
|
|
|
+from script_build_host.infrastructure.redaction import redact
|
|
|
|
|
+from script_build_host.tools.gateway import RetrievalResult
|
|
|
|
|
+
|
|
|
|
|
+_ATOM_FIELD_KEYS = ("主题原子点", "形式原子点", "作用原子点", "感受原子点")
|
|
|
|
|
+_ATOM_KEY_TO_COLUMN = {
|
|
|
|
|
+ "主题原子点": "主题",
|
|
|
|
|
+ "形式原子点": "形式",
|
|
|
|
|
+ "作用原子点": "作用",
|
|
|
|
|
+ "感受原子点": "感受",
|
|
|
|
|
+}
|
|
|
|
|
+_COLUMN_TO_ATOM_KEY = {value: key for key, value in _ATOM_KEY_TO_COLUMN.items()}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class LegacyOpenRouterClient:
|
|
|
|
|
+ """Small credential-contained client for the two old OpenRouter calls."""
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(
|
|
|
|
|
+ self,
|
|
|
|
|
+ http: SafeHttpClient,
|
|
|
|
|
+ *,
|
|
|
|
|
+ api_key: str,
|
|
|
|
|
+ embedding_endpoint: str,
|
|
|
|
|
+ chat_endpoint: str,
|
|
|
|
|
+ embedding_model: str,
|
|
|
|
|
+ embedding_dimension: int,
|
|
|
|
|
+ ) -> None:
|
|
|
|
|
+ self._http = http
|
|
|
|
|
+ self._headers = {
|
|
|
|
|
+ "Authorization": f"Bearer {api_key}",
|
|
|
|
|
+ "Content-Type": "application/json",
|
|
|
|
|
+ }
|
|
|
|
|
+ self.embedding_endpoint = embedding_endpoint
|
|
|
|
|
+ self.chat_endpoint = chat_endpoint
|
|
|
|
|
+ self.embedding_model = embedding_model
|
|
|
|
|
+ self.embedding_dimension = embedding_dimension
|
|
|
|
|
+
|
|
|
|
|
+ async def embed_one(self, text: str) -> list[float]:
|
|
|
|
|
+ payload = await self._http.json(
|
|
|
|
|
+ "POST",
|
|
|
|
|
+ self.embedding_endpoint,
|
|
|
|
|
+ json_body={
|
|
|
|
|
+ "model": self.embedding_model,
|
|
|
|
|
+ "input": [(text or "")[:300]],
|
|
|
|
|
+ "encoding_format": "float",
|
|
|
|
|
+ "dimensions": self.embedding_dimension,
|
|
|
|
|
+ },
|
|
|
|
|
+ headers=self._headers,
|
|
|
|
|
+ )
|
|
|
|
|
+ rows = payload.get("data") if isinstance(payload, dict) else None
|
|
|
|
|
+ if not isinstance(rows, list) or len(rows) != 1 or not isinstance(rows[0], dict):
|
|
|
|
|
+ raise ProtocolViolation("embedding response has an invalid data array")
|
|
|
|
|
+ vector = rows[0].get("embedding")
|
|
|
|
|
+ if not isinstance(vector, list) or len(vector) != self.embedding_dimension:
|
|
|
|
|
+ raise ProtocolViolation("embedding response dimension does not match the index")
|
|
|
|
|
+ if any(isinstance(value, bool) or not isinstance(value, (int, float)) for value in vector):
|
|
|
|
|
+ raise ProtocolViolation("embedding response contains a non-numeric value")
|
|
|
|
|
+ return [float(value) for value in vector]
|
|
|
|
|
+
|
|
|
|
|
+ async def chat_json(
|
|
|
|
|
+ self,
|
|
|
|
|
+ *,
|
|
|
|
|
+ model: str,
|
|
|
|
|
+ messages: list[dict[str, Any]],
|
|
|
|
|
+ max_tokens: int | None = None,
|
|
|
|
|
+ ) -> Any:
|
|
|
|
|
+ body: dict[str, Any] = {"model": model, "messages": messages, "temperature": 0}
|
|
|
|
|
+ if max_tokens is not None:
|
|
|
|
|
+ body["max_tokens"] = max_tokens
|
|
|
|
|
+ payload = await self._http.json(
|
|
|
|
|
+ "POST", self.chat_endpoint, json_body=body, headers=self._headers
|
|
|
|
|
+ )
|
|
|
|
|
+ choices = payload.get("choices") if isinstance(payload, dict) else None
|
|
|
|
|
+ if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict):
|
|
|
|
|
+ raise ProtocolViolation("chat response has no choice")
|
|
|
|
|
+ message = choices[0].get("message")
|
|
|
|
|
+ content = message.get("content") if isinstance(message, dict) else None
|
|
|
|
|
+ if not isinstance(content, str) or not content.strip():
|
|
|
|
|
+ raise ProtocolViolation("chat response has no text content")
|
|
|
|
|
+ return _parse_json_text(content)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class LegacyVectorDecodeRetrievalAdapter:
|
|
|
|
|
+ """Local legacy vector index plus the legacy OpenRouter query embedding."""
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(
|
|
|
|
|
+ self,
|
|
|
|
|
+ *,
|
|
|
|
|
+ index_root: Path,
|
|
|
|
|
+ raw_root: Path,
|
|
|
|
|
+ openrouter: LegacyOpenRouterClient | None,
|
|
|
|
|
+ min_score: float = 0.7,
|
|
|
|
|
+ ) -> None:
|
|
|
|
|
+ self.index_root = index_root.resolve()
|
|
|
|
|
+ self.raw_root = raw_root.resolve()
|
|
|
|
|
+ self.openrouter = openrouter
|
|
|
|
|
+ self.min_score = min_score
|
|
|
|
|
+ self._loaded: tuple[list[dict[str, Any]], Any, dict[str, Any]] | None = None
|
|
|
|
|
+ self._load_lock = asyncio.Lock()
|
|
|
|
|
+
|
|
|
|
|
+ async def retrieve(self, *, query: Mapping[str, Any], snapshot: Any) -> RetrievalResult:
|
|
|
|
|
+ del snapshot
|
|
|
|
|
+ chunks, embeddings, manifest = await self._index()
|
|
|
|
|
+ keyword = str(query.get("keyword") or "").strip()
|
|
|
|
|
+ account = str(query.get("account_name") or "").strip() or None
|
|
|
|
|
+ match_fields = query.get("match_fields") or {}
|
|
|
|
|
+ if not isinstance(match_fields, Mapping):
|
|
|
|
|
+ raise ProtocolViolation("decode match_fields must be an object")
|
|
|
|
|
+ candidate_indices = [
|
|
|
|
|
+ index
|
|
|
|
|
+ for index, chunk in enumerate(chunks)
|
|
|
|
|
+ if _chunk_matches_filter(chunk, match_fields)
|
|
|
|
|
+ and (account is None or chunk.get("account") == account)
|
|
|
|
|
+ ]
|
|
|
|
|
+ top_k = int(query.get("top_k", 3))
|
|
|
|
|
+ ranked: list[tuple[str, float | None, dict[str, Any]]] = []
|
|
|
|
|
+ if not keyword:
|
|
|
|
|
+ seen: set[str] = set()
|
|
|
|
|
+ for index in candidate_indices:
|
|
|
|
|
+ post_id = str(chunks[index]["post_id"])
|
|
|
|
|
+ if post_id in seen:
|
|
|
|
|
+ continue
|
|
|
|
|
+ seen.add(post_id)
|
|
|
|
|
+ ranked.append((post_id, None, chunks[index]))
|
|
|
|
|
+ if len(ranked) >= top_k:
|
|
|
|
|
+ break
|
|
|
|
|
+ else:
|
|
|
|
|
+ if self.openrouter is None:
|
|
|
|
|
+ raise ProtocolViolation(
|
|
|
|
|
+ "semantic decode search requires SCRIPT_BUILD_OPENROUTER_API_KEY"
|
|
|
|
|
+ )
|
|
|
|
|
+ vector = await self.openrouter.embed_one(keyword)
|
|
|
|
|
+ ranked.extend(
|
|
|
|
|
+ await asyncio.to_thread(
|
|
|
|
|
+ _rank_decode_posts,
|
|
|
|
|
+ chunks,
|
|
|
|
|
+ embeddings,
|
|
|
|
|
+ candidate_indices,
|
|
|
|
|
+ vector,
|
|
|
|
|
+ self.min_score,
|
|
|
|
|
+ top_k,
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ return_field = str(query["return_field"])
|
|
|
|
|
+ output: list[dict[str, Any]] = []
|
|
|
|
|
+ refs: list[str] = []
|
|
|
|
|
+ scores: list[float | None] = []
|
|
|
|
|
+ for rank, (post_id, score, chunk) in enumerate(ranked):
|
|
|
|
|
+ post = await asyncio.to_thread(self._read_post, str(chunk["json_relpath"]))
|
|
|
|
|
+ formatted = {
|
|
|
|
|
+ "post_id": post_id,
|
|
|
|
|
+ "score": round(score, 4) if score is not None else None,
|
|
|
|
|
+ "account": chunk.get("account"),
|
|
|
|
|
+ "return_field": return_field,
|
|
|
|
|
+ "data": _format_post_decode_result(post, return_field),
|
|
|
|
|
+ }
|
|
|
|
|
+ output.append(formatted)
|
|
|
|
|
+ refs.append(f"decode:{_safe_id(post_id, rank)}")
|
|
|
|
|
+ scores.append(round(score, 4) if score is not None else None)
|
|
|
|
|
+ safe_output = redact(output)
|
|
|
|
|
+ if not isinstance(safe_output, list):
|
|
|
|
|
+ raise ProtocolViolation("redacted decode result must remain an array")
|
|
|
|
|
+ index_digest = canonical_sha256(manifest).wire
|
|
|
|
|
+ return RetrievalResult(
|
|
|
|
|
+ source_refs=tuple(refs),
|
|
|
|
|
+ summary=json.dumps(safe_output, ensure_ascii=False, sort_keys=True),
|
|
|
|
|
+ supports=tuple(return_field for _ in refs),
|
|
|
|
|
+ confidence="ranked",
|
|
|
|
|
+ limitations=(),
|
|
|
|
|
+ metadata={
|
|
|
|
|
+ "index_manifest_sha256": index_digest,
|
|
|
|
|
+ "embedding_model": manifest.get("model"),
|
|
|
|
|
+ "hit_ids": refs,
|
|
|
|
|
+ "ranks": list(range(1, len(refs) + 1)),
|
|
|
|
|
+ "scores": scores,
|
|
|
|
|
+ },
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ async def _index(self) -> tuple[list[dict[str, Any]], Any, dict[str, Any]]:
|
|
|
|
|
+ if self._loaded is not None:
|
|
|
|
|
+ return self._loaded
|
|
|
|
|
+ async with self._load_lock:
|
|
|
|
|
+ if self._loaded is None:
|
|
|
|
|
+ self._loaded = await asyncio.to_thread(_load_decode_index, self.index_root)
|
|
|
|
|
+ return self._loaded
|
|
|
|
|
+
|
|
|
|
|
+ def _read_post(self, relpath_text: str) -> dict[str, Any]:
|
|
|
|
|
+ relpath = Path(relpath_text)
|
|
|
|
|
+ parts = relpath.parts
|
|
|
|
|
+ if parts and parts[0] == "script_decode_raw_data":
|
|
|
|
|
+ relpath = Path(*parts[1:])
|
|
|
|
|
+ path = (self.raw_root / relpath).resolve()
|
|
|
|
|
+ if not path.is_relative_to(self.raw_root) or not path.is_file():
|
|
|
|
|
+ raise ProtocolViolation("decode post path is outside the configured raw root")
|
|
|
|
|
+ try:
|
|
|
|
|
+ payload = json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
|
|
|
+ raise ProtocolViolation("decode post is not valid UTF-8 JSON") from exc
|
|
|
|
|
+ if not isinstance(payload, dict):
|
|
|
|
|
+ raise ProtocolViolation("decode post root must be an object")
|
|
|
|
|
+ return payload
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class LegacyLlmKnowledgeRetrievalAdapter:
|
|
|
|
|
+ def __init__(
|
|
|
|
|
+ self,
|
|
|
|
|
+ knowledge_path: Path,
|
|
|
|
|
+ *,
|
|
|
|
|
+ openrouter: LegacyOpenRouterClient,
|
|
|
|
|
+ model: str,
|
|
|
|
|
+ ) -> None:
|
|
|
|
|
+ self.openrouter = openrouter
|
|
|
|
|
+ self.model = model
|
|
|
|
|
+ try:
|
|
|
|
|
+ payload = json.loads(knowledge_path.read_text(encoding="utf-8"))
|
|
|
|
|
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
|
|
|
+ raise ProtocolViolation("knowledge source is not valid UTF-8 JSON") from exc
|
|
|
|
|
+ if not isinstance(payload, list):
|
|
|
|
|
+ raise ProtocolViolation("knowledge source root must be an array")
|
|
|
|
|
+ self.items = [item for item in payload if isinstance(item, dict)]
|
|
|
|
|
+ self.digest = canonical_sha256(payload).wire
|
|
|
|
|
+
|
|
|
|
|
+ async def retrieve(self, *, query: Mapping[str, Any], snapshot: Any) -> RetrievalResult:
|
|
|
|
|
+ del snapshot
|
|
|
|
|
+ max_count = int(query.get("max_count", 3))
|
|
|
|
|
+ index = [
|
|
|
|
|
+ {
|
|
|
|
|
+ "id": item.get("id"),
|
|
|
|
|
+ "title": item.get("title", ""),
|
|
|
|
|
+ "purpose": item.get("purpose", ""),
|
|
|
|
|
+ "steps": item.get("steps", []),
|
|
|
|
|
+ }
|
|
|
|
|
+ for item in self.items
|
|
|
|
|
+ ]
|
|
|
|
|
+ selected = await self.openrouter.chat_json(
|
|
|
|
|
+ model=self.model,
|
|
|
|
|
+ messages=[
|
|
|
|
|
+ {
|
|
|
|
|
+ "role": "system",
|
|
|
|
|
+ "content": (
|
|
|
|
|
+ "你是一个创作知识检索助手。根据给定的检索关键词,从知识条目列表中"
|
|
|
|
|
+ '找出最相关的条目 ID。只返回 JSON 对象,格式:{"ids": [<id>, ...]}。'
|
|
|
|
|
+ "不要输出任何其他内容。"
|
|
|
|
|
+ ),
|
|
|
|
|
+ },
|
|
|
|
|
+ {
|
|
|
|
|
+ "role": "user",
|
|
|
|
|
+ "content": (
|
|
|
|
|
+ f"检索关键词:{query.get('keyword', '')}\n最多返回 {max_count} 个最相关的"
|
|
|
|
|
+ f"条目 ID(按相关度从高到低排列)。\n\n知识条目列表:\n"
|
|
|
|
|
+ + json.dumps(index, ensure_ascii=False, indent=2)
|
|
|
|
|
+ ),
|
|
|
|
|
+ },
|
|
|
|
|
+ ],
|
|
|
|
|
+ )
|
|
|
|
|
+ ids = selected.get("ids", []) if isinstance(selected, dict) else []
|
|
|
|
|
+ by_id = {item.get("id"): item for item in self.items}
|
|
|
|
|
+ output = []
|
|
|
|
|
+ for identifier in ids[:max_count] if isinstance(ids, list) else []:
|
|
|
|
|
+ item = by_id.get(identifier)
|
|
|
|
|
+ if item is not None:
|
|
|
|
|
+ output.append(
|
|
|
|
|
+ {
|
|
|
|
|
+ "id": item.get("id"),
|
|
|
|
|
+ "title": item.get("title", ""),
|
|
|
|
|
+ "purpose": item.get("purpose", ""),
|
|
|
|
|
+ "content": _knowledge_content(item),
|
|
|
|
|
+ }
|
|
|
|
|
+ )
|
|
|
|
|
+ refs = tuple(
|
|
|
|
|
+ f"knowledge:{_safe_id(item.get('id'), rank)}" for rank, item in enumerate(output)
|
|
|
|
|
+ )
|
|
|
|
|
+ return RetrievalResult(
|
|
|
|
|
+ source_refs=refs,
|
|
|
|
|
+ summary=json.dumps(output, ensure_ascii=False, sort_keys=True),
|
|
|
|
|
+ supports=tuple(
|
|
|
|
|
+ str(item.get("title") or ref) for item, ref in zip(output, refs, strict=True)
|
|
|
|
|
+ ),
|
|
|
|
|
+ confidence="model-selected",
|
|
|
|
|
+ limitations=(),
|
|
|
|
|
+ metadata={"file_sha256": self.digest, "hit_ids": list(refs)},
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class LegacyExternalRetrievalAdapter:
|
|
|
|
|
+ """Exact active XHS/Zhihu retrieval path from the old Script tool."""
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(
|
|
|
|
|
+ self,
|
|
|
|
|
+ *,
|
|
|
|
|
+ xhs_search_endpoint: str,
|
|
|
|
|
+ xhs_detail_endpoint: str,
|
|
|
|
|
+ zhihu_search_endpoint: str,
|
|
|
|
|
+ http: SafeHttpClient,
|
|
|
|
|
+ openrouter: LegacyOpenRouterClient | None = None,
|
|
|
|
|
+ image_model: str = "google/gemini-3-flash-preview",
|
|
|
|
|
+ ) -> None:
|
|
|
|
|
+ self.xhs_search_endpoint = xhs_search_endpoint
|
|
|
|
|
+ self.xhs_detail_endpoint = xhs_detail_endpoint
|
|
|
|
|
+ self.zhihu_search_endpoint = zhihu_search_endpoint
|
|
|
|
|
+ self.http = http
|
|
|
|
|
+ self.openrouter = openrouter
|
|
|
|
|
+ self.image_model = image_model
|
|
|
|
|
+
|
|
|
|
|
+ async def retrieve(self, *, query: Mapping[str, Any], snapshot: Any) -> RetrievalResult:
|
|
|
|
|
+ del snapshot
|
|
|
|
|
+ keyword = str(query.get("keyword") or "")
|
|
|
|
|
+ max_count = int(query.get("max_count", 5))
|
|
|
|
|
+ platform = query.get("platform_channel", "xhs")
|
|
|
|
|
+ rows = (
|
|
|
|
|
+ await self._search_xhs(keyword, max_count)
|
|
|
|
|
+ if platform == "xhs"
|
|
|
|
|
+ else await self._search_zhihu(keyword, max_count)
|
|
|
|
|
+ )
|
|
|
|
|
+ await asyncio.gather(*(self._add_image_understanding(row) for row in rows))
|
|
|
|
|
+ safe_rows = redact(rows)
|
|
|
|
|
+ if not isinstance(safe_rows, list):
|
|
|
|
|
+ raise ProtocolViolation("redacted external result must remain an array")
|
|
|
|
|
+ refs = tuple(
|
|
|
|
|
+ f"external:{_safe_id(item.get('channel_content_id', item.get('id')), rank)}"
|
|
|
|
|
+ for rank, item in enumerate(safe_rows)
|
|
|
|
|
+ )
|
|
|
|
|
+ return RetrievalResult(
|
|
|
|
|
+ source_refs=refs,
|
|
|
|
|
+ summary=json.dumps(safe_rows, ensure_ascii=False, sort_keys=True),
|
|
|
|
|
+ supports=refs,
|
|
|
|
|
+ confidence="upstream",
|
|
|
|
|
+ limitations=("image understanding unavailable",) if self.openrouter is None else (),
|
|
|
|
|
+ metadata={"response_sha256": canonical_sha256(rows).wire},
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ async def _search_xhs(self, keyword: str, max_count: int) -> list[dict[str, Any]]:
|
|
|
|
|
+ cursor: str | int = ""
|
|
|
|
|
+ output: list[dict[str, Any]] = []
|
|
|
|
|
+ for _ in range(2):
|
|
|
|
|
+ body = await self.http.json(
|
|
|
|
|
+ "POST",
|
|
|
|
|
+ self.xhs_search_endpoint,
|
|
|
|
|
+ json_body={
|
|
|
|
|
+ "keyword": keyword,
|
|
|
|
|
+ "content_type": "图文",
|
|
|
|
|
+ "sort_type": "综合",
|
|
|
|
|
+ "publish_time": "",
|
|
|
|
|
+ "cursor": cursor,
|
|
|
|
|
+ },
|
|
|
|
|
+ )
|
|
|
|
|
+ if not isinstance(body, dict) or body.get("code") not in (0, "0"):
|
|
|
|
|
+ raise ProtocolViolation("XHS search returned a non-zero code")
|
|
|
|
|
+ block = body.get("data") or {}
|
|
|
|
|
+ raw_items = block.get("data") if isinstance(block, dict) else []
|
|
|
|
|
+ items = [item for item in raw_items or [] if isinstance(item, dict)]
|
|
|
|
|
+ ids = [str(item["id"]).strip() for item in items if str(item.get("id") or "").strip()]
|
|
|
|
|
+ detail_values = await asyncio.gather(
|
|
|
|
|
+ *(self._xhs_detail(identifier) for identifier in ids)
|
|
|
|
|
+ )
|
|
|
|
|
+ detail_map = {
|
|
|
|
|
+ identifier: value
|
|
|
|
|
+ for identifier, value in zip(ids, detail_values, strict=True)
|
|
|
|
|
+ if value is not None
|
|
|
|
|
+ }
|
|
|
|
|
+ for item in items:
|
|
|
|
|
+ identifier = str(item.get("id") or "").strip()
|
|
|
|
|
+ if identifier and identifier in detail_map:
|
|
|
|
|
+ output.append(detail_map[identifier])
|
|
|
|
|
+ elif not identifier:
|
|
|
|
|
+ output.append(_xhs_card(item))
|
|
|
|
|
+ if len(output) >= max_count:
|
|
|
|
|
+ return output[:max_count]
|
|
|
|
|
+ if not block.get("has_more") or block.get("next_cursor") in (None, ""):
|
|
|
|
|
+ break
|
|
|
|
|
+ cursor = block["next_cursor"]
|
|
|
|
|
+ return output[:max_count]
|
|
|
|
|
+
|
|
|
|
|
+ async def _xhs_detail(self, identifier: str) -> dict[str, Any] | None:
|
|
|
|
|
+ try:
|
|
|
|
|
+ body = await self.http.json(
|
|
|
|
|
+ "POST", self.xhs_detail_endpoint, json_body={"content_id": identifier}
|
|
|
|
|
+ )
|
|
|
|
|
+ except ProtocolViolation:
|
|
|
|
|
+ return None
|
|
|
|
|
+ inner = (
|
|
|
|
|
+ body.get("data") if isinstance(body, dict) and body.get("code") in (0, "0") else None
|
|
|
|
|
+ )
|
|
|
|
|
+ note = inner.get("data") if isinstance(inner, dict) else None
|
|
|
|
|
+ return _xhs_detail(note) if isinstance(note, dict) else None
|
|
|
|
|
+
|
|
|
|
|
+ async def _search_zhihu(self, keyword: str, max_count: int) -> list[dict[str, Any]]:
|
|
|
|
|
+ payload = {
|
|
|
|
|
+ "type": "zhihu",
|
|
|
|
|
+ "keyword": keyword,
|
|
|
|
|
+ "cursor": "",
|
|
|
|
|
+ "content_type": "图文",
|
|
|
|
|
+ "max_count": max_count,
|
|
|
|
|
+ }
|
|
|
|
|
+ last: Any = None
|
|
|
|
|
+ for _ in range(3):
|
|
|
|
|
+ body = await self.http.json("POST", self.zhihu_search_endpoint, json_body=payload)
|
|
|
|
|
+ last = body
|
|
|
|
|
+ data = (
|
|
|
|
|
+ body.get("data")
|
|
|
|
|
+ if isinstance(body, dict) and body.get("code") in (0, "0")
|
|
|
|
|
+ else None
|
|
|
|
|
+ )
|
|
|
|
|
+ if isinstance(data, list) and data:
|
|
|
|
|
+ return [item for item in data if isinstance(item, dict)][:max_count]
|
|
|
|
|
+ raise ProtocolViolation(f"Zhihu search returned no data ({canonical_sha256(last).wire})")
|
|
|
|
|
+
|
|
|
|
|
+ async def _add_image_understanding(self, row: dict[str, Any]) -> None:
|
|
|
|
|
+ raw_images = row.get("images")
|
|
|
|
|
+ images = _dedupe_urls(raw_images if isinstance(raw_images, list) else [])
|
|
|
|
|
+ row["images"] = images
|
|
|
|
|
+ if not images or self.openrouter is None:
|
|
|
|
|
+ row["images_text_escape"] = [{"image_url": url, "text": ""} for url in images]
|
|
|
|
|
+ return
|
|
|
|
|
+ prompt = (
|
|
|
|
|
+ f"帖子标题:{row.get('title', '')}\n帖子正文:{row.get('body_text', '')}\n\n"
|
|
|
|
|
+ "请详细描述每张图片的主体、场景、文字、布局、构图和色彩。只输出 JSON 数组,"
|
|
|
|
|
+ "每项包含 image_url 和 text。"
|
|
|
|
|
+ )
|
|
|
|
|
+ content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
|
|
|
|
|
+ content.extend({"type": "image_url", "image_url": {"url": url}} for url in images)
|
|
|
|
|
+ try:
|
|
|
|
|
+ parsed = await self.openrouter.chat_json(
|
|
|
|
|
+ model=self.image_model,
|
|
|
|
|
+ messages=[{"role": "user", "content": content}],
|
|
|
|
|
+ max_tokens=8192,
|
|
|
|
|
+ )
|
|
|
|
|
+ except ProtocolViolation:
|
|
|
|
|
+ parsed = []
|
|
|
|
|
+ descriptions: dict[str, str] = {}
|
|
|
|
|
+ if isinstance(parsed, list):
|
|
|
|
|
+ for index, item in enumerate(parsed):
|
|
|
|
|
+ if isinstance(item, dict):
|
|
|
|
|
+ url = str(
|
|
|
|
|
+ item.get("image_url")
|
|
|
|
|
+ or item.get("url")
|
|
|
|
|
+ or (images[index] if index < len(images) else "")
|
|
|
|
|
+ )
|
|
|
|
|
+ descriptions[url] = str(item.get("text") or item.get("description") or "")
|
|
|
|
|
+ row["images_text_escape"] = [
|
|
|
|
|
+ {"image_url": url, "text": html.escape(descriptions.get(url, ""))} for url in images
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _load_decode_index(index_root: Path) -> tuple[list[dict[str, Any]], Any, dict[str, Any]]:
|
|
|
|
|
+ try:
|
|
|
|
|
+ import numpy as np
|
|
|
|
|
+ except ImportError as exc: # pragma: no cover - deployment dependency check
|
|
|
|
|
+ raise ProtocolViolation("numpy is required for the legacy decode index") from exc
|
|
|
|
|
+ try:
|
|
|
|
|
+ manifest = json.loads((index_root / "manifest.json").read_text(encoding="utf-8"))
|
|
|
|
|
+ chunks = [
|
|
|
|
|
+ json.loads(line)
|
|
|
|
|
+ for line in (index_root / "meta.jsonl").read_text(encoding="utf-8").splitlines()
|
|
|
|
|
+ if line.strip()
|
|
|
|
|
+ ]
|
|
|
|
|
+ embeddings = np.load(index_root / "embeddings.npy", mmap_mode="r")
|
|
|
|
|
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
|
|
|
|
|
+ raise ProtocolViolation("legacy decode index is missing or invalid") from exc
|
|
|
|
|
+ if not isinstance(manifest, dict) or len(chunks) != embeddings.shape[0]:
|
|
|
|
|
+ raise ProtocolViolation("legacy decode index metadata and vectors do not match")
|
|
|
|
|
+ if manifest.get("dimension") != embeddings.shape[1]:
|
|
|
|
|
+ raise ProtocolViolation("legacy decode index dimension does not match its manifest")
|
|
|
|
|
+ return chunks, embeddings, manifest
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _rank_decode_posts(
|
|
|
|
|
+ chunks: list[dict[str, Any]],
|
|
|
|
|
+ embeddings: Any,
|
|
|
|
|
+ candidate_indices: list[int],
|
|
|
|
|
+ vector: list[float],
|
|
|
|
|
+ min_score: float,
|
|
|
|
|
+ top_k: int,
|
|
|
|
|
+) -> list[tuple[str, float, dict[str, Any]]]:
|
|
|
|
|
+ import numpy as np
|
|
|
|
|
+
|
|
|
|
|
+ if not candidate_indices:
|
|
|
|
|
+ return []
|
|
|
|
|
+ query = np.asarray(vector, dtype=np.float32)
|
|
|
|
|
+ matrix = embeddings[np.asarray(candidate_indices, dtype=np.int64)].astype(np.float32)
|
|
|
|
|
+ query_norm = np.linalg.norm(query)
|
|
|
|
|
+ norms = np.linalg.norm(matrix, axis=1)
|
|
|
|
|
+ scores = (matrix @ query) / (np.where(norms == 0, 1e-12, norms) * query_norm)
|
|
|
|
|
+ posts: dict[str, tuple[float, dict[str, Any]]] = {}
|
|
|
|
|
+ for local_index, raw_score in enumerate(scores):
|
|
|
|
|
+ score = float(raw_score)
|
|
|
|
|
+ if score < min_score:
|
|
|
|
|
+ continue
|
|
|
|
|
+ chunk = chunks[candidate_indices[local_index]]
|
|
|
|
|
+ post_id = str(chunk["post_id"])
|
|
|
|
|
+ if post_id not in posts or score > posts[post_id][0]:
|
|
|
|
|
+ posts[post_id] = (score, chunk)
|
|
|
|
|
+ return [
|
|
|
|
|
+ (post_id, score, chunk)
|
|
|
|
|
+ for post_id, (score, chunk) in sorted(
|
|
|
|
|
+ posts.items(), key=lambda item: (-item[1][0], item[0])
|
|
|
|
|
+ )[:top_k]
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _chunk_matches_filter(chunk: Mapping[str, Any], fields: Mapping[str, Any]) -> bool:
|
|
|
|
|
+ if not fields:
|
|
|
|
|
+ return True
|
|
|
|
|
+ for column, subfield in fields.items():
|
|
|
|
|
+ if chunk.get("column") == column and (
|
|
|
|
|
+ subfield is None or chunk.get("sub_field") == subfield
|
|
|
|
|
+ ):
|
|
|
|
|
+ return True
|
|
|
|
|
+ return False
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _format_post_decode_result(post: dict[str, Any], return_field: str) -> list[dict[str, Any]]:
|
|
|
|
|
+ paragraphs = [value for value in post.get("段落列表") or [] if isinstance(value, dict)]
|
|
|
|
|
+ if return_field == "主脉络":
|
|
|
|
|
+ return [_main_thread(value) for value in paragraphs]
|
|
|
|
|
+ if return_field == "元素":
|
|
|
|
|
+ return [_elements(value) for value in paragraphs]
|
|
|
|
|
+ if return_field in {"主题", "形式", "作用", "感受"}:
|
|
|
|
|
+ return [_column(value, return_field) for value in paragraphs]
|
|
|
|
|
+ raise ProtocolViolation("unsupported decode return field")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _atom_name(atom: Mapping[str, Any]) -> str:
|
|
|
|
|
+ return str(atom.get("维度名") or atom.get("维度") or "")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _atom_value(atom: Mapping[str, Any]) -> str:
|
|
|
|
|
+ return str(atom.get("维度值") or atom.get("原子点") or "")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _main_thread(paragraph: dict[str, Any]) -> dict[str, Any]:
|
|
|
|
|
+ dimensions = []
|
|
|
|
|
+ for key in _ATOM_FIELD_KEYS:
|
|
|
|
|
+ for atom in paragraph.get(key) or []:
|
|
|
|
|
+ if isinstance(atom, dict) and atom.get("维度类型") == "主维度":
|
|
|
|
|
+ dimensions.append({"维度名": _atom_name(atom), "维度值": _atom_value(atom)})
|
|
|
|
|
+ return {
|
|
|
|
|
+ "id": paragraph.get("id", ""),
|
|
|
|
|
+ "名称": paragraph.get("名称", ""),
|
|
|
|
|
+ "主题描述": paragraph.get("主题", ""),
|
|
|
|
|
+ "详细描述": paragraph.get("描述", ""),
|
|
|
|
|
+ "子项": [
|
|
|
|
|
+ _main_thread(value) for value in paragraph.get("子项") or [] if isinstance(value, dict)
|
|
|
|
|
+ ],
|
|
|
|
|
+ "主维度": dimensions,
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _elements(paragraph: dict[str, Any]) -> dict[str, Any]:
|
|
|
|
|
+ values = []
|
|
|
|
|
+ for key, column in _ATOM_KEY_TO_COLUMN.items():
|
|
|
|
|
+ for atom in paragraph.get(key) or []:
|
|
|
|
|
+ if isinstance(atom, dict) and atom.get("维度类型") in {"从维度", "子元素"}:
|
|
|
|
|
+ values.append(
|
|
|
|
|
+ {"类型": column, "维度名": _atom_name(atom), "维度值": _atom_value(atom)}
|
|
|
|
|
+ )
|
|
|
|
|
+ output: dict[str, Any] = {
|
|
|
|
|
+ "id": paragraph.get("id", ""),
|
|
|
|
|
+ "名称": paragraph.get("名称", ""),
|
|
|
|
|
+ "元素": values,
|
|
|
|
|
+ }
|
|
|
|
|
+ children = [value for value in paragraph.get("子项") or [] if isinstance(value, dict)]
|
|
|
|
|
+ if children:
|
|
|
|
|
+ output["子项"] = [_elements(value) for value in children]
|
|
|
|
|
+ return output
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _column(paragraph: dict[str, Any], column: str) -> dict[str, Any]:
|
|
|
|
|
+ atoms = []
|
|
|
|
|
+ for atom in paragraph.get(_COLUMN_TO_ATOM_KEY[column]) or []:
|
|
|
|
|
+ if isinstance(atom, dict):
|
|
|
|
|
+ atoms.append(
|
|
|
|
|
+ {
|
|
|
|
|
+ "维度名": _atom_name(atom),
|
|
|
|
|
+ "维度值": _atom_value(atom),
|
|
|
|
|
+ "维度类型": str(atom.get("维度类型") or ""),
|
|
|
|
|
+ "维度名描述": str(atom.get("维度名描述") or ""),
|
|
|
|
|
+ "维度值描述": str(atom.get("维度值描述") or ""),
|
|
|
|
|
+ }
|
|
|
|
|
+ )
|
|
|
|
|
+ output: dict[str, Any] = {
|
|
|
|
|
+ "id": paragraph.get("id", ""),
|
|
|
|
|
+ "名称": paragraph.get("名称", ""),
|
|
|
|
|
+ "描述": paragraph.get(column, ""),
|
|
|
|
|
+ "原子点": atoms,
|
|
|
|
|
+ }
|
|
|
|
|
+ children = [value for value in paragraph.get("子项") or [] if isinstance(value, dict)]
|
|
|
|
|
+ if children:
|
|
|
|
|
+ output["子项"] = [_column(value, column) for value in children]
|
|
|
|
|
+ return output
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _knowledge_content(item: Mapping[str, Any]) -> str:
|
|
|
|
|
+ parts = [f"目标:{item.get('purpose', '')}"]
|
|
|
|
|
+ for index, step in enumerate(item.get("steps", []) or [], 1):
|
|
|
|
|
+ if not isinstance(step, dict):
|
|
|
|
|
+ continue
|
|
|
|
|
+ outputs = step.get("outputs") or []
|
|
|
|
|
+ values = ";".join(
|
|
|
|
|
+ str(output.get("value"))
|
|
|
|
|
+ for output in outputs
|
|
|
|
|
+ if isinstance(output, dict) and output.get("value")
|
|
|
|
|
+ )
|
|
|
|
|
+ text = f"步骤{index}(目的:{step.get('intent', '')})\n 指引:{step.get('directive', '')}"
|
|
|
|
|
+ parts.append(text + (f"\n 产出:{values}" if values else ""))
|
|
|
|
|
+ return "\n".join(parts)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _xhs_detail(value: Mapping[str, Any]) -> dict[str, Any]:
|
|
|
|
|
+ images = [
|
|
|
|
|
+ item.get("image_url") if isinstance(item, dict) else item
|
|
|
|
|
+ for item in value.get("image_url_list") or []
|
|
|
|
|
+ ]
|
|
|
|
|
+ videos = [
|
|
|
|
|
+ (item.get("video_url") or item.get("url")) if isinstance(item, dict) else item
|
|
|
|
|
+ for item in value.get("video_url_list") or []
|
|
|
|
|
+ ]
|
|
|
|
|
+ return {
|
|
|
|
|
+ "channel_content_id": str(value.get("channel_content_id") or ""),
|
|
|
|
|
+ "title": value.get("title") or "",
|
|
|
|
|
+ "content_type": value.get("content_type") or "note",
|
|
|
|
|
+ "body_text": value.get("body_text") or "",
|
|
|
|
|
+ "like_count": _integer(value.get("like_count")),
|
|
|
|
|
+ "publish_timestamp": _timestamp_ms(value.get("publish_timestamp")),
|
|
|
|
|
+ "images": _dedupe_urls(images),
|
|
|
|
|
+ "videos": _dedupe_urls(videos),
|
|
|
|
|
+ "channel": "xhs",
|
|
|
|
|
+ "link": value.get("content_link") or "",
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _xhs_card(value: Mapping[str, Any]) -> dict[str, Any]:
|
|
|
|
|
+ raw_card = value.get("note_card")
|
|
|
|
|
+ card: Mapping[str, Any] = raw_card if isinstance(raw_card, dict) else {}
|
|
|
|
|
+ images = [
|
|
|
|
|
+ item.get("image_url") if isinstance(item, dict) else item
|
|
|
|
|
+ for item in card.get("image_list") or []
|
|
|
|
|
+ ]
|
|
|
|
|
+ raw_interact = card.get("interact_info")
|
|
|
|
|
+ interact: Mapping[str, Any] = raw_interact if isinstance(raw_interact, dict) else {}
|
|
|
|
|
+ identifier = str(value.get("id") or "")
|
|
|
|
|
+ return {
|
|
|
|
|
+ "channel_content_id": identifier,
|
|
|
|
|
+ "title": card.get("display_title") or "",
|
|
|
|
|
+ "content_type": "note",
|
|
|
|
|
+ "body_text": card.get("desc") or "",
|
|
|
|
|
+ "like_count": _integer(interact.get("liked_count")),
|
|
|
|
|
+ "publish_timestamp": _timestamp_ms(card.get("publish_timestamp")),
|
|
|
|
|
+ "images": _dedupe_urls(images),
|
|
|
|
|
+ "videos": [],
|
|
|
|
|
+ "channel": "xhs",
|
|
|
|
|
+ "link": f"https://www.xiaohongshu.com/explore/{identifier}" if identifier else "",
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _dedupe_urls(values: list[Any]) -> list[str]:
|
|
|
|
|
+ return list(dict.fromkeys(str(value).strip() for value in values if str(value or "").strip()))
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _integer(value: Any) -> int:
|
|
|
|
|
+ try:
|
|
|
|
|
+ return int(value)
|
|
|
|
|
+ except (TypeError, ValueError):
|
|
|
|
|
+ return 0
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _timestamp_ms(value: Any) -> int:
|
|
|
|
|
+ number = _integer(value)
|
|
|
|
|
+ return number * 1000 if 0 < number < 1_000_000_000_000 else number
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _parse_json_text(value: str) -> Any:
|
|
|
|
|
+ text = value.strip()
|
|
|
|
|
+ if text.startswith("```"):
|
|
|
|
|
+ text = text.split("```", 2)[1]
|
|
|
|
|
+ if text.startswith("json"):
|
|
|
|
|
+ text = text[4:]
|
|
|
|
|
+ try:
|
|
|
|
|
+ return json.loads(text.strip())
|
|
|
|
|
+ except json.JSONDecodeError as exc:
|
|
|
|
|
+ raise ProtocolViolation("model response is not valid JSON") from exc
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _safe_id(value: Any, rank: int) -> str:
|
|
|
|
|
+ text = str(value if value is not None else rank)
|
|
|
|
|
+ if (
|
|
|
|
|
+ text
|
|
|
|
|
+ and len(text) <= 128
|
|
|
|
|
+ and all(character.isalnum() or character in "._:-" for character in text)
|
|
|
|
|
+ ):
|
|
|
|
|
+ return text
|
|
|
|
|
+ return "sha256-" + sha256(text.encode()).hexdigest()[:20]
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+__all__ = [
|
|
|
|
|
+ "LegacyExternalRetrievalAdapter",
|
|
|
|
|
+ "LegacyLlmKnowledgeRetrievalAdapter",
|
|
|
|
|
+ "LegacyOpenRouterClient",
|
|
|
|
|
+ "LegacyVectorDecodeRetrievalAdapter",
|
|
|
|
|
+]
|