Sfoglia il codice sorgente

feat(脚本检索): 将查询向量切换为独立百炼客户端

拆分 embedding 与 OpenRouter 聊天凭据,支持直接读取 DASHSCOPE_API_KEY。\n\n运行时校验查询模型和维度必须与已发布索引一致,避免跨模型向量空间误检;同步补充模型错配测试。
SamLee 1 giorno fa
parent
commit
475daaa884

+ 2 - 0
script_build_host/src/script_build_host/adapters/__init__.py

@@ -5,6 +5,7 @@ from .legacy_retrieval import (
     LegacyLlmKnowledgeRetrievalAdapter,
     LegacyOpenRouterClient,
     LegacyVectorDecodeRetrievalAdapter,
+    OpenAICompatibleEmbeddingClient,
 )
 from .persona import FilePersonaSource
 from .prompts import DatabaseFirstPromptSource
@@ -32,6 +33,7 @@ __all__ = [
     "LegacyLlmKnowledgeRetrievalAdapter",
     "LegacyOpenRouterClient",
     "LegacyVectorDecodeRetrievalAdapter",
+    "OpenAICompatibleEmbeddingClient",
     "PatternRetrievalAdapter",
     "RawArtifactStore",
     "SafeHttpClient",

+ 39 - 11
script_build_host/src/script_build_host/adapters/legacy_retrieval.py

@@ -13,7 +13,7 @@ import json
 from collections.abc import Mapping
 from hashlib import sha256
 from pathlib import Path
-from typing import Any
+from typing import Any, Protocol
 
 from script_build_host.adapters.retrieval import SafeHttpClient
 from script_build_host.domain.errors import ProtocolViolation
@@ -31,8 +31,15 @@ _ATOM_KEY_TO_COLUMN = {
 _COLUMN_TO_ATOM_KEY = {value: key for key, value in _ATOM_KEY_TO_COLUMN.items()}
 
 
-class LegacyOpenRouterClient:
-    """Small credential-contained client for the two old OpenRouter calls."""
+class QueryEmbeddingClient(Protocol):
+    embedding_model: str
+    embedding_dimension: int
+
+    async def embed_one(self, text: str) -> list[float]: ...
+
+
+class OpenAICompatibleEmbeddingClient:
+    """Credential-contained client for an OpenAI-compatible embedding API."""
 
     def __init__(
         self,
@@ -40,7 +47,6 @@ class LegacyOpenRouterClient:
         *,
         api_key: str,
         embedding_endpoint: str,
-        chat_endpoint: str,
         embedding_model: str,
         embedding_dimension: int,
     ) -> None:
@@ -50,7 +56,6 @@ class LegacyOpenRouterClient:
             "Content-Type": "application/json",
         }
         self.embedding_endpoint = embedding_endpoint
-        self.chat_endpoint = chat_endpoint
         self.embedding_model = embedding_model
         self.embedding_dimension = embedding_dimension
 
@@ -76,6 +81,24 @@ class LegacyOpenRouterClient:
             raise ProtocolViolation("embedding response contains a non-numeric value")
         return [float(value) for value in vector]
 
+
+class LegacyOpenRouterClient:
+    """Credential-contained client for the legacy OpenRouter chat calls."""
+
+    def __init__(
+        self,
+        http: SafeHttpClient,
+        *,
+        api_key: str,
+        chat_endpoint: str,
+    ) -> None:
+        self._http = http
+        self._headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+        self.chat_endpoint = chat_endpoint
+
     async def chat_json(
         self,
         *,
@@ -100,19 +123,19 @@ class LegacyOpenRouterClient:
 
 
 class LegacyVectorDecodeRetrievalAdapter:
-    """Local legacy vector index plus the legacy OpenRouter query embedding."""
+    """Local vector index plus a matching query-embedding provider."""
 
     def __init__(
         self,
         *,
         index_root: Path,
         raw_root: Path,
-        openrouter: LegacyOpenRouterClient | None,
+        embedding_client: QueryEmbeddingClient | None,
         min_score: float = 0.7,
     ) -> None:
         self.index_root = index_root.resolve()
         self.raw_root = raw_root.resolve()
-        self.openrouter = openrouter
+        self.embedding_client = embedding_client
         self.min_score = min_score
         self._loaded: tuple[list[dict[str, Any]], Any, dict[str, Any]] | None = None
         self._load_lock = asyncio.Lock()
@@ -144,11 +167,15 @@ class LegacyVectorDecodeRetrievalAdapter:
                 if len(ranked) >= top_k:
                     break
         else:
-            if self.openrouter is None:
+            if self.embedding_client is None:
                 raise ProtocolViolation(
-                    "semantic decode search requires SCRIPT_BUILD_OPENROUTER_API_KEY"
+                    "semantic decode search requires SCRIPT_BUILD_EMBEDDING_API_KEY"
                 )
-            vector = await self.openrouter.embed_one(keyword)
+            if manifest.get("model") != self.embedding_client.embedding_model:
+                raise ProtocolViolation("query embedding model does not match the decode index")
+            if manifest.get("dimension") != self.embedding_client.embedding_dimension:
+                raise ProtocolViolation("query embedding dimension does not match the decode index")
+            vector = await self.embedding_client.embed_one(keyword)
             ranked.extend(
                 await asyncio.to_thread(
                     _rank_decode_posts,
@@ -717,4 +744,5 @@ __all__ = [
     "LegacyLlmKnowledgeRetrievalAdapter",
     "LegacyOpenRouterClient",
     "LegacyVectorDecodeRetrievalAdapter",
+    "OpenAICompatibleEmbeddingClient",
 ]

+ 15 - 3
script_build_host/src/script_build_host/infrastructure/config.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Self
 from urllib.parse import urlsplit
 
-from pydantic import Field, SecretStr, model_validator
+from pydantic import AliasChoices, Field, SecretStr, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from sqlalchemy.engine import make_url
 
@@ -25,7 +25,11 @@ from script_build_host.domain.task_contracts import (
 
 class ScriptBuildSettings(BaseSettings):
     model_config = SettingsConfigDict(
-        env_prefix="SCRIPT_BUILD_", env_file=".env", extra="ignore", case_sensitive=False
+        env_prefix="SCRIPT_BUILD_",
+        env_file=".env",
+        extra="ignore",
+        case_sensitive=False,
+        populate_by_name=True,
     )
 
     environment: str = "production"
@@ -53,9 +57,17 @@ class ScriptBuildSettings(BaseSettings):
     xhs_detail_endpoint: str | None = None
     zhihu_search_endpoint: str | None = None
     image_endpoint: str | None = None
+    embedding_api_key: SecretStr | None = Field(
+        default=None,
+        repr=False,
+        validation_alias=AliasChoices(
+            "SCRIPT_BUILD_EMBEDDING_API_KEY",
+            "DASHSCOPE_API_KEY",
+        ),
+    )
     openrouter_api_key: SecretStr | None = Field(default=None, repr=False)
     openrouter_chat_endpoint: str = "https://openrouter.ai/api/v1/chat/completions"
-    embedding_model: str = "qwen/qwen3-embedding-8b"
+    embedding_model: str = "text-embedding-v4"
     embedding_dimension: int = Field(default=512, ge=1, le=8192)
     knowledge_model: str = "google/gemini-3-flash-preview"
     external_image_model: str = "google/gemini-3-flash-preview"

+ 13 - 5
script_build_host/src/script_build_host/production.py

@@ -21,6 +21,7 @@ from script_build_host.adapters import (
     LegacyLlmKnowledgeRetrievalAdapter,
     LegacyOpenRouterClient,
     LegacyVectorDecodeRetrievalAdapter,
+    OpenAICompatibleEmbeddingClient,
     PatternRetrievalAdapter,
     SafeHttpClient,
     SafeImageAdapter,
@@ -174,15 +175,22 @@ def compose_production_host(
         timeout_seconds=settings.request_timeout_seconds,
         max_response_bytes=settings.max_response_bytes,
     )
+    embedding_client: OpenAICompatibleEmbeddingClient | None = None
+    if settings.embedding_api_key is not None and settings.embedding_endpoint:
+        embedding_client = OpenAICompatibleEmbeddingClient(
+            safe_http,
+            api_key=settings.embedding_api_key.get_secret_value(),
+            embedding_endpoint=settings.embedding_endpoint,
+            embedding_model=settings.embedding_model,
+            embedding_dimension=settings.embedding_dimension,
+        )
+
     openrouter: LegacyOpenRouterClient | None = None
-    if settings.openrouter_api_key is not None and settings.embedding_endpoint:
+    if settings.openrouter_api_key is not None:
         openrouter = LegacyOpenRouterClient(
             safe_http,
             api_key=settings.openrouter_api_key.get_secret_value(),
-            embedding_endpoint=settings.embedding_endpoint,
             chat_endpoint=settings.openrouter_chat_endpoint,
-            embedding_model=settings.embedding_model,
-            embedding_dimension=settings.embedding_dimension,
         )
 
     retrieval_adapters: dict[str, Any] = {}
@@ -202,7 +210,7 @@ def compose_production_host(
         retrieval_adapters["decode"] = LegacyVectorDecodeRetrievalAdapter(
             index_root=settings.decode_index_root,
             raw_root=settings.decode_raw_root,
-            openrouter=openrouter,
+            embedding_client=embedding_client,
             min_score=settings.decode_min_score,
         )
     elif decode_path is not None:

+ 39 - 6
script_build_host/tests/test_legacy_active_inputs.py

@@ -12,6 +12,7 @@ from script_build_host.adapters.legacy_retrieval import (
     LegacyLlmKnowledgeRetrievalAdapter,
     LegacyOpenRouterClient,
     LegacyVectorDecodeRetrievalAdapter,
+    OpenAICompatibleEmbeddingClient,
 )
 from script_build_host.adapters.retrieval import SafeHttpClient
 from script_build_host.infrastructure.outbound import OutboundPolicy
@@ -105,16 +106,15 @@ async def test_legacy_decode_supports_account_only_and_semantic_vector_search(tm
             client,
             OutboundPolicy(frozenset({"openrouter.example"}), resolver=_public_resolver),
         )
-        openrouter = LegacyOpenRouterClient(
+        embedding_client = OpenAICompatibleEmbeddingClient(
             safe,
             api_key="secret",
             embedding_endpoint="https://openrouter.example/embeddings",
-            chat_endpoint="https://openrouter.example/chat",
             embedding_model="embed",
             embedding_dimension=2,
         )
         adapter = LegacyVectorDecodeRetrievalAdapter(
-            index_root=index, raw_root=raw, openrouter=openrouter
+            index_root=index, raw_root=raw, embedding_client=embedding_client
         )
         account_result = await adapter.retrieve(
             query={
@@ -142,6 +142,42 @@ async def test_legacy_decode_supports_account_only_and_semantic_vector_search(tm
     assert "主题正文" in semantic_result.summary
 
 
+@pytest.mark.asyncio
+async def test_legacy_decode_rejects_query_model_that_differs_from_index(tmp_path) -> None:
+    index, raw = _decode_fixture(tmp_path)
+
+    async with httpx.AsyncClient(
+        transport=httpx.MockTransport(lambda _: httpx.Response(500))
+    ) as client:
+        safe = SafeHttpClient(
+            client,
+            OutboundPolicy(frozenset({"embedding.example"}), resolver=_public_resolver),
+        )
+        embedding_client = OpenAICompatibleEmbeddingClient(
+            safe,
+            api_key="secret",
+            embedding_endpoint="https://embedding.example/embeddings",
+            embedding_model="different-model",
+            embedding_dimension=2,
+        )
+        adapter = LegacyVectorDecodeRetrievalAdapter(
+            index_root=index,
+            raw_root=raw,
+            embedding_client=embedding_client,
+        )
+        with pytest.raises(Exception, match="model does not match"):
+            await adapter.retrieve(
+                query={
+                    "return_field": "主题",
+                    "keyword": "测试",
+                    "account_name": None,
+                    "match_fields": {},
+                    "top_k": 3,
+                },
+                snapshot=SimpleNamespace(),
+            )
+
+
 @pytest.mark.asyncio
 async def test_legacy_knowledge_uses_full_llm_selection_contract(tmp_path) -> None:
     source = tmp_path / "knowledge.json"
@@ -182,10 +218,7 @@ async def test_legacy_knowledge_uses_full_llm_selection_contract(tmp_path) -> No
         openrouter = LegacyOpenRouterClient(
             safe,
             api_key="secret",
-            embedding_endpoint="https://openrouter.example/embeddings",
             chat_endpoint="https://openrouter.example/chat",
-            embedding_model="embed",
-            embedding_dimension=2,
         )
         result = await LegacyLlmKnowledgeRetrievalAdapter(
             source, openrouter=openrouter, model="knowledge-model"