|
|
@@ -16,13 +16,11 @@ import json
|
|
|
import random
|
|
|
from pathlib import Path
|
|
|
|
|
|
-import httpx
|
|
|
-
|
|
|
+from acquisition.query_filter import filter_queries # 共享筛选器(query_filter.txt)
|
|
|
from core.config import Settings
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
|
TREES = ROOT / "scope_trees" / "trees_index.json"
|
|
|
-FILTER_PROMPT = ROOT / "acquisition" / "query_filter.txt" # 筛选词在 acquisition/
|
|
|
OUT = ROOT / "data" / "queries" / "creation_demo.json"
|
|
|
PER = 20
|
|
|
BATCH_N = 16 # 全 demo 统一抽这么多个「实质 / 形式」，各族共用同一批，方便切页签比较
|
|
|
@@ -81,28 +79,6 @@ def _nonleaf(idx, source_type, depths=(3, 4), under=None):
|
|
|
return out
|
|
|
|
|
|
|
|
|
-def _filter(queries, settings):
|
|
|
- """把一批原串喂 query_filter.txt(LLM 只做 keep/排除)。返回 [{keep,reason}] 对齐顺序。"""
|
|
|
- user = json.dumps([{"idx": i, "query": q} for i, q in enumerate(queries)], ensure_ascii=False)
|
|
|
- api = settings.openrouter_base_url.rstrip("/") + "/chat/completions"
|
|
|
- headers = {"Authorization": f"Bearer {settings.openrouter_api_key}", "Content-Type": "application/json"}
|
|
|
- body = {"model": settings.llm_model, "messages": [
|
|
|
- {"role": "system", "content": FILTER_PROMPT.read_text("utf-8")},
|
|
|
- {"role": "user", "content": user}], "response_format": {"type": "json_object"}}
|
|
|
- try:
|
|
|
- resp = httpx.post(api, headers=headers, json=body, timeout=120)
|
|
|
- resp.raise_for_status()
|
|
|
- txt = resp.json()["choices"][0]["message"]["content"]
|
|
|
- # query_filter 要求输出数组；有的模型会包一层 {"result":[...]}，都兜住
|
|
|
- data = json.loads(txt)
|
|
|
- arr = data if isinstance(data, list) else next((v for v in data.values() if isinstance(v, list)), [])
|
|
|
- by = {d.get("idx"): d for d in arr if isinstance(d, dict)}
|
|
|
- return [{"keep": bool(by.get(i, {}).get("keep", True)),
|
|
|
- "reason": str(by.get(i, {}).get("reason", ""))[:50]} for i in range(len(queries))]
|
|
|
- except Exception as exc:
|
|
|
- return [{"keep": True, "reason": f"筛选失败:{str(exc)[:30]}"} for _ in queries]
|
|
|
-
|
|
|
-
|
|
|
def main():
|
|
|
settings = Settings.from_env()
|
|
|
rng = random.Random(7)
|
|
|
@@ -155,7 +131,7 @@ def main():
|
|
|
continue
|
|
|
seen.add(q)
|
|
|
items.append({"query": q, "parts": parts})
|
|
|
- verdicts = _filter([it["query"] for it in items], settings)
|
|
|
+ verdicts = filter_queries([it["query"] for it in items], settings)
|
|
|
for it, v in zip(items, verdicts):
|
|
|
it.update(v)
|
|
|
kept = sum(1 for it in items if it["keep"])
|
SamLee