server.py 59 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665
  1. """
  2. KnowHub Server
  3. Agent 工具使用经验的共享平台。
  4. FastAPI + Milvus Lite(知识)+ SQLite(资源),单文件部署。
  5. """
  6. import os
  7. import re
  8. import json
  9. import sqlite3
  10. import asyncio
  11. import base64
  12. import time
  13. import uuid
  14. from contextlib import asynccontextmanager
  15. from datetime import datetime, timezone
  16. from typing import Optional
  17. from pathlib import Path
  18. from cryptography.hazmat.primitives.ciphers.aead import AESGCM
  19. from fastapi import FastAPI, HTTPException, Query, Header
  20. from fastapi.responses import HTMLResponse
  21. from pydantic import BaseModel, Field
  22. # 导入 LLM 调用(需要 agent 模块在 Python path 中)
  23. import sys
  24. sys.path.insert(0, str(Path(__file__).parent.parent))
  25. # 加载环境变量
  26. from dotenv import load_dotenv
  27. load_dotenv(Path(__file__).parent.parent / ".env")
  28. from agent.llm.openrouter import openrouter_llm_call
  29. # 导入向量存储和 embedding
  30. from knowhub.vector_store import MilvusStore
  31. from knowhub.embeddings import get_embedding, get_embeddings_batch
  32. BRAND_NAME = os.getenv("BRAND_NAME", "KnowHub")
  33. BRAND_API_ENV = os.getenv("BRAND_API_ENV", "KNOWHUB_API")
  34. BRAND_DB = os.getenv("BRAND_DB", "knowhub.db")
  35. # 组织密钥配置(格式:org1:key1_base64,org2:key2_base64)
  36. ORG_KEYS_RAW = os.getenv("ORG_KEYS", "")
  37. ORG_KEYS = {}
  38. if ORG_KEYS_RAW:
  39. for pair in ORG_KEYS_RAW.split(","):
  40. if ":" in pair:
  41. org, key_b64 = pair.split(":", 1)
  42. ORG_KEYS[org.strip()] = key_b64.strip()
  43. DB_PATH = Path(__file__).parent / BRAND_DB
  44. MILVUS_DATA_DIR = Path(__file__).parent / "milvus_data"
  45. # 全局 Milvus 存储实例
  46. milvus_store: Optional[MilvusStore] = None
  47. # --- 数据库 ---
  48. def get_db() -> sqlite3.Connection:
  49. conn = sqlite3.connect(str(DB_PATH))
  50. conn.row_factory = sqlite3.Row
  51. conn.execute("PRAGMA journal_mode=WAL")
  52. return conn
  53. # --- 加密/解密 ---
  54. def get_org_key(resource_id: str) -> Optional[bytes]:
  55. """从content_id提取组织前缀,返回对应密钥"""
  56. if "/" in resource_id:
  57. org = resource_id.split("/")[0]
  58. if org in ORG_KEYS:
  59. return base64.b64decode(ORG_KEYS[org])
  60. return None
  61. def encrypt_content(resource_id: str, plaintext: str) -> str:
  62. """加密内容,返回格式:encrypted:AES256-GCM:{base64_data}"""
  63. if not plaintext:
  64. return ""
  65. key = get_org_key(resource_id)
  66. if not key:
  67. # 没有配置密钥,明文存储(不推荐)
  68. return plaintext
  69. aesgcm = AESGCM(key)
  70. nonce = os.urandom(12) # 96-bit nonce
  71. ciphertext = aesgcm.encrypt(nonce, plaintext.encode("utf-8"), None)
  72. # 组合 nonce + ciphertext
  73. encrypted_data = nonce + ciphertext
  74. encoded = base64.b64encode(encrypted_data).decode("ascii")
  75. return f"encrypted:AES256-GCM:{encoded}"
  76. def decrypt_content(resource_id: str, encrypted_text: str, provided_key: Optional[str] = None) -> str:
  77. """解密内容,如果没有提供密钥或密钥错误,返回[ENCRYPTED]"""
  78. if not encrypted_text:
  79. return ""
  80. if not encrypted_text.startswith("encrypted:AES256-GCM:"):
  81. # 未加密的内容,直接返回
  82. return encrypted_text
  83. # 提取加密数据
  84. encoded = encrypted_text.split(":", 2)[2]
  85. encrypted_data = base64.b64decode(encoded)
  86. nonce = encrypted_data[:12]
  87. ciphertext = encrypted_data[12:]
  88. # 获取密钥
  89. key = None
  90. if provided_key:
  91. # 使用提供的密钥
  92. try:
  93. key = base64.b64decode(provided_key)
  94. except Exception:
  95. return "[ENCRYPTED]"
  96. else:
  97. # 从配置中获取
  98. key = get_org_key(resource_id)
  99. if not key:
  100. return "[ENCRYPTED]"
  101. try:
  102. aesgcm = AESGCM(key)
  103. plaintext = aesgcm.decrypt(nonce, ciphertext, None)
  104. return plaintext.decode("utf-8")
  105. except Exception:
  106. return "[ENCRYPTED]"
  107. def serialize_milvus_result(data):
  108. """将 Milvus 返回的数据转换为可序列化的字典"""
  109. if data is None:
  110. return None
  111. elif isinstance(data, (str, int, float, bool)):
  112. return data
  113. elif isinstance(data, dict):
  114. return {k: serialize_milvus_result(v) for k, v in data.items()}
  115. elif isinstance(data, (list, tuple)):
  116. return [serialize_milvus_result(item) for item in data]
  117. elif hasattr(data, '__iter__') and not isinstance(data, (str, bytes)):
  118. # 处理 RepeatedScalarContainer 等可迭代对象
  119. # 先转换为列表,确保返回的是真正的列表而不是字符串表示
  120. try:
  121. return [serialize_milvus_result(item) for item in list(data)]
  122. except:
  123. # 如果迭代失败,尝试转换为字符串
  124. return str(data)
  125. elif hasattr(data, 'to_dict'):
  126. return serialize_milvus_result(data.to_dict())
  127. elif hasattr(data, '__dict__'):
  128. return serialize_milvus_result(vars(data))
  129. else:
  130. # 最后的fallback
  131. return data
  132. def init_db():
  133. """初始化 SQLite(仅用于 resources)"""
  134. conn = get_db()
  135. conn.execute("""
  136. CREATE TABLE IF NOT EXISTS experiences (
  137. id INTEGER PRIMARY KEY AUTOINCREMENT,
  138. name TEXT NOT NULL,
  139. url TEXT DEFAULT '',
  140. category TEXT DEFAULT '',
  141. task TEXT NOT NULL,
  142. score INTEGER CHECK(score BETWEEN 1 AND 5),
  143. outcome TEXT DEFAULT '',
  144. tips TEXT DEFAULT '',
  145. content_id TEXT DEFAULT '',
  146. submitted_by TEXT DEFAULT '',
  147. created_at TEXT NOT NULL
  148. )
  149. """)
  150. conn.execute("CREATE INDEX IF NOT EXISTS idx_name ON experiences(name)")
  151. conn.execute("""
  152. CREATE TABLE IF NOT EXISTS resources (
  153. id TEXT PRIMARY KEY,
  154. title TEXT DEFAULT '',
  155. body TEXT NOT NULL,
  156. secure_body TEXT DEFAULT '',
  157. content_type TEXT DEFAULT 'text',
  158. metadata TEXT DEFAULT '{}',
  159. sort_order INTEGER DEFAULT 0,
  160. submitted_by TEXT DEFAULT '',
  161. created_at TEXT NOT NULL,
  162. updated_at TEXT DEFAULT ''
  163. )
  164. """)
  165. conn.commit()
  166. conn.close()
  167. # --- Models ---
  168. class ResourceIn(BaseModel):
  169. id: str
  170. title: str = ""
  171. body: str
  172. secure_body: str = ""
  173. content_type: str = "text" # text|code|credential|cookie
  174. metadata: dict = {}
  175. sort_order: int = 0
  176. submitted_by: str = ""
  177. class ResourcePatchIn(BaseModel):
  178. """PATCH /api/resource/{id} 请求体"""
  179. title: Optional[str] = None
  180. body: Optional[str] = None
  181. secure_body: Optional[str] = None
  182. content_type: Optional[str] = None
  183. metadata: Optional[dict] = None
  184. # Knowledge Models
  185. class KnowledgeIn(BaseModel):
  186. task: str
  187. content: str
  188. types: list[str] = ["strategy"]
  189. tags: dict = {}
  190. scopes: list[str] = ["org:cybertogether"]
  191. owner: str = ""
  192. message_id: str = ""
  193. resource_ids: list[str] = []
  194. source: dict = {} # {name, category, urls, agent_id, submitted_by, timestamp}
  195. eval: dict = {} # {score, helpful, harmful, confidence}
  196. class KnowledgeOut(BaseModel):
  197. id: str
  198. message_id: str
  199. types: list[str]
  200. task: str
  201. tags: dict
  202. scopes: list[str]
  203. owner: str
  204. content: str
  205. resource_ids: list[str]
  206. source: dict
  207. eval: dict
  208. created_at: str
  209. updated_at: str
  210. class KnowledgeUpdateIn(BaseModel):
  211. add_helpful_case: Optional[dict] = None
  212. add_harmful_case: Optional[dict] = None
  213. update_score: Optional[int] = Field(default=None, ge=1, le=5)
  214. evolve_feedback: Optional[str] = None
  215. class KnowledgePatchIn(BaseModel):
  216. """PATCH /api/knowledge/{id} 请求体(直接字段编辑)"""
  217. task: Optional[str] = None
  218. content: Optional[str] = None
  219. types: Optional[list[str]] = None
  220. tags: Optional[dict] = None
  221. scopes: Optional[list[str]] = None
  222. owner: Optional[str] = None
  223. class MessageExtractIn(BaseModel):
  224. """POST /api/extract 请求体(消息历史提取)"""
  225. messages: list[dict] # [{role: str, content: str}, ...]
  226. agent_id: str = "unknown"
  227. submitted_by: str # 必填,作为 owner
  228. session_key: str = ""
  229. class KnowledgeBatchUpdateIn(BaseModel):
  230. feedback_list: list[dict]
  231. class KnowledgeSearchResponse(BaseModel):
  232. results: list[dict]
  233. count: int
  234. class ResourceNode(BaseModel):
  235. id: str
  236. title: str
  237. class ResourceOut(BaseModel):
  238. id: str
  239. title: str
  240. body: str
  241. secure_body: str = ""
  242. content_type: str = "text"
  243. metadata: dict = {}
  244. toc: Optional[ResourceNode] = None
  245. children: list[ResourceNode]
  246. prev: Optional[ResourceNode] = None
  247. next: Optional[ResourceNode] = None
  248. # --- App ---
  249. @asynccontextmanager
  250. async def lifespan(app: FastAPI):
  251. global milvus_store
  252. # 初始化 SQLite(resources)
  253. init_db()
  254. # 初始化 Milvus Lite(knowledge)
  255. milvus_store = MilvusStore(data_dir=str(MILVUS_DATA_DIR))
  256. yield
  257. # 清理(Milvus Lite 会自动处理)
  258. app = FastAPI(title=BRAND_NAME, lifespan=lifespan)
  259. # --- Knowledge API ---
  260. @app.post("/api/resource", status_code=201)
  261. def submit_resource(resource: ResourceIn):
  262. conn = get_db()
  263. try:
  264. now = datetime.now(timezone.utc).isoformat()
  265. # 加密敏感内容
  266. encrypted_secure_body = encrypt_content(resource.id, resource.secure_body)
  267. conn.execute(
  268. "INSERT OR REPLACE INTO resources"
  269. "(id, title, body, secure_body, content_type, metadata, sort_order, submitted_by, created_at, updated_at)"
  270. " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
  271. (
  272. resource.id,
  273. resource.title,
  274. resource.body,
  275. encrypted_secure_body,
  276. resource.content_type,
  277. json.dumps(resource.metadata),
  278. resource.sort_order,
  279. resource.submitted_by,
  280. now,
  281. now,
  282. ),
  283. )
  284. conn.commit()
  285. return {"status": "ok", "id": resource.id}
  286. finally:
  287. conn.close()
  288. @app.get("/api/resource/{resource_id:path}", response_model=ResourceOut)
  289. def get_resource(resource_id: str, x_org_key: Optional[str] = Header(None)):
  290. conn = get_db()
  291. try:
  292. row = conn.execute(
  293. "SELECT id, title, body, secure_body, content_type, metadata, sort_order FROM resources WHERE id = ?",
  294. (resource_id,),
  295. ).fetchone()
  296. if not row:
  297. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  298. # 解密敏感内容
  299. secure_body = decrypt_content(resource_id, row["secure_body"] or "", x_org_key)
  300. # 解析metadata
  301. metadata = json.loads(row["metadata"] or "{}")
  302. # 计算导航上下文
  303. root_id = resource_id.split("/")[0] if "/" in resource_id else resource_id
  304. # TOC (根节点)
  305. toc = None
  306. if "/" in resource_id:
  307. toc_row = conn.execute(
  308. "SELECT id, title FROM resources WHERE id = ?",
  309. (root_id,),
  310. ).fetchone()
  311. if toc_row:
  312. toc = ResourceNode(id=toc_row["id"], title=toc_row["title"])
  313. # Children (子节点)
  314. children = []
  315. children_rows = conn.execute(
  316. "SELECT id, title FROM resources WHERE id LIKE ? AND id != ? ORDER BY sort_order",
  317. (f"{resource_id}/%", resource_id),
  318. ).fetchall()
  319. children = [ResourceNode(id=r["id"], title=r["title"]) for r in children_rows]
  320. # Prev/Next (同级节点)
  321. prev_node = None
  322. next_node = None
  323. if "/" in resource_id:
  324. siblings = conn.execute(
  325. "SELECT id, title, sort_order FROM resources WHERE id LIKE ? AND id NOT LIKE ? ORDER BY sort_order",
  326. (f"{root_id}/%", f"{root_id}/%/%"),
  327. ).fetchall()
  328. for i, sib in enumerate(siblings):
  329. if sib["id"] == resource_id:
  330. if i > 0:
  331. prev_node = ResourceNode(id=siblings[i-1]["id"], title=siblings[i-1]["title"])
  332. if i < len(siblings) - 1:
  333. next_node = ResourceNode(id=siblings[i+1]["id"], title=siblings[i+1]["title"])
  334. break
  335. return ResourceOut(
  336. id=row["id"],
  337. title=row["title"],
  338. body=row["body"],
  339. secure_body=secure_body,
  340. content_type=row["content_type"],
  341. metadata=metadata,
  342. toc=toc,
  343. children=children,
  344. prev=prev_node,
  345. next=next_node,
  346. )
  347. finally:
  348. conn.close()
  349. @app.patch("/api/resource/{resource_id:path}")
  350. def patch_resource(resource_id: str, patch: ResourcePatchIn):
  351. """更新resource字段"""
  352. conn = get_db()
  353. try:
  354. # 检查是否存在
  355. row = conn.execute("SELECT id FROM resources WHERE id = ?", (resource_id,)).fetchone()
  356. if not row:
  357. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  358. # 构建更新语句
  359. updates = []
  360. params = []
  361. if patch.title is not None:
  362. updates.append("title = ?")
  363. params.append(patch.title)
  364. if patch.body is not None:
  365. updates.append("body = ?")
  366. params.append(patch.body)
  367. if patch.secure_body is not None:
  368. encrypted = encrypt_content(resource_id, patch.secure_body)
  369. updates.append("secure_body = ?")
  370. params.append(encrypted)
  371. if patch.content_type is not None:
  372. updates.append("content_type = ?")
  373. params.append(patch.content_type)
  374. if patch.metadata is not None:
  375. updates.append("metadata = ?")
  376. params.append(json.dumps(patch.metadata))
  377. if not updates:
  378. return {"status": "ok", "message": "No fields to update"}
  379. # 添加updated_at
  380. updates.append("updated_at = ?")
  381. params.append(datetime.now(timezone.utc).isoformat())
  382. # 执行更新
  383. params.append(resource_id)
  384. sql = f"UPDATE resources SET {', '.join(updates)} WHERE id = ?"
  385. conn.execute(sql, params)
  386. conn.commit()
  387. return {"status": "ok", "id": resource_id}
  388. finally:
  389. conn.close()
  390. @app.get("/api/resource")
  391. def list_resources(
  392. content_type: Optional[str] = Query(None),
  393. limit: int = Query(100, ge=1, le=1000)
  394. ):
  395. """列出所有resource"""
  396. conn = get_db()
  397. try:
  398. sql = "SELECT id, title, content_type, metadata, created_at FROM resources"
  399. params = []
  400. if content_type:
  401. sql += " WHERE content_type = ?"
  402. params.append(content_type)
  403. sql += " ORDER BY id LIMIT ?"
  404. params.append(limit)
  405. rows = conn.execute(sql, params).fetchall()
  406. results = []
  407. for row in rows:
  408. results.append({
  409. "id": row["id"],
  410. "title": row["title"],
  411. "content_type": row["content_type"],
  412. "metadata": json.loads(row["metadata"] or "{}"),
  413. "created_at": row["created_at"],
  414. })
  415. return {"results": results, "count": len(results)}
  416. finally:
  417. conn.close()
  418. # --- Knowledge API ---
  419. # ===== Knowledge API =====
  420. async def _llm_rerank(query: str, candidates: list[dict], top_k: int) -> list[str]:
  421. """
  422. 使用 LLM 对候选知识进行精排
  423. Args:
  424. query: 查询文本
  425. candidates: 候选知识列表
  426. top_k: 返回数量
  427. Returns:
  428. 排序后的知识 ID 列表
  429. """
  430. if not candidates:
  431. return []
  432. # 构造 prompt
  433. candidates_text = "\n".join([
  434. f"[{i+1}] ID: {c['id']}\nTask: {c['task']}\nContent: {c['content'][:200]}..."
  435. for i, c in enumerate(candidates)
  436. ])
  437. prompt = f"""你是知识检索专家。根据用户查询,从候选知识中选出最相关的 {top_k} 条。
  438. 用户查询:"{query}"
  439. 候选知识:
  440. {candidates_text}
  441. 请输出最相关的 {top_k} 个知识 ID,按相关性从高到低排序,用逗号分隔。
  442. 只输出 ID,不要其他内容。"""
  443. try:
  444. response = await openrouter_llm_call(
  445. messages=[{"role": "user", "content": prompt}],
  446. model="google/gemini-2.5-flash-lite"
  447. )
  448. content = response.get("content", "").strip()
  449. # 解析 ID 列表
  450. selected_ids = [
  451. idx.strip()
  452. for idx in re.split(r'[,\s]+', content)
  453. if idx.strip().startswith(("knowledge-", "research-"))
  454. ]
  455. return selected_ids[:top_k]
  456. except Exception as e:
  457. print(f"[LLM Rerank] 失败: {e}")
  458. return []
  459. @app.get("/api/knowledge/search")
  460. async def search_knowledge_api(
  461. q: str = Query(..., description="查询文本"),
  462. top_k: int = Query(default=5, ge=1, le=20),
  463. min_score: int = Query(default=3, ge=1, le=5),
  464. types: Optional[str] = None,
  465. owner: Optional[str] = None
  466. ):
  467. """检索知识(向量召回 + LLM 精排)"""
  468. try:
  469. # 1. 生成查询向量
  470. query_embedding = await get_embedding(q)
  471. # 2. 构建过滤表达式
  472. filters = []
  473. if types:
  474. type_list = [t.strip() for t in types.split(',') if t.strip()]
  475. for t in type_list:
  476. filters.append(f'array_contains(types, "{t}")')
  477. if owner:
  478. filters.append(f'owner == "{owner}"')
  479. # 添加 min_score 过滤
  480. filters.append(f'eval["score"] >= {min_score}')
  481. filter_expr = ' and '.join(filters) if filters else None
  482. # 3. 向量召回(3*k 个候选)
  483. recall_limit = top_k * 3
  484. candidates = milvus_store.search(
  485. query_embedding=query_embedding,
  486. filters=filter_expr,
  487. limit=recall_limit
  488. )
  489. if not candidates:
  490. return {"results": [], "count": 0, "reranked": False}
  491. # 转换为可序列化的格式
  492. serialized_candidates = [serialize_milvus_result(c) for c in candidates]
  493. # 4. LLM 精排
  494. reranked_ids = await _llm_rerank(q, serialized_candidates, top_k)
  495. if reranked_ids:
  496. # 按 LLM 排序返回
  497. id_to_candidate = {c["id"]: c for c in serialized_candidates}
  498. results = [id_to_candidate[id] for id in reranked_ids if id in id_to_candidate]
  499. return {"results": results, "count": len(results), "reranked": True}
  500. else:
  501. # Fallback:直接返回向量召回的 top k
  502. print(f"[Knowledge Search] LLM 精排失败,fallback 到向量 top-{top_k}")
  503. return {"results": serialized_candidates[:top_k], "count": len(serialized_candidates[:top_k]), "reranked": False}
  504. except Exception as e:
  505. print(f"[Knowledge Search] 错误: {e}")
  506. raise HTTPException(status_code=500, detail=str(e))
  507. @app.post("/api/knowledge", status_code=201)
  508. async def save_knowledge(knowledge: KnowledgeIn):
  509. """保存新知识"""
  510. try:
  511. # 生成 ID
  512. timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
  513. random_suffix = uuid.uuid4().hex[:4]
  514. knowledge_id = f"knowledge-{timestamp}-{random_suffix}"
  515. now = int(time.time())
  516. # 设置默认值
  517. owner = knowledge.owner or f"agent:{knowledge.source.get('agent_id', 'unknown')}"
  518. # 准备 source
  519. source = {
  520. "name": knowledge.source.get("name", ""),
  521. "category": knowledge.source.get("category", ""),
  522. "urls": knowledge.source.get("urls", []),
  523. "agent_id": knowledge.source.get("agent_id", "unknown"),
  524. "submitted_by": knowledge.source.get("submitted_by", ""),
  525. "timestamp": datetime.now(timezone.utc).isoformat(),
  526. "message_id": knowledge.message_id
  527. }
  528. # 准备 eval
  529. eval_data = {
  530. "score": knowledge.eval.get("score", 3),
  531. "helpful": knowledge.eval.get("helpful", 1),
  532. "harmful": knowledge.eval.get("harmful", 0),
  533. "confidence": knowledge.eval.get("confidence", 0.5),
  534. "helpful_history": [],
  535. "harmful_history": []
  536. }
  537. # 生成向量(只基于 task,因为搜索时用户描述的是任务场景)
  538. embedding = await get_embedding(knowledge.task)
  539. # 准备插入数据
  540. insert_data = {
  541. "id": knowledge_id,
  542. "embedding": embedding,
  543. "message_id": knowledge.message_id,
  544. "task": knowledge.task,
  545. "content": knowledge.content,
  546. "types": knowledge.types,
  547. "tags": knowledge.tags,
  548. "scopes": knowledge.scopes,
  549. "owner": owner,
  550. "resource_ids": knowledge.resource_ids,
  551. "source": source,
  552. "eval": eval_data,
  553. "created_at": now,
  554. "updated_at": now,
  555. }
  556. print(f"[Save Knowledge] 插入数据: {json.dumps({k: v for k, v in insert_data.items() if k != 'embedding'}, ensure_ascii=False)}")
  557. # 插入 Milvus
  558. milvus_store.insert(insert_data)
  559. return {"status": "ok", "knowledge_id": knowledge_id}
  560. except Exception as e:
  561. print(f"[Save Knowledge] 错误: {e}")
  562. raise HTTPException(status_code=500, detail=str(e))
  563. @app.get("/api/knowledge")
  564. def list_knowledge(
  565. limit: int = Query(default=100, ge=1, le=1000),
  566. types: Optional[str] = None,
  567. scopes: Optional[str] = None,
  568. owner: Optional[str] = None,
  569. tags: Optional[str] = None
  570. ):
  571. """列出知识(支持后端筛选)"""
  572. try:
  573. # 构建过滤表达式
  574. filters = []
  575. # types 支持多个,用 AND 连接(交集:必须同时包含所有选中的type)
  576. if types:
  577. type_list = [t.strip() for t in types.split(',') if t.strip()]
  578. for t in type_list:
  579. filters.append(f'array_contains(types, "{t}")')
  580. if scopes:
  581. filters.append(f'array_contains(scopes, "{scopes}")')
  582. if owner:
  583. filters.append(f'owner like "%{owner}%"')
  584. # tags 支持多个,用 AND 连接(交集:必须同时包含所有选中的tag)
  585. if tags:
  586. tag_list = [t.strip() for t in tags.split(',') if t.strip()]
  587. for t in tag_list:
  588. filters.append(f'json_contains(tags, \'"{t}"\')')
  589. # 如果没有过滤条件,查询所有
  590. filter_expr = ' and '.join(filters) if filters else 'id != ""'
  591. # 查询 Milvus
  592. results = milvus_store.query(filter_expr, limit=limit)
  593. # 转换为可序列化的格式
  594. serialized_results = [serialize_milvus_result(r) for r in results]
  595. return {"results": serialized_results, "count": len(serialized_results)}
  596. except Exception as e:
  597. print(f"[List Knowledge] 错误: {e}")
  598. raise HTTPException(status_code=500, detail=str(e))
  599. @app.get("/api/knowledge/meta/tags")
  600. def get_all_tags():
  601. """获取所有已有的 tags"""
  602. try:
  603. # 查询所有知识
  604. results = milvus_store.query('id != ""', limit=10000)
  605. all_tags = set()
  606. for item in results:
  607. # 转换为标准字典
  608. serialized_item = serialize_milvus_result(item)
  609. tags_dict = serialized_item.get("tags", {})
  610. if isinstance(tags_dict, dict):
  611. for key in tags_dict.keys():
  612. all_tags.add(key)
  613. return {"tags": sorted(list(all_tags))}
  614. except Exception as e:
  615. print(f"[Get Tags] 错误: {e}")
  616. raise HTTPException(status_code=500, detail=str(e))
  617. @app.get("/api/knowledge/{knowledge_id}")
  618. def get_knowledge(knowledge_id: str):
  619. """获取单条知识"""
  620. try:
  621. result = milvus_store.get_by_id(knowledge_id)
  622. if not result:
  623. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  624. return serialize_milvus_result(result)
  625. except HTTPException:
  626. raise
  627. except Exception as e:
  628. print(f"[Get Knowledge] 错误: {e}")
  629. raise HTTPException(status_code=500, detail=str(e))
  630. async def _evolve_knowledge_with_llm(old_content: str, feedback: str) -> str:
  631. """使用 LLM 进行知识进化重写"""
  632. prompt = f"""你是一个 AI Agent 知识库管理员。请根据反馈建议,对现有的知识内容进行重写进化。
  633. 【原知识内容】:
  634. {old_content}
  635. 【实战反馈建议】:
  636. {feedback}
  637. 【重写要求】:
  638. 1. 融合知识:将反馈中的避坑指南、新参数或修正后的选择逻辑融入原知识,使其更具通用性和准确性。
  639. 2. 保持结构:如果原内容有特定格式(如 Markdown、代码示例等),请保持该格式。
  640. 3. 语言:简洁直接,使用中文。
  641. 4. 禁止:严禁输出任何开场白、解释语或额外的 Markdown 标题,直接返回重写后的正文。
  642. """
  643. try:
  644. response = await openrouter_llm_call(
  645. messages=[{"role": "user", "content": prompt}],
  646. model="google/gemini-2.5-flash-lite"
  647. )
  648. evolved = response.get("content", "").strip()
  649. if len(evolved) < 5:
  650. raise ValueError("LLM output too short")
  651. return evolved
  652. except Exception as e:
  653. print(f"知识进化失败,采用追加模式回退: {e}")
  654. return f"{old_content}\n\n---\n[Update {datetime.now().strftime('%Y-%m-%d')}]: {feedback}"
  655. @app.put("/api/knowledge/{knowledge_id}")
  656. async def update_knowledge(knowledge_id: str, update: KnowledgeUpdateIn):
  657. """更新知识评估,支持知识进化"""
  658. try:
  659. # 获取现有知识
  660. existing = milvus_store.get_by_id(knowledge_id)
  661. if not existing:
  662. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  663. eval_data = existing.get("eval", {})
  664. # 更新评分
  665. if update.update_score is not None:
  666. eval_data["score"] = update.update_score
  667. # 添加有效案例
  668. if update.add_helpful_case:
  669. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  670. if "helpful_history" not in eval_data:
  671. eval_data["helpful_history"] = []
  672. eval_data["helpful_history"].append(update.add_helpful_case)
  673. # 添加有害案例
  674. if update.add_harmful_case:
  675. eval_data["harmful"] = eval_data.get("harmful", 0) + 1
  676. if "harmful_history" not in eval_data:
  677. eval_data["harmful_history"] = []
  678. eval_data["harmful_history"].append(update.add_harmful_case)
  679. # 知识进化
  680. content = existing["content"]
  681. need_reembed = False
  682. if update.evolve_feedback:
  683. content = await _evolve_knowledge_with_llm(content, update.evolve_feedback)
  684. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  685. need_reembed = True
  686. # 准备更新数据
  687. updates = {
  688. "content": content,
  689. "eval": eval_data,
  690. }
  691. # 如果内容变化,重新生成向量
  692. if need_reembed:
  693. embedding = await get_embedding(existing['task'])
  694. updates["embedding"] = embedding
  695. # 更新 Milvus
  696. milvus_store.update(knowledge_id, updates)
  697. return {"status": "ok", "knowledge_id": knowledge_id}
  698. except HTTPException:
  699. raise
  700. except Exception as e:
  701. print(f"[Update Knowledge] 错误: {e}")
  702. raise HTTPException(status_code=500, detail=str(e))
  703. @app.patch("/api/knowledge/{knowledge_id}")
  704. async def patch_knowledge(knowledge_id: str, patch: KnowledgePatchIn):
  705. """直接编辑知识字段"""
  706. try:
  707. # 获取现有知识
  708. existing = milvus_store.get_by_id(knowledge_id)
  709. if not existing:
  710. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  711. updates = {}
  712. need_reembed = False
  713. if patch.task is not None:
  714. updates["task"] = patch.task
  715. need_reembed = True
  716. if patch.content is not None:
  717. updates["content"] = patch.content
  718. # content 变化不需要重新生成 embedding(只基于 task)
  719. if patch.types is not None:
  720. updates["types"] = patch.types
  721. if patch.tags is not None:
  722. updates["tags"] = patch.tags
  723. if patch.scopes is not None:
  724. updates["scopes"] = patch.scopes
  725. if patch.owner is not None:
  726. updates["owner"] = patch.owner
  727. if not updates:
  728. return {"status": "ok", "knowledge_id": knowledge_id}
  729. # 如果 task 变化,重新生成向量
  730. if need_reembed:
  731. task = updates.get("task", existing["task"])
  732. embedding = await get_embedding(task)
  733. updates["embedding"] = embedding
  734. # 更新 Milvus
  735. milvus_store.update(knowledge_id, updates)
  736. return {"status": "ok", "knowledge_id": knowledge_id}
  737. except HTTPException:
  738. raise
  739. except Exception as e:
  740. print(f"[Patch Knowledge] 错误: {e}")
  741. raise HTTPException(status_code=500, detail=str(e))
  742. @app.post("/api/knowledge/batch_update")
  743. async def batch_update_knowledge(batch: KnowledgeBatchUpdateIn):
  744. """批量反馈知识有效性"""
  745. if not batch.feedback_list:
  746. return {"status": "ok", "updated": 0}
  747. try:
  748. # 先处理无需进化的,收集需要进化的
  749. evolution_tasks = [] # [(knowledge_id, old_content, feedback, eval_data)]
  750. simple_updates = [] # [(knowledge_id, is_effective, eval_data)]
  751. for item in batch.feedback_list:
  752. knowledge_id = item.get("knowledge_id")
  753. is_effective = item.get("is_effective")
  754. feedback = item.get("feedback", "")
  755. if not knowledge_id:
  756. continue
  757. existing = milvus_store.get_by_id(knowledge_id)
  758. if not existing:
  759. continue
  760. eval_data = existing.get("eval", {})
  761. if is_effective and feedback:
  762. evolution_tasks.append((knowledge_id, existing["content"], feedback, eval_data, existing["task"]))
  763. else:
  764. simple_updates.append((knowledge_id, is_effective, eval_data))
  765. # 执行简单更新
  766. for knowledge_id, is_effective, eval_data in simple_updates:
  767. if is_effective:
  768. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  769. else:
  770. eval_data["harmful"] = eval_data.get("harmful", 0) + 1
  771. milvus_store.update(knowledge_id, {"eval": eval_data})
  772. # 并发执行知识进化
  773. if evolution_tasks:
  774. print(f"🧬 并发处理 {len(evolution_tasks)} 条知识进化...")
  775. evolved_results = await asyncio.gather(
  776. *[_evolve_knowledge_with_llm(old, fb) for _, old, fb, _, _ in evolution_tasks]
  777. )
  778. for (knowledge_id, _, _, eval_data, task), evolved_content in zip(evolution_tasks, evolved_results):
  779. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  780. # 重新生成向量(只基于 task)
  781. embedding = await get_embedding(task)
  782. milvus_store.update(knowledge_id, {
  783. "content": evolved_content,
  784. "eval": eval_data,
  785. "embedding": embedding
  786. })
  787. return {"status": "ok", "updated": len(simple_updates) + len(evolution_tasks)}
  788. except Exception as e:
  789. print(f"[Batch Update] 错误: {e}")
  790. raise HTTPException(status_code=500, detail=str(e))
  791. @app.post("/api/knowledge/slim")
  792. async def slim_knowledge(model: str = "google/gemini-2.5-flash-lite"):
  793. """知识库瘦身:合并语义相似知识"""
  794. try:
  795. # 获取所有知识
  796. all_knowledge = milvus_store.query('id != ""', limit=10000)
  797. # 转换为可序列化的格式
  798. all_knowledge = [serialize_milvus_result(item) for item in all_knowledge]
  799. if len(all_knowledge) < 2:
  800. return {"status": "ok", "message": f"知识库仅有 {len(all_knowledge)} 条,无需瘦身"}
  801. # 构造发给大模型的内容
  802. entries_text = ""
  803. for item in all_knowledge:
  804. eval_data = item.get("eval", {})
  805. types = item.get("types", [])
  806. entries_text += f"[ID: {item['id']}] [Types: {','.join(types)}] "
  807. entries_text += f"[Helpful: {eval_data.get('helpful', 0)}, Harmful: {eval_data.get('harmful', 0)}] [Score: {eval_data.get('score', 3)}]\n"
  808. entries_text += f"Task: {item['task']}\n"
  809. entries_text += f"Content: {item['content'][:200]}...\n\n"
  810. prompt = f"""你是一个 AI Agent 知识库管理员。以下是当前知识库的全部条目,请执行瘦身操作:
  811. 【任务】:
  812. 1. 识别语义高度相似或重复的知识,将它们合并为一条更精炼、更通用的知识。
  813. 2. 合并时保留 helpful 最高的那条的 ID(helpful 取各条之和)。
  814. 3. 对于独立的、无重复的知识,保持原样不动。
  815. 【当前知识库】:
  816. {entries_text}
  817. 【输出格式要求】:
  818. 严格按以下格式输出每条知识,条目之间用 === 分隔:
  819. ID: <保留的id>
  820. TYPES: <逗号分隔的type列表>
  821. HELPFUL: <合并后的helpful计数>
  822. HARMFUL: <合并后的harmful计数>
  823. SCORE: <评分>
  824. TASK: <任务描述>
  825. CONTENT: <合并后的知识内容>
  826. ===
  827. 最后输出合并报告:
  828. REPORT: 原有 X 条,合并后 Y 条,精简了 Z 条。
  829. 禁止输出任何开场白或解释。"""
  830. print(f"\n[知识瘦身] 正在调用 {model} 分析 {len(all_knowledge)} 条知识...")
  831. response = await openrouter_llm_call(
  832. messages=[{"role": "user", "content": prompt}],
  833. model=model
  834. )
  835. content = response.get("content", "").strip()
  836. if not content:
  837. raise HTTPException(status_code=500, detail="LLM 返回为空")
  838. # 解析大模型输出
  839. report_line = ""
  840. new_entries = []
  841. blocks = [b.strip() for b in content.split("===") if b.strip()]
  842. for block in blocks:
  843. if block.startswith("REPORT:"):
  844. report_line = block
  845. continue
  846. lines = block.split("\n")
  847. kid, types, helpful, harmful, score, task, content_lines = None, [], 0, 0, 3, "", []
  848. current_field = None
  849. for line in lines:
  850. if line.startswith("ID:"):
  851. kid = line[3:].strip()
  852. current_field = None
  853. elif line.startswith("TYPES:"):
  854. types_str = line[6:].strip()
  855. types = [t.strip() for t in types_str.split(",") if t.strip()]
  856. current_field = None
  857. elif line.startswith("HELPFUL:"):
  858. try:
  859. helpful = int(line[8:].strip())
  860. except Exception:
  861. helpful = 0
  862. current_field = None
  863. elif line.startswith("HARMFUL:"):
  864. try:
  865. harmful = int(line[8:].strip())
  866. except Exception:
  867. harmful = 0
  868. current_field = None
  869. elif line.startswith("SCORE:"):
  870. try:
  871. score = int(line[6:].strip())
  872. except Exception:
  873. score = 3
  874. current_field = None
  875. elif line.startswith("TASK:"):
  876. task = line[5:].strip()
  877. current_field = "task"
  878. elif line.startswith("CONTENT:"):
  879. content_lines.append(line[8:].strip())
  880. current_field = "content"
  881. elif current_field == "task":
  882. task += "\n" + line
  883. elif current_field == "content":
  884. content_lines.append(line)
  885. if kid and content_lines:
  886. new_entries.append({
  887. "id": kid,
  888. "types": types if types else ["strategy"],
  889. "helpful": helpful,
  890. "harmful": harmful,
  891. "score": score,
  892. "task": task.strip(),
  893. "content": "\n".join(content_lines).strip()
  894. })
  895. if not new_entries:
  896. raise HTTPException(status_code=500, detail="解析大模型输出失败")
  897. # 生成向量并重建知识库
  898. print(f"[知识瘦身] 正在为 {len(new_entries)} 条知识生成向量...")
  899. # 批量生成向量(只基于 task)
  900. texts = [e['task'] for e in new_entries]
  901. embeddings = await get_embeddings_batch(texts)
  902. # 清空并重建
  903. now = int(time.time())
  904. milvus_store.drop_collection()
  905. milvus_store._init_collection()
  906. knowledge_list = []
  907. for e, embedding in zip(new_entries, embeddings):
  908. eval_data = {
  909. "score": e["score"],
  910. "helpful": e["helpful"],
  911. "harmful": e["harmful"],
  912. "confidence": 0.9,
  913. "helpful_history": [],
  914. "harmful_history": []
  915. }
  916. source = {
  917. "name": "slim",
  918. "category": "exp",
  919. "urls": [],
  920. "agent_id": "slim",
  921. "submitted_by": "system",
  922. "timestamp": datetime.now(timezone.utc).isoformat()
  923. }
  924. knowledge_list.append({
  925. "id": e["id"],
  926. "embedding": embedding,
  927. "message_id": "",
  928. "task": e["task"],
  929. "content": e["content"],
  930. "types": e["types"],
  931. "tags": {},
  932. "scopes": ["org:cybertogether"],
  933. "owner": "agent:slim",
  934. "resource_ids": [],
  935. "source": source,
  936. "eval": eval_data,
  937. "created_at": now,
  938. "updated_at": now
  939. })
  940. milvus_store.insert_batch(knowledge_list)
  941. result_msg = f"瘦身完成:{len(all_knowledge)} → {len(new_entries)} 条知识"
  942. if report_line:
  943. result_msg += f"\n{report_line}"
  944. print(f"[知识瘦身] {result_msg}")
  945. return {"status": "ok", "before": len(all_knowledge), "after": len(new_entries), "report": report_line}
  946. except HTTPException:
  947. raise
  948. except Exception as e:
  949. print(f"[Slim Knowledge] 错误: {e}")
  950. raise HTTPException(status_code=500, detail=str(e))
  951. @app.post("/api/extract")
  952. async def extract_knowledge_from_messages(extract_req: MessageExtractIn):
  953. """从消息历史中提取知识(LLM 分析)"""
  954. if not extract_req.submitted_by:
  955. raise HTTPException(status_code=400, detail="submitted_by is required")
  956. messages = extract_req.messages
  957. if not messages or len(messages) == 0:
  958. return {"status": "ok", "extracted_count": 0, "knowledge_ids": []}
  959. # 构造消息历史文本
  960. messages_text = ""
  961. for msg in messages:
  962. role = msg.get("role", "unknown")
  963. content = msg.get("content", "")
  964. messages_text += f"[{role}]: {content}\n\n"
  965. # LLM 提取知识
  966. prompt = f"""你是一个知识提取专家。请从以下 Agent 对话历史中提取有价值的知识。
  967. 【对话历史】:
  968. {messages_text}
  969. 【提取要求】:
  970. 1. 识别对话中的关键知识点(工具使用经验、问题解决方案、最佳实践、踩坑经验等)
  971. 2. 每条知识必须包含:
  972. - task: 任务场景描述(在什么情况下,要完成什么目标)
  973. - content: 核心知识内容(具体可操作的方法、注意事项)
  974. - types: 知识类型(从 strategy/tool/user_profile/usecase/definition/plan 中选择)
  975. - score: 评分 1-5(根据知识的价值和可操作性)
  976. 3. 只提取有实际价值的知识,不要提取泛泛而谈的内容,一次就成功或比较简单的经验就不要记录了。
  977. 4. 如果没有值得提取的知识,返回空列表
  978. 【输出格式】:
  979. 严格按以下 JSON 格式输出,每条知识之间用逗号分隔:
  980. [
  981. {{
  982. "task": "任务场景描述",
  983. "content": "核心知识内容",
  984. "types": ["strategy"],
  985. "score": 4
  986. }},
  987. {{
  988. "task": "另一个任务场景",
  989. "content": "另一个知识内容",
  990. "types": ["tool"],
  991. "score": 5
  992. }}
  993. ]
  994. 如果没有知识,输出: []
  995. **注意**:只记录经过多次尝试、或经过用户指导才成功的知识,一次就成功或比较简单的经验就不要记录了。
  996. 禁止输出任何解释或额外文本,只输出 JSON 数组。"""
  997. try:
  998. print(f"\n[Extract] 正在从 {len(messages)} 条消息中提取知识...")
  999. response = await openrouter_llm_call(
  1000. messages=[{"role": "user", "content": prompt}],
  1001. model="google/gemini-2.5-flash-lite"
  1002. )
  1003. content = response.get("content", "").strip()
  1004. # 尝试解析 JSON
  1005. # 移除可能的 markdown 代码块标记
  1006. if content.startswith("```json"):
  1007. content = content[7:]
  1008. if content.startswith("```"):
  1009. content = content[3:]
  1010. if content.endswith("```"):
  1011. content = content[:-3]
  1012. content = content.strip()
  1013. extracted_knowledge = json.loads(content)
  1014. if not isinstance(extracted_knowledge, list):
  1015. raise ValueError("LLM output is not a list")
  1016. if not extracted_knowledge:
  1017. return {"status": "ok", "extracted_count": 0, "knowledge_ids": []}
  1018. # 批量生成向量(只基于 task)
  1019. texts = [item.get('task', '') for item in extracted_knowledge]
  1020. embeddings = await get_embeddings_batch(texts)
  1021. # 保存提取的知识
  1022. knowledge_ids = []
  1023. now = int(time.time())
  1024. knowledge_list = []
  1025. for item, embedding in zip(extracted_knowledge, embeddings):
  1026. task = item.get("task", "")
  1027. knowledge_content = item.get("content", "")
  1028. types = item.get("types", ["strategy"])
  1029. score = item.get("score", 3)
  1030. if not task or not knowledge_content:
  1031. continue
  1032. # 生成 ID
  1033. timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
  1034. random_suffix = uuid.uuid4().hex[:4]
  1035. knowledge_id = f"knowledge-{timestamp}-{random_suffix}"
  1036. # 准备数据
  1037. source = {
  1038. "name": "message_extraction",
  1039. "category": "exp",
  1040. "urls": [],
  1041. "agent_id": extract_req.agent_id,
  1042. "submitted_by": extract_req.submitted_by,
  1043. "timestamp": datetime.now(timezone.utc).isoformat(),
  1044. "session_key": extract_req.session_key
  1045. }
  1046. eval_data = {
  1047. "score": score,
  1048. "helpful": 1,
  1049. "harmful": 0,
  1050. "confidence": 0.7,
  1051. "helpful_history": [],
  1052. "harmful_history": []
  1053. }
  1054. knowledge_list.append({
  1055. "id": knowledge_id,
  1056. "embedding": embedding,
  1057. "message_id": "",
  1058. "task": task,
  1059. "content": knowledge_content,
  1060. "types": types,
  1061. "tags": {},
  1062. "scopes": ["org:cybertogether"],
  1063. "owner": extract_req.submitted_by,
  1064. "resource_ids": [],
  1065. "source": source,
  1066. "eval": eval_data,
  1067. "created_at": now,
  1068. "updated_at": now,
  1069. })
  1070. knowledge_ids.append(knowledge_id)
  1071. # 批量插入
  1072. if knowledge_list:
  1073. milvus_store.insert_batch(knowledge_list)
  1074. print(f"[Extract] 成功提取并保存 {len(knowledge_ids)} 条知识")
  1075. return {
  1076. "status": "ok",
  1077. "extracted_count": len(knowledge_ids),
  1078. "knowledge_ids": knowledge_ids
  1079. }
  1080. except json.JSONDecodeError as e:
  1081. print(f"[Extract] JSON 解析失败: {e}")
  1082. print(f"[Extract] LLM 输出: {content[:500]}")
  1083. return {"status": "error", "error": "Failed to parse LLM output", "extracted_count": 0}
  1084. except Exception as e:
  1085. print(f"[Extract] 提取失败: {e}")
  1086. return {"status": "error", "error": str(e), "extracted_count": 0}
  1087. @app.get("/", response_class=HTMLResponse)
  1088. def frontend():
  1089. """KnowHub 管理前端"""
  1090. return """<!DOCTYPE html>
  1091. <html lang="zh-CN">
  1092. <head>
  1093. <meta charset="UTF-8">
  1094. <meta name="viewport" content="width=device-width, initial-scale=1.0">
  1095. <title>KnowHub 管理</title>
  1096. <script src="https://cdn.tailwindcss.com"></script>
  1097. </head>
  1098. <body class="bg-gray-50">
  1099. <div class="container mx-auto px-4 py-8 max-w-7xl">
  1100. <div class="flex justify-between items-center mb-8">
  1101. <h1 class="text-3xl font-bold text-gray-800">KnowHub 全局知识库</h1>
  1102. <button onclick="openAddModal()" class="bg-blue-600 hover:bg-blue-700 text-white px-6 py-2 rounded-lg">
  1103. + 新增知识
  1104. </button>
  1105. </div>
  1106. <!-- 筛选栏 -->
  1107. <div class="bg-white rounded-lg shadow p-6 mb-6">
  1108. <div class="grid grid-cols-1 md:grid-cols-4 gap-4">
  1109. <div>
  1110. <label class="block text-sm font-medium text-gray-700 mb-2">类型 (Types)</label>
  1111. <div class="space-y-2">
  1112. <label class="flex items-center"><input type="checkbox" value="strategy" class="mr-2 type-filter"> Strategy</label>
  1113. <label class="flex items-center"><input type="checkbox" value="tool" class="mr-2 type-filter"> Tool</label>
  1114. <label class="flex items-center"><input type="checkbox" value="user_profile" class="mr-2 type-filter"> User Profile</label>
  1115. <label class="flex items-center"><input type="checkbox" value="usecase" class="mr-2 type-filter"> Usecase</label>
  1116. <label class="flex items-center"><input type="checkbox" value="definition" class="mr-2 type-filter"> Definition</label>
  1117. <label class="flex items-center"><input type="checkbox" value="plan" class="mr-2 type-filter"> Plan</label>
  1118. </div>
  1119. </div>
  1120. <div>
  1121. <label class="block text-sm font-medium text-gray-700 mb-2">Tags</label>
  1122. <div id="tagsFilterContainer" class="space-y-2 max-h-40 overflow-y-auto">
  1123. <p class="text-sm text-gray-500">加载中...</p>
  1124. </div>
  1125. </div>
  1126. <div>
  1127. <label class="block text-sm font-medium text-gray-700 mb-2">Owner</label>
  1128. <input type="text" id="ownerFilter" placeholder="输入 owner" class="w-full border rounded px-3 py-2">
  1129. </div>
  1130. <div>
  1131. <label class="block text-sm font-medium text-gray-700 mb-2">Scopes</label>
  1132. <input type="text" id="scopesFilter" placeholder="输入 scope" class="w-full border rounded px-3 py-2">
  1133. </div>
  1134. </div>
  1135. <button onclick="applyFilters()" class="mt-4 bg-gray-600 hover:bg-gray-700 text-white px-4 py-2 rounded">
  1136. 应用筛选
  1137. </button>
  1138. </div>
  1139. <!-- 知识列表 -->
  1140. <div id="knowledgeList" class="space-y-4"></div>
  1141. </div>
  1142. <!-- 新增/编辑 Modal -->
  1143. <div id="modal" class="hidden fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center p-4">
  1144. <div class="bg-white rounded-lg max-w-2xl w-full max-h-[90vh] overflow-y-auto p-6">
  1145. <h2 id="modalTitle" class="text-2xl font-bold mb-4">新增知识</h2>
  1146. <form id="knowledgeForm" class="space-y-4">
  1147. <input type="hidden" id="editId">
  1148. <div>
  1149. <label class="block text-sm font-medium mb-1">Task *</label>
  1150. <input type="text" id="taskInput" required class="w-full border rounded px-3 py-2">
  1151. </div>
  1152. <div>
  1153. <label class="block text-sm font-medium mb-1">Content *</label>
  1154. <textarea id="contentInput" required rows="6" class="w-full border rounded px-3 py-2"></textarea>
  1155. </div>
  1156. <div>
  1157. <label class="block text-sm font-medium mb-1">Types (多选)</label>
  1158. <div class="space-y-1">
  1159. <label class="flex items-center"><input type="checkbox" value="strategy" class="mr-2 type-checkbox"> Strategy</label>
  1160. <label class="flex items-center"><input type="checkbox" value="tool" class="mr-2 type-checkbox"> Tool</label>
  1161. <label class="flex items-center"><input type="checkbox" value="user_profile" class="mr-2 type-checkbox"> User Profile</label>
  1162. <label class="flex items-center"><input type="checkbox" value="usecase" class="mr-2 type-checkbox"> Usecase</label>
  1163. <label class="flex items-center"><input type="checkbox" value="definition" class="mr-2 type-checkbox"> Definition</label>
  1164. <label class="flex items-center"><input type="checkbox" value="plan" class="mr-2 type-checkbox"> Plan</label>
  1165. </div>
  1166. </div>
  1167. <div>
  1168. <label class="block text-sm font-medium mb-1">Tags (JSON)</label>
  1169. <textarea id="tagsInput" rows="2" placeholder='{"key": "value"}' class="w-full border rounded px-3 py-2"></textarea>
  1170. </div>
  1171. <div>
  1172. <label class="block text-sm font-medium mb-1">Scopes (逗号分隔)</label>
  1173. <input type="text" id="scopesInput" placeholder="org:cybertogether" class="w-full border rounded px-3 py-2">
  1174. </div>
  1175. <div>
  1176. <label class="block text-sm font-medium mb-1">Owner</label>
  1177. <input type="text" id="ownerInput" class="w-full border rounded px-3 py-2">
  1178. </div>
  1179. <div class="flex gap-2 pt-4">
  1180. <button type="submit" class="bg-blue-600 hover:bg-blue-700 text-white px-6 py-2 rounded">保存</button>
  1181. <button type="button" onclick="closeModal()" class="bg-gray-300 hover:bg-gray-400 px-6 py-2 rounded">取消</button>
  1182. </div>
  1183. </form>
  1184. </div>
  1185. </div>
  1186. <script>
  1187. let allKnowledge = [];
  1188. let availableTags = [];
  1189. async function loadTags() {
  1190. const res = await fetch('/api/knowledge/meta/tags');
  1191. const data = await res.json();
  1192. availableTags = data.tags;
  1193. renderTagsFilter();
  1194. }
  1195. function renderTagsFilter() {
  1196. const container = document.getElementById('tagsFilterContainer');
  1197. if (availableTags.length === 0) {
  1198. container.innerHTML = '<p class="text-sm text-gray-500">暂无 tags</p>';
  1199. return;
  1200. }
  1201. container.innerHTML = availableTags.map(tag =>
  1202. `<label class="flex items-center"><input type="checkbox" value="${escapeHtml(tag)}" class="mr-2 tag-filter"> ${escapeHtml(tag)}</label>`
  1203. ).join('');
  1204. }
  1205. async function loadKnowledge() {
  1206. const params = new URLSearchParams();
  1207. params.append('limit', '1000');
  1208. const selectedTypes = Array.from(document.querySelectorAll('.type-filter:checked')).map(el => el.value);
  1209. if (selectedTypes.length > 0) {
  1210. params.append('types', selectedTypes.join(','));
  1211. }
  1212. const selectedTags = Array.from(document.querySelectorAll('.tag-filter:checked')).map(el => el.value);
  1213. if (selectedTags.length > 0) {
  1214. params.append('tags', selectedTags.join(','));
  1215. }
  1216. const ownerFilter = document.getElementById('ownerFilter').value.trim();
  1217. if (ownerFilter) {
  1218. params.append('owner', ownerFilter);
  1219. }
  1220. const scopesFilter = document.getElementById('scopesFilter').value.trim();
  1221. if (scopesFilter) {
  1222. params.append('scopes', scopesFilter);
  1223. }
  1224. try {
  1225. const res = await fetch(`/api/knowledge?${params.toString()}`);
  1226. if (!res.ok) {
  1227. console.error('加载失败:', res.status, res.statusText);
  1228. document.getElementById('knowledgeList').innerHTML = '<p class="text-red-500 text-center py-8">加载失败,请刷新页面重试</p>';
  1229. return;
  1230. }
  1231. const data = await res.json();
  1232. allKnowledge = data.results || [];
  1233. renderKnowledge(allKnowledge);
  1234. } catch (error) {
  1235. console.error('加载错误:', error);
  1236. document.getElementById('knowledgeList').innerHTML = '<p class="text-red-500 text-center py-8">加载错误: ' + error.message + '</p>';
  1237. }
  1238. }
  1239. function applyFilters() {
  1240. loadKnowledge();
  1241. }
  1242. function renderKnowledge(list) {
  1243. const container = document.getElementById('knowledgeList');
  1244. if (list.length === 0) {
  1245. container.innerHTML = '<p class="text-gray-500 text-center py-8">暂无知识</p>';
  1246. return;
  1247. }
  1248. container.innerHTML = list.map(k => {
  1249. // 确保types是数组
  1250. let types = [];
  1251. if (Array.isArray(k.types)) {
  1252. types = k.types;
  1253. } else if (typeof k.types === 'string') {
  1254. // 如果是JSON字符串(以[开头),尝试解析
  1255. if (k.types.startsWith('[')) {
  1256. try {
  1257. types = JSON.parse(k.types);
  1258. } catch (e) {
  1259. console.error('解析types失败:', k.types, e);
  1260. types = [k.types];
  1261. }
  1262. } else {
  1263. // 如果是普通字符串,包装成数组
  1264. types = [k.types];
  1265. }
  1266. }
  1267. const eval_data = k.eval || {};
  1268. return `
  1269. <div class="bg-white rounded-lg shadow p-6 hover:shadow-lg transition cursor-pointer" onclick="openEditModal('${k.id}')">
  1270. <div class="flex justify-between items-start mb-2">
  1271. <div class="flex gap-2 flex-wrap">
  1272. ${types.map(t => `<span class="bg-blue-100 text-blue-800 text-xs px-2 py-1 rounded">${t}</span>`).join('')}
  1273. </div>
  1274. <span class="text-sm text-gray-500">${eval_data.score || 3}/5</span>
  1275. </div>
  1276. <h3 class="text-lg font-semibold text-gray-800 mb-2">${escapeHtml(k.task)}</h3>
  1277. <p class="text-sm text-gray-600 mb-2">${escapeHtml(k.content.substring(0, 150))}${k.content.length > 150 ? '...' : ''}</p>
  1278. <div class="flex justify-between text-xs text-gray-500">
  1279. <span>Owner: ${k.owner || 'N/A'}</span>
  1280. <span>${new Date(k.created_at).toLocaleDateString()}</span>
  1281. </div>
  1282. </div>
  1283. `;
  1284. }).join('');
  1285. }
  1286. function openAddModal() {
  1287. document.getElementById('modalTitle').textContent = '新增知识';
  1288. document.getElementById('knowledgeForm').reset();
  1289. document.getElementById('editId').value = '';
  1290. document.querySelectorAll('.type-checkbox').forEach(el => el.checked = false);
  1291. document.getElementById('modal').classList.remove('hidden');
  1292. }
  1293. async function openEditModal(id) {
  1294. const k = allKnowledge.find(item => item.id === id);
  1295. if (!k) return;
  1296. document.getElementById('modalTitle').textContent = '编辑知识';
  1297. document.getElementById('editId').value = k.id;
  1298. document.getElementById('taskInput').value = k.task;
  1299. document.getElementById('contentInput').value = k.content;
  1300. document.getElementById('tagsInput').value = JSON.stringify(k.tags);
  1301. document.getElementById('scopesInput').value = k.scopes.join(', ');
  1302. document.getElementById('ownerInput').value = k.owner;
  1303. document.querySelectorAll('.type-checkbox').forEach(el => {
  1304. el.checked = k.types.includes(el.value);
  1305. });
  1306. document.getElementById('modal').classList.remove('hidden');
  1307. }
  1308. function closeModal() {
  1309. document.getElementById('modal').classList.add('hidden');
  1310. }
  1311. document.getElementById('knowledgeForm').addEventListener('submit', async (e) => {
  1312. e.preventDefault();
  1313. const editId = document.getElementById('editId').value;
  1314. const task = document.getElementById('taskInput').value;
  1315. const content = document.getElementById('contentInput').value;
  1316. const types = Array.from(document.querySelectorAll('.type-checkbox:checked')).map(el => el.value);
  1317. const tagsText = document.getElementById('tagsInput').value.trim();
  1318. const scopesText = document.getElementById('scopesInput').value.trim();
  1319. const owner = document.getElementById('ownerInput').value.trim();
  1320. let tags = {};
  1321. if (tagsText) {
  1322. try {
  1323. tags = JSON.parse(tagsText);
  1324. } catch (e) {
  1325. alert('Tags JSON 格式错误');
  1326. return;
  1327. }
  1328. }
  1329. const scopes = scopesText ? scopesText.split(',').map(s => s.trim()).filter(s => s) : ['org:cybertogether'];
  1330. if (editId) {
  1331. // 编辑
  1332. const res = await fetch(`/api/knowledge/${editId}`, {
  1333. method: 'PATCH',
  1334. headers: {'Content-Type': 'application/json'},
  1335. body: JSON.stringify({task, content, types, tags, scopes, owner})
  1336. });
  1337. if (!res.ok) {
  1338. alert('更新失败');
  1339. return;
  1340. }
  1341. } else {
  1342. // 新增
  1343. const res = await fetch('/api/knowledge', {
  1344. method: 'POST',
  1345. headers: {'Content-Type': 'application/json'},
  1346. body: JSON.stringify({task, content, types, tags, scopes, owner})
  1347. });
  1348. if (!res.ok) {
  1349. alert('新增失败');
  1350. return;
  1351. }
  1352. }
  1353. closeModal();
  1354. await loadKnowledge();
  1355. });
  1356. function escapeHtml(text) {
  1357. const div = document.createElement('div');
  1358. div.textContent = text;
  1359. return div.innerHTML;
  1360. }
  1361. loadTags();
  1362. loadKnowledge();
  1363. </script>
  1364. </body>
  1365. </html>"""
  1366. if __name__ == "__main__":
  1367. import uvicorn
  1368. uvicorn.run(app, host="0.0.0.0", port=9999)