server.py 82 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321
  1. """
  2. KnowHub Server
  3. Agent 工具使用经验的共享平台。
  4. FastAPI + Milvus Lite(知识)+ SQLite(资源),单文件部署。
  5. """
  6. import os
  7. import re
  8. import json
  9. import asyncio
  10. import base64
  11. import time
  12. import uuid
  13. from contextlib import asynccontextmanager
  14. from datetime import datetime, timezone
  15. from typing import Optional, List, Dict
  16. from pathlib import Path
  17. from cryptography.hazmat.primitives.ciphers.aead import AESGCM
  18. from fastapi import FastAPI, HTTPException, Query, Header, Body, BackgroundTasks
  19. from fastapi.responses import HTMLResponse, FileResponse
  20. from fastapi.staticfiles import StaticFiles
  21. from pydantic import BaseModel, Field
  22. # 导入 LLM 调用(需要 agent 模块在 Python path 中)
  23. import sys
  24. sys.path.insert(0, str(Path(__file__).parent.parent))
  25. # 加载环境变量
  26. from dotenv import load_dotenv
  27. load_dotenv(Path(__file__).parent.parent / ".env")
  28. from agent.llm import create_openrouter_llm_call, create_qwen_llm_call
  29. from knowhub.kb_manage_prompts import (
  30. DEDUP_RELATION_PROMPT,
  31. TOOL_ANALYSIS_PROMPT,
  32. RERANK_PROMPT_TEMPLATE,
  33. KNOWLEDGE_EVOLVE_PROMPT_TEMPLATE,
  34. KNOWLEDGE_SLIM_PROMPT_TEMPLATE,
  35. MESSAGE_EXTRACT_PROMPT_TEMPLATE,
  36. )
  37. _dedup_llm = create_openrouter_llm_call(model="google/gemini-2.5-flash-lite")
  38. _tool_analysis_llm = create_qwen_llm_call(model="qwen3.5-plus")
  39. # 导入向量存储和 embedding
  40. from knowhub.knowhub_db.pg_store import PostgreSQLStore
  41. from knowhub.knowhub_db.pg_resource_store import PostgreSQLResourceStore
  42. from knowhub.knowhub_db.pg_tool_store import PostgreSQLToolStore
  43. from knowhub.knowhub_db.pg_capability_store import PostgreSQLCapabilityStore
  44. from knowhub.knowhub_db.pg_requirement_store import PostgreSQLRequirementStore
  45. from knowhub.embeddings import get_embedding, get_embeddings_batch
  46. BRAND_NAME = os.getenv("BRAND_NAME", "KnowHub")
  47. BRAND_API_ENV = os.getenv("BRAND_API_ENV", "KNOWHUB_API")
  48. BRAND_DB = os.getenv("BRAND_DB", "knowhub.db")
  49. # 组织密钥配置(格式:org1:key1_base64,org2:key2_base64)
  50. ORG_KEYS_RAW = os.getenv("ORG_KEYS", "")
  51. ORG_KEYS = {}
  52. if ORG_KEYS_RAW:
  53. for pair in ORG_KEYS_RAW.split(","):
  54. if ":" in pair:
  55. org, key_b64 = pair.split(":", 1)
  56. ORG_KEYS[org.strip()] = key_b64.strip()
  57. DB_PATH = Path(__file__).parent / BRAND_DB
  58. # 全局 PostgreSQL 存储实例
  59. pg_store: Optional[PostgreSQLStore] = None
  60. pg_resource_store: Optional[PostgreSQLResourceStore] = None
  61. pg_tool_store: Optional[PostgreSQLToolStore] = None
  62. pg_capability_store: Optional[PostgreSQLCapabilityStore] = None
  63. pg_requirement_store: Optional[PostgreSQLRequirementStore] = None
  64. # --- 加密/解密 ---
  65. def get_org_key(resource_id: str) -> Optional[bytes]:
  66. """从content_id提取组织前缀,返回对应密钥"""
  67. if "/" in resource_id:
  68. org = resource_id.split("/")[0]
  69. if org in ORG_KEYS:
  70. return base64.b64decode(ORG_KEYS[org])
  71. return None
  72. def encrypt_content(resource_id: str, plaintext: str) -> str:
  73. """加密内容,返回格式:encrypted:AES256-GCM:{base64_data}"""
  74. if not plaintext:
  75. return ""
  76. key = get_org_key(resource_id)
  77. if not key:
  78. # 没有配置密钥,明文存储(不推荐)
  79. return plaintext
  80. aesgcm = AESGCM(key)
  81. nonce = os.urandom(12) # 96-bit nonce
  82. ciphertext = aesgcm.encrypt(nonce, plaintext.encode("utf-8"), None)
  83. # 组合 nonce + ciphertext
  84. encrypted_data = nonce + ciphertext
  85. encoded = base64.b64encode(encrypted_data).decode("ascii")
  86. return f"encrypted:AES256-GCM:{encoded}"
  87. def decrypt_content(resource_id: str, encrypted_text: str, provided_key: Optional[str] = None) -> str:
  88. """解密内容,如果没有提供密钥或密钥错误,返回[ENCRYPTED]"""
  89. if not encrypted_text:
  90. return ""
  91. if not encrypted_text.startswith("encrypted:AES256-GCM:"):
  92. # 未加密的内容,直接返回
  93. return encrypted_text
  94. # 提取加密数据
  95. encoded = encrypted_text.split(":", 2)[2]
  96. encrypted_data = base64.b64decode(encoded)
  97. nonce = encrypted_data[:12]
  98. ciphertext = encrypted_data[12:]
  99. # 获取密钥
  100. key = None
  101. if provided_key:
  102. # 使用提供的密钥
  103. try:
  104. key = base64.b64decode(provided_key)
  105. except Exception:
  106. return "[ENCRYPTED]"
  107. else:
  108. # 从配置中获取
  109. key = get_org_key(resource_id)
  110. if not key:
  111. return "[ENCRYPTED]"
  112. try:
  113. aesgcm = AESGCM(key)
  114. plaintext = aesgcm.decrypt(nonce, ciphertext, None)
  115. return plaintext.decode("utf-8")
  116. except Exception:
  117. return "[ENCRYPTED]"
  118. def serialize_milvus_result(data):
  119. """将 Milvus 返回的数据转换为可序列化的字典"""
  120. # 基本类型直接返回
  121. if data is None or isinstance(data, (str, int, float, bool)):
  122. return data
  123. # 字典类型递归处理
  124. if isinstance(data, dict):
  125. return {k: serialize_milvus_result(v) for k, v in data.items()}
  126. # 列表/元组类型递归处理
  127. if isinstance(data, (list, tuple)):
  128. return [serialize_milvus_result(item) for item in data]
  129. # 尝试转换为字典(对于有 to_dict 方法的对象)
  130. if hasattr(data, 'to_dict') and callable(getattr(data, 'to_dict')):
  131. try:
  132. return serialize_milvus_result(data.to_dict())
  133. except:
  134. pass
  135. # 尝试转换为列表(对于可迭代对象,如 RepeatedScalarContainer)
  136. if hasattr(data, '__iter__') and not isinstance(data, (str, bytes, dict)):
  137. try:
  138. # 强制转换为列表并递归处理
  139. result = []
  140. for item in data:
  141. result.append(serialize_milvus_result(item))
  142. return result
  143. except:
  144. pass
  145. # 尝试获取对象的属性字典
  146. if hasattr(data, '__dict__'):
  147. try:
  148. return serialize_milvus_result(vars(data))
  149. except:
  150. pass
  151. # 最后的 fallback:对于无法处理的类型,返回 None 而不是字符串表示
  152. # 这样可以避免产生无法序列化的字符串
  153. return None
  154. # --- Models ---
  155. class ResourceIn(BaseModel):
  156. id: str
  157. title: str = ""
  158. body: str
  159. secure_body: str = ""
  160. content_type: str = "text" # text|code|credential|cookie
  161. metadata: dict = {}
  162. sort_order: int = 0
  163. submitted_by: str = ""
  164. class ResourcePatchIn(BaseModel):
  165. """PATCH /api/resource/{id} 请求体"""
  166. title: Optional[str] = None
  167. body: Optional[str] = None
  168. secure_body: Optional[str] = None
  169. content_type: Optional[str] = None
  170. metadata: Optional[dict] = None
  171. # Knowledge Models
  172. class KnowledgeIn(BaseModel):
  173. task: str
  174. content: str
  175. types: list[str] = ["strategy"]
  176. tags: dict = {}
  177. scopes: list[str] = ["org:cybertogether"]
  178. owner: str = ""
  179. message_id: str = ""
  180. resource_ids: list[str] = []
  181. source: dict = {} # {name, category, urls, agent_id, submitted_by, timestamp}
  182. eval: dict = {} # {score, helpful, harmful, confidence}
  183. support_capability: list[str] = []
  184. tools: list[str] = []
  185. class KnowledgeOut(BaseModel):
  186. id: str
  187. message_id: str
  188. types: list[str]
  189. task: str
  190. tags: dict
  191. scopes: list[str]
  192. owner: str
  193. content: str
  194. resource_ids: list[str]
  195. source: dict
  196. eval: dict
  197. created_at: str
  198. updated_at: str
  199. class KnowledgeUpdateIn(BaseModel):
  200. add_helpful_case: Optional[dict] = None
  201. add_harmful_case: Optional[dict] = None
  202. update_score: Optional[int] = Field(default=None, ge=1, le=5)
  203. evolve_feedback: Optional[str] = None
  204. class KnowledgePatchIn(BaseModel):
  205. """PATCH /api/knowledge/{id} 请求体(直接字段编辑)"""
  206. task: Optional[str] = None
  207. content: Optional[str] = None
  208. types: Optional[list[str]] = None
  209. tags: Optional[dict] = None
  210. scopes: Optional[list[str]] = None
  211. owner: Optional[str] = None
  212. support_capability: Optional[list[str]] = None
  213. tools: Optional[list[str]] = None
  214. class MessageExtractIn(BaseModel):
  215. """POST /api/extract 请求体(消息历史提取)"""
  216. messages: list[dict] # [{role: str, content: str}, ...]
  217. agent_id: str = "unknown"
  218. submitted_by: str # 必填,作为 owner
  219. session_key: str = ""
  220. class KnowledgeBatchUpdateIn(BaseModel):
  221. feedback_list: list[dict]
  222. class KnowledgeVerifyIn(BaseModel):
  223. action: str # "approve" | "reject"
  224. verified_by: str = "user"
  225. class KnowledgeBatchVerifyIn(BaseModel):
  226. knowledge_ids: List[str]
  227. action: str # "approve"
  228. verified_by: str
  229. class KnowledgeSearchResponse(BaseModel):
  230. results: list[dict]
  231. count: int
  232. # --- Tool Models ---
  233. class ToolIn(BaseModel):
  234. id: str
  235. name: str = ""
  236. version: Optional[str] = None
  237. introduction: str = ""
  238. tutorial: str = ""
  239. input: dict | str = ""
  240. output: dict | str = ""
  241. status: str = "未接入"
  242. capabilities: list[str] = []
  243. tool_knowledge: list[str] = []
  244. case_knowledge: list[str] = []
  245. process_knowledge: list[str] = []
  246. implemented_tool_ids: list[str] = []
  247. class ToolPatchIn(BaseModel):
  248. name: Optional[str] = None
  249. version: Optional[str] = None
  250. introduction: Optional[str] = None
  251. tutorial: Optional[str] = None
  252. input: Optional[dict | str] = None
  253. output: Optional[dict | str] = None
  254. status: Optional[str] = None
  255. capabilities: Optional[list[str]] = None
  256. tool_knowledge: Optional[list[str]] = None
  257. case_knowledge: Optional[list[str]] = None
  258. process_knowledge: Optional[list[str]] = None
  259. implemented_tool_ids: Optional[list[str]] = None
  260. # --- Capability Models ---
  261. class CapabilityIn(BaseModel):
  262. id: str
  263. name: str = ""
  264. criterion: str = ""
  265. description: str = ""
  266. requirements: list[str] = []
  267. implements: dict = {}
  268. tools: list[str] = []
  269. source_knowledge: list[str] = []
  270. class CapabilityPatchIn(BaseModel):
  271. name: Optional[str] = None
  272. criterion: Optional[str] = None
  273. description: Optional[str] = None
  274. requirements: Optional[list[str]] = None
  275. implements: Optional[dict] = None
  276. tools: Optional[list[str]] = None
  277. source_knowledge: Optional[list[str]] = None
  278. # --- Requirement Models ---
  279. class RequirementIn(BaseModel):
  280. id: str
  281. description: str = ""
  282. atomics: list[str] = []
  283. source_nodes: list[dict] = []
  284. status: str = "未满足"
  285. match_result: str = ""
  286. class RequirementPatchIn(BaseModel):
  287. description: Optional[str] = None
  288. atomics: Optional[list[str]] = None
  289. source_nodes: Optional[list[dict]] = None
  290. status: Optional[str] = None
  291. match_result: Optional[str] = None
  292. class ResourceNode(BaseModel):
  293. id: str
  294. title: str
  295. class ResourceOut(BaseModel):
  296. id: str
  297. title: str
  298. body: str
  299. secure_body: str = ""
  300. content_type: str = "text"
  301. metadata: dict = {}
  302. toc: Optional[ResourceNode] = None
  303. children: list[ResourceNode]
  304. prev: Optional[ResourceNode] = None
  305. next: Optional[ResourceNode] = None
  306. # --- Dedup: Globals & Prompt ---
  307. knowledge_processor: Optional["KnowledgeProcessor"] = None
  308. # --- Dedup: RelationCache ---
  309. class RelationCache:
  310. """关系缓存,存储在内存中"""
  311. def __init__(self):
  312. self._cache: Dict[str, List[str]] = {}
  313. def load(self) -> dict:
  314. return self._cache
  315. def save(self, cache: dict):
  316. self._cache = cache
  317. def add_relation(self, relation_type: str, knowledge_id: str):
  318. if relation_type not in self._cache:
  319. self._cache[relation_type] = []
  320. if knowledge_id not in self._cache[relation_type]:
  321. self._cache[relation_type].append(knowledge_id)
  322. # --- Dedup: KnowledgeProcessor ---
  323. class KnowledgeProcessor:
  324. def __init__(self):
  325. self._lock = asyncio.Lock()
  326. self._relation_cache = RelationCache()
  327. async def process_pending(self):
  328. """持续处理 pending 和 dedup_passed 知识直到队列为空,有锁防并发"""
  329. if self._lock.locked():
  330. return
  331. async with self._lock:
  332. # 第一阶段:处理 pending(去重)
  333. while True:
  334. try:
  335. pending = pg_store.query('status == "pending"', limit=50)
  336. except Exception as e:
  337. print(f"[KnowledgeProcessor] 查询 pending 失败: {e}")
  338. break
  339. if not pending:
  340. break
  341. for knowledge in pending:
  342. await self._process_one(knowledge)
  343. # 第二阶段:处理 dedup_passed(工具关联)
  344. while True:
  345. try:
  346. dedup_passed = pg_store.query('status == "dedup_passed"', limit=50)
  347. except Exception as e:
  348. print(f"[KnowledgeProcessor] 查询 dedup_passed 失败: {e}")
  349. break
  350. if not dedup_passed:
  351. break
  352. for knowledge in dedup_passed:
  353. await self._analyze_tool_relation(knowledge)
  354. async def _process_one(self, knowledge: dict):
  355. kid = knowledge["id"]
  356. now = int(time.time())
  357. # 乐观锁:pending → processing(时间戳存秒级)
  358. try:
  359. pg_store.update(kid, {"status": "processing", "updated_at": now})
  360. except Exception as e:
  361. print(f"[KnowledgeProcessor] 锁定 {kid} 失败: {e}")
  362. return
  363. try:
  364. # 向量召回 top-10(只召回 approved/checked)
  365. embedding = knowledge.get("task_embedding") or knowledge.get("embedding")
  366. if not embedding:
  367. embedding = await get_embedding(knowledge["task"])
  368. candidates = pg_store.search(
  369. query_embedding=embedding,
  370. filters='(status == "approved" or status == "checked")',
  371. limit=10
  372. )
  373. candidates = [c for c in candidates if c["id"] != kid]
  374. # 只保留相似度 >= 0.75 的候选,低于阈值的 task 语义差异太大,直接视为 none
  375. candidates = [c for c in candidates if c.get("score", 0) >= 0.75]
  376. if not candidates:
  377. pg_store.update(kid, {"status": "dedup_passed", "updated_at": now})
  378. return
  379. llm_result = await self._llm_judge_relations(knowledge, candidates)
  380. await self._apply_decision(knowledge, llm_result)
  381. except Exception as e:
  382. print(f"[KnowledgeProcessor] 处理 {kid} 失败: {e},回退到 pending")
  383. try:
  384. pg_store.update(kid, {"status": "pending", "updated_at": int(time.time())})
  385. except Exception:
  386. pass
  387. async def _llm_judge_relations(self, new_knowledge: dict, candidates: list) -> dict:
  388. existing_list = "\n".join([
  389. f"[{i+1}] ID: {c['id']} | Task: {c['task']} | Content: {c['content'][:300]}"
  390. for i, c in enumerate(candidates)
  391. ])
  392. prompt = DEDUP_RELATION_PROMPT.format(
  393. new_task=new_knowledge["task"],
  394. new_content=new_knowledge["content"],
  395. existing_list=existing_list
  396. )
  397. for attempt in range(3):
  398. try:
  399. response = await _dedup_llm(
  400. messages=[{"role": "user", "content": prompt}],
  401. )
  402. content = response.get("content", "").strip()
  403. # 清理 markdown 代码块
  404. if "```" in content:
  405. parts = content.split("```")
  406. for part in parts:
  407. part = part.strip()
  408. if part.startswith("json"):
  409. part = part[4:].strip()
  410. try:
  411. result = json.loads(part)
  412. if "final_decision" in result:
  413. content = part
  414. break
  415. except Exception:
  416. continue
  417. result = json.loads(content)
  418. assert result.get("final_decision") in ("approved", "rejected")
  419. return result
  420. except Exception as e:
  421. print(f"[LLM Judge] 第{attempt+1}次失败: {e}")
  422. if attempt < 2:
  423. await asyncio.sleep(1)
  424. return {"final_decision": "approved", "relations": []}
  425. async def _apply_decision(self, new_knowledge: dict, llm_result: dict):
  426. kid = new_knowledge["id"]
  427. final_decision = llm_result.get("final_decision", "approved")
  428. relations = llm_result.get("relations", [])
  429. now = int(time.time())
  430. # 强制规则:如果存在 duplicate 或 subset 关系,必须 rejected
  431. if any(rel.get("type") in ("duplicate", "subset") for rel in relations):
  432. final_decision = "rejected"
  433. if final_decision == "rejected":
  434. # 记录 rejected 知识的关系(便于溯源为什么被拒绝)
  435. rejected_relationships = []
  436. for rel in relations:
  437. old_id = rel.get("old_id")
  438. rel_type = rel.get("type", "none")
  439. if old_id and rel_type != "none":
  440. rejected_relationships.append({"type": rel_type, "target": old_id})
  441. if rel_type in ("duplicate", "subset") and old_id:
  442. try:
  443. old = pg_store.get_by_id(old_id)
  444. if not old:
  445. continue
  446. eval_data = old.get("eval") or {}
  447. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  448. helpful_history = eval_data.get("helpful_history") or []
  449. helpful_history.append({
  450. "source": "dedup",
  451. "related_id": kid,
  452. "relation_type": rel_type,
  453. "timestamp": now
  454. })
  455. eval_data["helpful_history"] = helpful_history
  456. pg_store.update(old_id, {"eval": eval_data, "updated_at": now})
  457. except Exception as e:
  458. print(f"[Apply Decision] 更新旧知识 {old_id} helpful 失败: {e}")
  459. pg_store.update(kid, {"status": "rejected", "relationships": json.dumps(rejected_relationships), "updated_at": now})
  460. else:
  461. new_relationships = []
  462. for rel in relations:
  463. rel_type = rel.get("type", "none")
  464. reverse_type = rel.get("reverse_type", "none")
  465. old_id = rel.get("old_id")
  466. if not old_id or rel_type == "none":
  467. continue
  468. new_relationships.append({"type": rel_type, "target": old_id})
  469. self._relation_cache.add_relation(rel_type, kid)
  470. self._relation_cache.add_relation(rel_type, old_id)
  471. if reverse_type and reverse_type != "none":
  472. try:
  473. old = pg_store.get_by_id(old_id)
  474. if old:
  475. old_rels = old.get("relationships") or []
  476. old_rels.append({"type": reverse_type, "target": kid})
  477. pg_store.update(old_id, {"relationships": json.dumps(old_rels), "updated_at": now})
  478. self._relation_cache.add_relation(reverse_type, old_id)
  479. self._relation_cache.add_relation(reverse_type, kid)
  480. except Exception as e:
  481. print(f"[Apply Decision] 更新旧知识关系 {old_id} 失败: {e}")
  482. pg_store.update(kid, {
  483. "status": "dedup_passed",
  484. "relationships": json.dumps(new_relationships),
  485. "updated_at": now
  486. })
  487. async def _llm_analyze_tools(self, knowledge: dict) -> dict:
  488. """使用 LLM 分析知识中涉及的工具(复用迁移脚本逻辑)"""
  489. task = (knowledge.get("task") or "")[:600]
  490. content = (knowledge.get("content") or "")[:1200]
  491. prompt = TOOL_ANALYSIS_PROMPT.format(task=task, content=content)
  492. try:
  493. response = await _tool_analysis_llm(
  494. messages=[{"role": "user", "content": prompt}],
  495. max_tokens=2048,
  496. temperature=0.1,
  497. )
  498. raw = (response.get("content") or "").strip()
  499. if raw.startswith("```"):
  500. lines = raw.split("\n")
  501. inner = []
  502. in_block = False
  503. for line in lines:
  504. if line.startswith("```"):
  505. in_block = not in_block
  506. continue
  507. if in_block:
  508. inner.append(line)
  509. raw = "\n".join(inner).strip()
  510. return json.loads(raw)
  511. except Exception as e:
  512. print(f"[Tool Analysis LLM] 调用失败: {e}")
  513. raise
  514. async def _create_or_get_tool_resource(self, tool_info: dict) -> Optional[str]:
  515. """创建或获取工具资源(存入 PostgreSQL tool_table)"""
  516. category = tool_info.get("category", "other")
  517. slug = tool_info.get("slug", "")
  518. if not slug:
  519. return None
  520. tool_id = f"tools/{category}/{slug}"
  521. now_ts = int(time.time())
  522. cursor = pg_store._get_cursor()
  523. try:
  524. cursor.execute("SELECT id FROM tool_table WHERE id = %s", (tool_id,))
  525. if cursor.fetchone():
  526. return tool_id
  527. cursor.execute("""
  528. INSERT INTO tool_table (id, name, version, introduction, tutorial, input, output,
  529. updated_time, status, tool_knowledge, case_knowledge, process_knowledge)
  530. VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
  531. """, (
  532. tool_id,
  533. tool_info.get("name", slug),
  534. tool_info.get("version") or None,
  535. tool_info.get("description", ""),
  536. tool_info.get("usage", ""),
  537. json.dumps(tool_info.get("input", "")),
  538. json.dumps(tool_info.get("output", "")),
  539. now_ts,
  540. tool_info.get("status", "未接入"),
  541. json.dumps([]),
  542. json.dumps([]),
  543. json.dumps([]),
  544. ))
  545. pg_store.conn.commit()
  546. print(f"[Tool Resource] 创建新工具: {tool_id}")
  547. return tool_id
  548. finally:
  549. cursor.close()
  550. async def _update_tool_knowledge_index(self, tool_id: str, knowledge_id: str):
  551. """更新工具的 tool_knowledge 关联索引(PostgreSQL tool_table)"""
  552. now_ts = int(time.time())
  553. cursor = pg_store._get_cursor()
  554. try:
  555. cursor.execute("SELECT tool_knowledge FROM tool_table WHERE id = %s", (tool_id,))
  556. row = cursor.fetchone()
  557. if not row:
  558. return
  559. knowledge_ids = row["tool_knowledge"] if isinstance(row["tool_knowledge"], list) else json.loads(row["tool_knowledge"] or "[]")
  560. if knowledge_id not in knowledge_ids:
  561. knowledge_ids.append(knowledge_id)
  562. cursor.execute(
  563. "UPDATE tool_table SET tool_knowledge = %s, updated_time = %s WHERE id = %s",
  564. (json.dumps(knowledge_ids), now_ts, tool_id)
  565. )
  566. pg_store.conn.commit()
  567. finally:
  568. cursor.close()
  569. async def _analyze_tool_relation(self, knowledge: dict):
  570. """分析知识与工具的关联关系"""
  571. kid = knowledge["id"]
  572. now = int(time.time())
  573. # 乐观锁:dedup_passed → analyzing
  574. try:
  575. pg_store.update(kid, {"status": "analyzing", "updated_at": now})
  576. except Exception as e:
  577. print(f"[Tool Analysis] 锁定 {kid} 失败: {e}")
  578. return
  579. try:
  580. tool_analysis = await self._llm_analyze_tools(knowledge)
  581. has_tools = bool(tool_analysis and tool_analysis.get("has_tools"))
  582. existing_tags = knowledge.get("tags") or {}
  583. has_tool_tag = existing_tags.get("tool") is True
  584. # 情况1:LLM 判定无工具,但有 tool tag → 重新分析一次
  585. if not has_tools and has_tool_tag:
  586. print(f"[Tool Analysis] {kid} LLM 判定无工具但有 tool tag,重新分析")
  587. tool_analysis = await self._llm_analyze_tools(knowledge)
  588. has_tools = bool(tool_analysis and tool_analysis.get("has_tools"))
  589. # 重新分析后仍然不一致 → 知识模糊,rejected
  590. if not has_tools:
  591. pg_store.update(kid, {"status": "rejected", "updated_at": now})
  592. print(f"[Tool Analysis] {kid} 两次判定不一致,知识模糊,rejected")
  593. return
  594. # 情况2:无工具且无 tool tag → 直接 approved
  595. if not has_tools:
  596. pg_store.update(kid, {"status": "approved", "updated_at": now})
  597. return
  598. # 情况3/4:有工具 → 创建资源并关联
  599. tool_ids = []
  600. for tool_info in (tool_analysis.get("tools") or []):
  601. tool_id = await self._create_or_get_tool_resource(tool_info)
  602. if tool_id:
  603. tool_ids.append(tool_id)
  604. existing_resource_ids = knowledge.get("resource_ids") or []
  605. updated_resource_ids = list(set(existing_resource_ids + tool_ids))
  606. updates: dict = {
  607. "status": "approved",
  608. "resource_ids": updated_resource_ids,
  609. "updated_at": now
  610. }
  611. # 有工具但无 tool tag → 添加 tag
  612. if not has_tool_tag:
  613. updated_tags = dict(existing_tags)
  614. updated_tags["tool"] = True
  615. updates["tags"] = updated_tags
  616. print(f"[Tool Analysis] {kid} 添加 tool tag")
  617. pg_store.update(kid, updates)
  618. for tool_id in tool_ids:
  619. await self._update_tool_knowledge_index(tool_id, kid)
  620. print(f"[Tool Analysis] {kid} 关联了 {len(tool_ids)} 个工具")
  621. except Exception as e:
  622. print(f"[Tool Analysis] {kid} 分析失败: {e},回退到 dedup_passed")
  623. try:
  624. pg_store.update(kid, {"status": "dedup_passed", "updated_at": int(time.time())})
  625. except Exception:
  626. pass
  627. async def _periodic_processor():
  628. """每60秒检测超时条目并回滚:processing(>5min)→pending,analyzing(>10min)→dedup_passed"""
  629. while True:
  630. await asyncio.sleep(60)
  631. try:
  632. now = int(time.time())
  633. # 回滚超时的 processing(5分钟 → pending)
  634. timeout_5min = now - 300
  635. processing = pg_store.query('status == "processing"', limit=200)
  636. for item in processing:
  637. updated_at = item.get("updated_at", 0) or 0
  638. updated_at_sec = updated_at // 1000 if updated_at > 1_000_000_000_000 else updated_at
  639. if updated_at_sec < timeout_5min:
  640. print(f"[Periodic] 回滚超时 processing → pending: {item['id']}")
  641. pg_store.update(item["id"], {"status": "pending", "updated_at": int(time.time())})
  642. # 回滚超时的 analyzing(10分钟 → dedup_passed)
  643. timeout_10min = now - 600
  644. analyzing = pg_store.query('status == "analyzing"', limit=200)
  645. for item in analyzing:
  646. updated_at = item.get("updated_at", 0) or 0
  647. updated_at_sec = updated_at // 1000 if updated_at > 1_000_000_000_000 else updated_at
  648. if updated_at_sec < timeout_10min:
  649. print(f"[Periodic] 回滚超时 analyzing → dedup_passed: {item['id']}")
  650. pg_store.update(item["id"], {"status": "dedup_passed", "updated_at": int(time.time())})
  651. except Exception as e:
  652. print(f"[Periodic] 定时任务错误: {e}")
  653. # --- App ---
  654. @asynccontextmanager
  655. async def lifespan(app: FastAPI):
  656. global pg_store, pg_resource_store, pg_tool_store, pg_capability_store, pg_requirement_store, knowledge_processor
  657. # 初始化 PostgreSQL(knowledge + resources + tools + capabilities + requirements)
  658. pg_store = PostgreSQLStore()
  659. pg_resource_store = PostgreSQLResourceStore()
  660. pg_tool_store = PostgreSQLToolStore()
  661. pg_capability_store = PostgreSQLCapabilityStore()
  662. pg_requirement_store = PostgreSQLRequirementStore()
  663. # 初始化去重处理器 + 启动定时兜底任务
  664. knowledge_processor = KnowledgeProcessor()
  665. periodic_task = asyncio.create_task(_periodic_processor())
  666. yield
  667. # 清理
  668. periodic_task.cancel()
  669. try:
  670. await periodic_task
  671. except asyncio.CancelledError:
  672. pass
  673. pg_store.close()
  674. pg_resource_store.close()
  675. pg_tool_store.close()
  676. pg_capability_store.close()
  677. pg_requirement_store.close()
  678. app = FastAPI(title=BRAND_NAME, lifespan=lifespan)
  679. # 挂载静态文件
  680. STATIC_DIR = Path(__file__).parent / "frontend" / "dist"
  681. if STATIC_DIR.exists():
  682. app.mount("/assets", StaticFiles(directory=str(STATIC_DIR / "assets")), name="assets")
  683. # --- Knowledge API ---
  684. @app.post("/api/resource", status_code=201)
  685. def submit_resource(resource: ResourceIn):
  686. """提交资源(存入 PostgreSQL resources 表)"""
  687. try:
  688. # 加密敏感内容
  689. encrypted_secure_body = encrypt_content(resource.id, resource.secure_body)
  690. pg_resource_store.insert_or_update({
  691. 'id': resource.id,
  692. 'title': resource.title,
  693. 'body': resource.body,
  694. 'secure_body': encrypted_secure_body,
  695. 'content_type': resource.content_type,
  696. 'metadata': resource.metadata,
  697. 'sort_order': resource.sort_order,
  698. 'submitted_by': resource.submitted_by
  699. })
  700. return {"status": "ok", "id": resource.id}
  701. except Exception as e:
  702. raise HTTPException(status_code=500, detail=str(e))
  703. @app.get("/api/resource/{resource_id:path}", response_model=ResourceOut)
  704. def get_resource(resource_id: str, x_org_key: Optional[str] = Header(None)):
  705. """获取资源详情(从 PostgreSQL)"""
  706. try:
  707. row = pg_resource_store.get_by_id(resource_id)
  708. if not row:
  709. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  710. # 解密敏感内容
  711. secure_body = decrypt_content(resource_id, row.get("secure_body", ""), x_org_key)
  712. # 计算导航上下文
  713. root_id = resource_id.split("/")[0] if "/" in resource_id else resource_id
  714. # TOC (根节点)
  715. toc = None
  716. if "/" in resource_id:
  717. toc_row = pg_resource_store.get_by_id(root_id)
  718. if toc_row:
  719. toc = ResourceNode(id=toc_row["id"], title=toc_row["title"])
  720. # Children (子节点)
  721. children_rows = pg_resource_store.list_resources(prefix=f"{resource_id}/", limit=1000)
  722. children = [ResourceNode(id=r["id"], title=r["title"]) for r in children_rows
  723. if r["id"].count("/") == resource_id.count("/") + 1]
  724. # Prev/Next (同级节点)
  725. prev_node, next_node = pg_resource_store.get_siblings(resource_id)
  726. prev = ResourceNode(id=prev_node["id"], title=prev_node["title"]) if prev_node else None
  727. next = ResourceNode(id=next_node["id"], title=next_node["title"]) if next_node else None
  728. return ResourceOut(
  729. id=row["id"],
  730. title=row["title"],
  731. body=row["body"],
  732. secure_body=secure_body,
  733. content_type=row["content_type"],
  734. metadata=row.get("metadata", {}),
  735. toc=toc,
  736. children=children,
  737. prev=prev,
  738. next=next,
  739. )
  740. except HTTPException:
  741. raise
  742. except Exception as e:
  743. raise HTTPException(status_code=500, detail=str(e))
  744. @app.patch("/api/resource/{resource_id:path}")
  745. def patch_resource(resource_id: str, patch: ResourcePatchIn):
  746. """更新resource字段(PostgreSQL)"""
  747. try:
  748. # 检查是否存在
  749. if not pg_resource_store.get_by_id(resource_id):
  750. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  751. # 构建更新字典
  752. updates = {}
  753. if patch.title is not None:
  754. updates['title'] = patch.title
  755. if patch.body is not None:
  756. updates['body'] = patch.body
  757. if patch.secure_body is not None:
  758. updates['secure_body'] = encrypt_content(resource_id, patch.secure_body)
  759. if patch.content_type is not None:
  760. updates['content_type'] = patch.content_type
  761. if patch.metadata is not None:
  762. updates['metadata'] = patch.metadata
  763. if not updates:
  764. return {"status": "ok", "message": "No fields to update"}
  765. pg_resource_store.update(resource_id, updates)
  766. return {"status": "ok", "id": resource_id}
  767. except HTTPException:
  768. raise
  769. except Exception as e:
  770. raise HTTPException(status_code=500, detail=str(e))
  771. @app.get("/api/resource")
  772. def list_resources(
  773. content_type: Optional[str] = Query(None),
  774. limit: int = Query(100, ge=1, le=1000)
  775. ):
  776. """列出所有resource(PostgreSQL)"""
  777. try:
  778. results = pg_resource_store.list_resources(
  779. content_type=content_type,
  780. limit=limit
  781. )
  782. return {"results": results, "count": len(results)}
  783. except Exception as e:
  784. raise HTTPException(status_code=500, detail=str(e))
  785. @app.delete("/api/resource/{resource_id:path}")
  786. def delete_resource(resource_id: str):
  787. """删除单个resource(PostgreSQL)"""
  788. try:
  789. if not pg_resource_store.get_by_id(resource_id):
  790. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  791. pg_resource_store.delete(resource_id)
  792. return {"status": "ok", "id": resource_id}
  793. except HTTPException:
  794. raise
  795. except Exception as e:
  796. raise HTTPException(status_code=500, detail=str(e))
  797. # --- Knowledge API ---
  798. # ===== Knowledge API =====
  799. async def _llm_rerank(query: str, candidates: list[dict], top_k: int) -> list[str]:
  800. """
  801. 使用 LLM 对候选知识进行精排
  802. Args:
  803. query: 查询文本
  804. candidates: 候选知识列表
  805. top_k: 返回数量
  806. Returns:
  807. 排序后的知识 ID 列表
  808. """
  809. if not candidates:
  810. return []
  811. # 构造 prompt
  812. candidates_text = "\n".join([
  813. f"[{i+1}] ID: {c['id']}\nTask: {c['task']}\nContent: {c['content'][:200]}..."
  814. for i, c in enumerate(candidates)
  815. ])
  816. prompt = RERANK_PROMPT_TEMPLATE.format(
  817. top_k=top_k,
  818. query=query,
  819. candidates_text=candidates_text
  820. )
  821. try:
  822. response = await _dedup_llm(
  823. messages=[{"role": "user", "content": prompt}],
  824. )
  825. content = response.get("content", "").strip()
  826. # 解析 ID 列表
  827. selected_ids = [
  828. idx.strip()
  829. for idx in re.split(r'[,\s]+', content)
  830. if idx.strip().startswith(("knowledge-", "research-"))
  831. ]
  832. return selected_ids[:top_k]
  833. except Exception as e:
  834. print(f"[LLM Rerank] 失败: {e}")
  835. return []
  836. @app.get("/api/knowledge/search")
  837. async def search_knowledge_api(
  838. q: str = Query(..., description="查询文本"),
  839. top_k: int = Query(default=5, ge=1, le=20),
  840. min_score: int = Query(default=3, ge=1, le=5),
  841. types: Optional[str] = None,
  842. owner: Optional[str] = None
  843. ):
  844. """检索知识(向量召回 + LLM 精排)"""
  845. try:
  846. # 1. 生成查询向量
  847. query_embedding = await get_embedding(q)
  848. # 2. 构建过滤表达式
  849. filters = []
  850. if types:
  851. type_list = [t.strip() for t in types.split(',') if t.strip()]
  852. for t in type_list:
  853. filters.append(f'array_contains(types, "{t}")')
  854. if owner:
  855. owner_list = [o.strip() for o in owner.split(',') if o.strip()]
  856. if len(owner_list) == 1:
  857. filters.append(f'owner == "{owner_list[0]}"')
  858. else:
  859. # 多个owner用OR连接
  860. owner_filters = [f'owner == "{o}"' for o in owner_list]
  861. filters.append(f'({" or ".join(owner_filters)})')
  862. # 添加 min_score 过滤
  863. filters.append(f'eval["score"] >= {min_score}')
  864. # 只搜索 approved 和 checked 的知识
  865. filters.append('(status == "approved" or status == "checked")')
  866. filter_expr = ' and '.join(filters) if filters else None
  867. # 3. 向量召回(3*k 个候选)
  868. recall_limit = top_k * 3
  869. candidates = pg_store.search(
  870. query_embedding=query_embedding,
  871. filters=filter_expr,
  872. limit=recall_limit
  873. )
  874. if not candidates:
  875. return {"results": [], "count": 0, "reranked": False}
  876. # 转换为可序列化的格式
  877. serialized_candidates = [serialize_milvus_result(c) for c in candidates]
  878. # 为了保证搜索的极致速度,直接返回向量召回的 top-k(跳过缓慢的 LLM 精排)
  879. return {"results": serialized_candidates[:top_k], "count": len(serialized_candidates[:top_k]), "reranked": False}
  880. except Exception as e:
  881. print(f"[Knowledge Search] 错误: {e}")
  882. raise HTTPException(status_code=500, detail=str(e))
  883. @app.post("/api/knowledge", status_code=201)
  884. async def save_knowledge(knowledge: KnowledgeIn, background_tasks: BackgroundTasks):
  885. """保存新知识"""
  886. try:
  887. # 生成 ID
  888. timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
  889. random_suffix = uuid.uuid4().hex[:4]
  890. knowledge_id = f"knowledge-{timestamp}-{random_suffix}"
  891. now = int(time.time())
  892. # 设置默认值
  893. owner = knowledge.owner or f"agent:{knowledge.source.get('agent_id', 'unknown')}"
  894. # 准备 source
  895. source = {
  896. "name": knowledge.source.get("name", ""),
  897. "category": knowledge.source.get("category", ""),
  898. "urls": knowledge.source.get("urls", []),
  899. "agent_id": knowledge.source.get("agent_id", "unknown"),
  900. "submitted_by": knowledge.source.get("submitted_by", ""),
  901. "timestamp": datetime.now(timezone.utc).isoformat(),
  902. "message_id": knowledge.message_id
  903. }
  904. # 准备 eval
  905. eval_data = {
  906. "score": knowledge.eval.get("score", 3),
  907. "helpful": knowledge.eval.get("helpful", 1),
  908. "harmful": knowledge.eval.get("harmful", 0),
  909. "confidence": knowledge.eval.get("confidence", 0.5),
  910. "helpful_history": [],
  911. "harmful_history": []
  912. }
  913. # 生成向量(task_embedding + content_embedding 双向量)
  914. task_embedding = await get_embedding(knowledge.task)
  915. content_embedding = await get_embedding(knowledge.content)
  916. # 提取 tag keys(用于高效筛选)
  917. tag_keys = list(knowledge.tags.keys()) if isinstance(knowledge.tags, dict) else []
  918. # 准备插入数据
  919. insert_data = {
  920. "id": knowledge_id,
  921. "task_embedding": task_embedding,
  922. "content_embedding": content_embedding,
  923. "message_id": knowledge.message_id,
  924. "task": knowledge.task,
  925. "content": knowledge.content,
  926. "types": knowledge.types,
  927. "tags": knowledge.tags,
  928. "tag_keys": tag_keys,
  929. "scopes": knowledge.scopes,
  930. "owner": owner,
  931. "resource_ids": knowledge.resource_ids,
  932. "source": source,
  933. "eval": eval_data,
  934. "created_at": now,
  935. "updated_at": now,
  936. "status": "pending",
  937. "relationships": json.dumps([]),
  938. "support_capability": knowledge.support_capability,
  939. "tools": knowledge.tools,
  940. }
  941. print(f"[Save Knowledge] 插入数据: {json.dumps({k: v for k, v in insert_data.items() if k != 'embedding'}, ensure_ascii=False)}")
  942. # 插入 Milvus
  943. pg_store.insert(insert_data)
  944. # 触发后台去重处理
  945. background_tasks.add_task(knowledge_processor.process_pending)
  946. return {"status": "pending", "knowledge_id": knowledge_id, "message": "知识已入队,正在处理去重..."}
  947. except Exception as e:
  948. print(f"[Save Knowledge] 错误: {e}")
  949. raise HTTPException(status_code=500, detail=str(e))
  950. @app.get("/api/knowledge")
  951. def list_knowledge(
  952. page: int = Query(default=1, ge=1),
  953. page_size: int = Query(default=20, ge=1, le=1000),
  954. types: Optional[str] = None,
  955. scopes: Optional[str] = None,
  956. owner: Optional[str] = None,
  957. tags: Optional[str] = None,
  958. status: Optional[str] = None
  959. ):
  960. """列出知识(支持后端筛选和分页)"""
  961. try:
  962. # 构建过滤表达式
  963. filters = []
  964. # types 支持多个,用 AND 连接(交集:必须同时包含所有选中的type)
  965. if types:
  966. type_list = [t.strip() for t in types.split(',') if t.strip()]
  967. for t in type_list:
  968. filters.append(f'array_contains(types, "{t}")')
  969. if scopes:
  970. filters.append(f'array_contains(scopes, "{scopes}")')
  971. if owner:
  972. owner_list = [o.strip() for o in owner.split(',') if o.strip()]
  973. if len(owner_list) == 1:
  974. filters.append(f'owner == "{owner_list[0]}"')
  975. else:
  976. # 多个owner用OR连接
  977. owner_filters = [f'owner == "{o}"' for o in owner_list]
  978. filters.append(f'({" or ".join(owner_filters)})')
  979. # tags 支持多个,用 AND 连接(使用 tag_keys 数组进行高效筛选)
  980. if tags:
  981. tag_list = [t.strip() for t in tags.split(',') if t.strip()]
  982. for t in tag_list:
  983. filters.append(f'array_contains(tag_keys, "{t}")')
  984. # 只返回指定 status 的知识(默认 approved 和 checked)
  985. status_list = [s.strip() for s in (status or "approved,checked").split(',') if s.strip()]
  986. status_conditions = ' or '.join([f'status == "{s}"' for s in status_list])
  987. filters.append(f'({status_conditions})')
  988. # 如果没有过滤条件,查询所有
  989. filter_expr = ' and '.join(filters) if filters else 'id != ""'
  990. # 查询 Milvus(先获取所有符合条件的数据)
  991. # Milvus 的 limit 是总数限制,我们需要获取足够多的数据来支持分页
  992. max_limit = 10000 # 设置一个合理的上限
  993. results = pg_store.query(filter_expr, limit=max_limit)
  994. # 转换为可序列化的格式
  995. serialized_results = [serialize_milvus_result(r) for r in results]
  996. # 按 created_at 降序排序(最新的在前)
  997. serialized_results.sort(key=lambda x: x.get('created_at', 0), reverse=True)
  998. # 计算分页
  999. total = len(serialized_results)
  1000. total_pages = (total + page_size - 1) // page_size # 向上取整
  1001. start_idx = (page - 1) * page_size
  1002. end_idx = start_idx + page_size
  1003. page_results = serialized_results[start_idx:end_idx]
  1004. return {
  1005. "results": page_results,
  1006. "pagination": {
  1007. "page": page,
  1008. "page_size": page_size,
  1009. "total": total,
  1010. "total_pages": total_pages
  1011. }
  1012. }
  1013. except Exception as e:
  1014. print(f"[List Knowledge] 错误: {e}")
  1015. raise HTTPException(status_code=500, detail=str(e))
  1016. @app.get("/api/knowledge/meta/tags")
  1017. def get_all_tags():
  1018. """获取所有已有的 tags"""
  1019. try:
  1020. # 查询所有知识
  1021. results = pg_store.query('id != ""', limit=10000)
  1022. all_tags = set()
  1023. for item in results:
  1024. # 转换为标准字典
  1025. serialized_item = serialize_milvus_result(item)
  1026. tags_dict = serialized_item.get("tags", {})
  1027. if isinstance(tags_dict, dict):
  1028. for key in tags_dict.keys():
  1029. all_tags.add(key)
  1030. return {"tags": sorted(list(all_tags))}
  1031. except Exception as e:
  1032. print(f"[Get Tags] 错误: {e}")
  1033. raise HTTPException(status_code=500, detail=str(e))
  1034. @app.get("/api/knowledge/pending")
  1035. def get_pending_knowledge(limit: int = Query(default=50, ge=1, le=200)):
  1036. """查询待处理队列(pending + processing + dedup_passed + analyzing)"""
  1037. try:
  1038. pending = pg_store.query(
  1039. 'status == "pending" or status == "processing" or status == "dedup_passed" or status == "analyzing"',
  1040. limit=limit
  1041. )
  1042. serialized = [serialize_milvus_result(r) for r in pending]
  1043. return {"results": serialized, "count": len(serialized)}
  1044. except Exception as e:
  1045. print(f"[Pending] 错误: {e}")
  1046. raise HTTPException(status_code=500, detail=str(e))
  1047. @app.post("/api/knowledge/process")
  1048. async def trigger_process(force: bool = Query(default=False)):
  1049. """手动触发去重处理。force=true 时先回滚所有 processing → pending,analyzing → dedup_passed"""
  1050. try:
  1051. if force:
  1052. processing = pg_store.query('status == "processing"', limit=200)
  1053. for item in processing:
  1054. pg_store.update(item["id"], {"status": "pending", "updated_at": int(time.time())})
  1055. print(f"[Manual Process] 回滚 {len(processing)} 条 processing → pending")
  1056. analyzing = pg_store.query('status == "analyzing"', limit=200)
  1057. for item in analyzing:
  1058. pg_store.update(item["id"], {"status": "dedup_passed", "updated_at": int(time.time())})
  1059. print(f"[Manual Process] 回滚 {len(analyzing)} 条 analyzing → dedup_passed")
  1060. asyncio.create_task(knowledge_processor.process_pending())
  1061. return {"status": "ok", "message": "处理任务已触发"}
  1062. except Exception as e:
  1063. print(f"[Manual Process] 错误: {e}")
  1064. raise HTTPException(status_code=500, detail=str(e))
  1065. @app.post("/api/knowledge/migrate")
  1066. async def migrate_knowledge_schema():
  1067. """手动触发 schema 迁移(PostgreSQL不需要此功能)"""
  1068. return {"status": "ok", "message": "PostgreSQL不需要schema迁移"}
  1069. @app.get("/api/knowledge/status/{knowledge_id}")
  1070. def get_knowledge_status(knowledge_id: str):
  1071. """查询单条知识的处理状态和关系"""
  1072. try:
  1073. result = pg_store.get_by_id(knowledge_id)
  1074. if not result:
  1075. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1076. serialized = serialize_milvus_result(result)
  1077. return {
  1078. "id": knowledge_id,
  1079. "status": serialized.get("status", "approved"),
  1080. "relationships": serialized.get("relationships", []),
  1081. "created_at": serialized.get("created_at"),
  1082. "updated_at": serialized.get("updated_at"),
  1083. }
  1084. except HTTPException:
  1085. raise
  1086. except Exception as e:
  1087. print(f"[Knowledge Status] 错误: {e}")
  1088. raise HTTPException(status_code=500, detail=str(e))
  1089. @app.get("/api/knowledge/{knowledge_id}")
  1090. def get_knowledge(knowledge_id: str):
  1091. """获取单条知识"""
  1092. try:
  1093. result = pg_store.get_by_id(knowledge_id)
  1094. if not result:
  1095. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1096. return serialize_milvus_result(result)
  1097. except HTTPException:
  1098. raise
  1099. except Exception as e:
  1100. print(f"[Get Knowledge] 错误: {e}")
  1101. raise HTTPException(status_code=500, detail=str(e))
  1102. async def _evolve_knowledge_with_llm(old_content: str, feedback: str) -> str:
  1103. """使用 LLM 进行知识进化重写"""
  1104. prompt = KNOWLEDGE_EVOLVE_PROMPT_TEMPLATE.format(
  1105. old_content=old_content,
  1106. feedback=feedback
  1107. )
  1108. try:
  1109. response = await _dedup_llm(
  1110. messages=[{"role": "user", "content": prompt}],
  1111. )
  1112. evolved = response.get("content", "").strip()
  1113. if len(evolved) < 5:
  1114. raise ValueError("LLM output too short")
  1115. return evolved
  1116. except Exception as e:
  1117. print(f"知识进化失败,采用追加模式回退: {e}")
  1118. return f"{old_content}\n\n---\n[Update {datetime.now().strftime('%Y-%m-%d')}]: {feedback}"
  1119. @app.put("/api/knowledge/{knowledge_id}")
  1120. async def update_knowledge(knowledge_id: str, update: KnowledgeUpdateIn):
  1121. """更新知识评估,支持知识进化"""
  1122. try:
  1123. # 获取现有知识
  1124. existing = pg_store.get_by_id(knowledge_id)
  1125. if not existing:
  1126. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1127. eval_data = existing.get("eval", {})
  1128. # 更新评分
  1129. if update.update_score is not None:
  1130. eval_data["score"] = update.update_score
  1131. # 添加有效案例
  1132. if update.add_helpful_case:
  1133. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1134. if "helpful_history" not in eval_data:
  1135. eval_data["helpful_history"] = []
  1136. eval_data["helpful_history"].append(update.add_helpful_case)
  1137. # 添加有害案例
  1138. if update.add_harmful_case:
  1139. eval_data["harmful"] = eval_data.get("harmful", 0) + 1
  1140. if "harmful_history" not in eval_data:
  1141. eval_data["harmful_history"] = []
  1142. eval_data["harmful_history"].append(update.add_harmful_case)
  1143. # 知识进化
  1144. content = existing["content"]
  1145. need_reembed = False
  1146. if update.evolve_feedback:
  1147. content = await _evolve_knowledge_with_llm(content, update.evolve_feedback)
  1148. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1149. need_reembed = True
  1150. # 准备更新数据
  1151. updates = {
  1152. "content": content,
  1153. "eval": eval_data,
  1154. }
  1155. # 如果内容变化,重新生成向量
  1156. if need_reembed:
  1157. embedding = await get_embedding(existing['task'])
  1158. updates["task_embedding"] = embedding
  1159. # 更新 Milvus
  1160. pg_store.update(knowledge_id, updates)
  1161. return {"status": "ok", "knowledge_id": knowledge_id}
  1162. except HTTPException:
  1163. raise
  1164. except Exception as e:
  1165. print(f"[Update Knowledge] 错误: {e}")
  1166. raise HTTPException(status_code=500, detail=str(e))
  1167. @app.patch("/api/knowledge/{knowledge_id}")
  1168. async def patch_knowledge(knowledge_id: str, patch: KnowledgePatchIn):
  1169. """直接编辑知识字段"""
  1170. try:
  1171. # 获取现有知识
  1172. existing = pg_store.get_by_id(knowledge_id)
  1173. if not existing:
  1174. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1175. updates = {}
  1176. need_reembed = False
  1177. need_content_reembed = False
  1178. if patch.task is not None:
  1179. updates["task"] = patch.task
  1180. need_reembed = True
  1181. if patch.content is not None:
  1182. updates["content"] = patch.content
  1183. need_content_reembed = True
  1184. if patch.types is not None:
  1185. updates["types"] = patch.types
  1186. if patch.tags is not None:
  1187. updates["tags"] = patch.tags
  1188. # 同时更新 tag_keys
  1189. updates["tag_keys"] = list(patch.tags.keys()) if isinstance(patch.tags, dict) else []
  1190. if patch.scopes is not None:
  1191. updates["scopes"] = patch.scopes
  1192. if patch.owner is not None:
  1193. updates["owner"] = patch.owner
  1194. if patch.support_capability is not None:
  1195. updates["support_capability"] = patch.support_capability
  1196. if patch.tools is not None:
  1197. updates["tools"] = patch.tools
  1198. if not updates:
  1199. return {"status": "ok", "knowledge_id": knowledge_id}
  1200. # 如果 task 变化,重新生成 task_embedding
  1201. if need_reembed:
  1202. task = updates.get("task", existing["task"])
  1203. embedding = await get_embedding(task)
  1204. updates["task_embedding"] = embedding
  1205. # 如果 content 变化,重新生成 content_embedding
  1206. if need_content_reembed:
  1207. content = updates.get("content", existing["content"])
  1208. content_embedding = await get_embedding(content)
  1209. updates["content_embedding"] = content_embedding
  1210. # 更新 Milvus
  1211. pg_store.update(knowledge_id, updates)
  1212. return {"status": "ok", "knowledge_id": knowledge_id}
  1213. except HTTPException:
  1214. raise
  1215. except Exception as e:
  1216. print(f"[Patch Knowledge] 错误: {e}")
  1217. raise HTTPException(status_code=500, detail=str(e))
  1218. @app.delete("/api/knowledge/{knowledge_id}")
  1219. def delete_knowledge(knowledge_id: str):
  1220. """删除单条知识"""
  1221. try:
  1222. # 检查知识是否存在
  1223. existing = pg_store.get_by_id(knowledge_id)
  1224. if not existing:
  1225. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1226. # 从 PostgreSQL 删除
  1227. pg_store.delete(knowledge_id)
  1228. print(f"[Delete Knowledge] 已删除知识: {knowledge_id}")
  1229. return {"status": "ok", "knowledge_id": knowledge_id}
  1230. except HTTPException:
  1231. raise
  1232. except Exception as e:
  1233. print(f"[Delete Knowledge] 错误: {e}")
  1234. raise HTTPException(status_code=500, detail=str(e))
  1235. @app.post("/api/knowledge/batch_delete")
  1236. def batch_delete_knowledge(knowledge_ids: List[str] = Body(...)):
  1237. """批量删除知识"""
  1238. try:
  1239. if not knowledge_ids:
  1240. raise HTTPException(status_code=400, detail="knowledge_ids cannot be empty")
  1241. # 批量删除
  1242. cursor = pg_store._get_cursor()
  1243. try:
  1244. cursor.execute(
  1245. "DELETE FROM knowledge WHERE id = ANY(%s)",
  1246. (knowledge_ids,)
  1247. )
  1248. pg_store.conn.commit()
  1249. deleted_count = cursor.rowcount
  1250. print(f"[Batch Delete] 已删除 {deleted_count} 条知识")
  1251. return {"status": "ok", "deleted_count": deleted_count}
  1252. finally:
  1253. cursor.close()
  1254. except HTTPException:
  1255. raise
  1256. except Exception as e:
  1257. print(f"[Batch Delete] 错误: {e}")
  1258. raise HTTPException(status_code=500, detail=str(e))
  1259. @app.post("/api/knowledge/batch_verify")
  1260. async def batch_verify_knowledge(batch: KnowledgeBatchVerifyIn):
  1261. """批量验证通过(approved → checked)"""
  1262. if not batch.knowledge_ids:
  1263. return {"status": "ok", "updated": 0}
  1264. try:
  1265. now_iso = datetime.now(timezone.utc).isoformat()
  1266. updated_count = 0
  1267. for kid in batch.knowledge_ids:
  1268. existing = pg_store.get_by_id(kid)
  1269. if not existing:
  1270. continue
  1271. eval_data = existing.get("eval") or {}
  1272. eval_data["verification"] = {
  1273. "status": "checked",
  1274. "verified_by": batch.verified_by,
  1275. "verified_at": now_iso,
  1276. "note": None,
  1277. "issue_type": None,
  1278. "issue_action": None,
  1279. }
  1280. pg_store.update(kid, {"eval": eval_data, "status": "checked", "updated_at": int(time.time())})
  1281. updated_count += 1
  1282. return {"status": "ok", "updated": updated_count}
  1283. except Exception as e:
  1284. print(f"[Batch Verify] 错误: {e}")
  1285. raise HTTPException(status_code=500, detail=str(e))
  1286. @app.post("/api/knowledge/{knowledge_id}/verify")
  1287. async def verify_knowledge(knowledge_id: str, verify: KnowledgeVerifyIn):
  1288. """知识验证:approve 切换 approved↔checked,reject 设为 rejected"""
  1289. try:
  1290. existing = pg_store.get_by_id(knowledge_id)
  1291. if not existing:
  1292. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1293. current_status = existing.get("status", "approved")
  1294. if verify.action == "approve":
  1295. # checked → approved(取消验证),其他 → checked
  1296. new_status = "approved" if current_status == "checked" else "checked"
  1297. pg_store.update(knowledge_id, {
  1298. "status": new_status,
  1299. "updated_at": int(time.time())
  1300. })
  1301. return {"status": "ok", "new_status": new_status,
  1302. "message": "已取消验证" if new_status == "approved" else "验证通过"}
  1303. elif verify.action == "reject":
  1304. pg_store.update(knowledge_id, {
  1305. "status": "rejected",
  1306. "updated_at": int(time.time())
  1307. })
  1308. return {"status": "ok", "new_status": "rejected", "message": "已拒绝"}
  1309. else:
  1310. raise HTTPException(status_code=400, detail=f"Unknown action: {verify.action}")
  1311. except HTTPException:
  1312. raise
  1313. except Exception as e:
  1314. print(f"[Verify Knowledge] 错误: {e}")
  1315. raise HTTPException(status_code=500, detail=str(e))
  1316. @app.post("/api/knowledge/batch_update")
  1317. async def batch_update_knowledge(batch: KnowledgeBatchUpdateIn):
  1318. """批量反馈知识有效性"""
  1319. if not batch.feedback_list:
  1320. return {"status": "ok", "updated": 0}
  1321. try:
  1322. # 先处理无需进化的,收集需要进化的
  1323. evolution_tasks = [] # [(knowledge_id, old_content, feedback, eval_data)]
  1324. simple_updates = [] # [(knowledge_id, is_effective, eval_data)]
  1325. for item in batch.feedback_list:
  1326. knowledge_id = item.get("knowledge_id")
  1327. is_effective = item.get("is_effective")
  1328. feedback = item.get("feedback", "")
  1329. if not knowledge_id:
  1330. continue
  1331. existing = pg_store.get_by_id(knowledge_id)
  1332. if not existing:
  1333. continue
  1334. eval_data = existing.get("eval", {})
  1335. if is_effective and feedback:
  1336. evolution_tasks.append((knowledge_id, existing["content"], feedback, eval_data, existing["task"]))
  1337. else:
  1338. simple_updates.append((knowledge_id, is_effective, eval_data))
  1339. # 执行简单更新
  1340. for knowledge_id, is_effective, eval_data in simple_updates:
  1341. if is_effective:
  1342. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1343. else:
  1344. eval_data["harmful"] = eval_data.get("harmful", 0) + 1
  1345. pg_store.update(knowledge_id, {"eval": eval_data})
  1346. # 并发执行知识进化
  1347. if evolution_tasks:
  1348. print(f"🧬 并发处理 {len(evolution_tasks)} 条知识进化...")
  1349. evolved_results = await asyncio.gather(
  1350. *[_evolve_knowledge_with_llm(old, fb) for _, old, fb, _, _ in evolution_tasks]
  1351. )
  1352. for (knowledge_id, _, _, eval_data, task), evolved_content in zip(evolution_tasks, evolved_results):
  1353. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1354. # 重新生成向量(只基于 task)
  1355. embedding = await get_embedding(task)
  1356. pg_store.update(knowledge_id, {
  1357. "content": evolved_content,
  1358. "eval": eval_data,
  1359. "task_embedding": embedding
  1360. })
  1361. return {"status": "ok", "updated": len(simple_updates) + len(evolution_tasks)}
  1362. except Exception as e:
  1363. print(f"[Batch Update] 错误: {e}")
  1364. raise HTTPException(status_code=500, detail=str(e))
  1365. @app.post("/api/knowledge/slim")
  1366. async def slim_knowledge(model: str = "google/gemini-2.5-flash-lite"):
  1367. """知识库瘦身:合并语义相似知识"""
  1368. try:
  1369. # 获取所有知识
  1370. all_knowledge = pg_store.query('id != ""', limit=10000)
  1371. # 转换为可序列化的格式
  1372. all_knowledge = [serialize_milvus_result(item) for item in all_knowledge]
  1373. if len(all_knowledge) < 2:
  1374. return {"status": "ok", "message": f"知识库仅有 {len(all_knowledge)} 条,无需瘦身"}
  1375. # 构造发给大模型的内容
  1376. entries_text = ""
  1377. for item in all_knowledge:
  1378. eval_data = item.get("eval", {})
  1379. types = item.get("types", [])
  1380. entries_text += f"[ID: {item['id']}] [Types: {','.join(types)}] "
  1381. entries_text += f"[Helpful: {eval_data.get('helpful', 0)}, Harmful: {eval_data.get('harmful', 0)}] [Score: {eval_data.get('score', 3)}]\n"
  1382. entries_text += f"Task: {item['task']}\n"
  1383. entries_text += f"Content: {item['content'][:200]}...\n\n"
  1384. prompt = KNOWLEDGE_SLIM_PROMPT_TEMPLATE.format(entries_text=entries_text)
  1385. print(f"\n[知识瘦身] 正在调用 {model} 分析 {len(all_knowledge)} 条知识...")
  1386. slim_llm = create_openrouter_llm_call(model=model)
  1387. response = await slim_llm(
  1388. messages=[{"role": "user", "content": prompt}],
  1389. )
  1390. content = response.get("content", "").strip()
  1391. if not content:
  1392. raise HTTPException(status_code=500, detail="LLM 返回为空")
  1393. # 解析大模型输出
  1394. report_line = ""
  1395. new_entries = []
  1396. blocks = [b.strip() for b in content.split("===") if b.strip()]
  1397. for block in blocks:
  1398. if block.startswith("REPORT:"):
  1399. report_line = block
  1400. continue
  1401. lines = block.split("\n")
  1402. kid, types, helpful, harmful, score, task, content_lines = None, [], 0, 0, 3, "", []
  1403. current_field = None
  1404. for line in lines:
  1405. if line.startswith("ID:"):
  1406. kid = line[3:].strip()
  1407. current_field = None
  1408. elif line.startswith("TYPES:"):
  1409. types_str = line[6:].strip()
  1410. types = [t.strip() for t in types_str.split(",") if t.strip()]
  1411. current_field = None
  1412. elif line.startswith("HELPFUL:"):
  1413. try:
  1414. helpful = int(line[8:].strip())
  1415. except Exception:
  1416. helpful = 0
  1417. current_field = None
  1418. elif line.startswith("HARMFUL:"):
  1419. try:
  1420. harmful = int(line[8:].strip())
  1421. except Exception:
  1422. harmful = 0
  1423. current_field = None
  1424. elif line.startswith("SCORE:"):
  1425. try:
  1426. score = int(line[6:].strip())
  1427. except Exception:
  1428. score = 3
  1429. current_field = None
  1430. elif line.startswith("TASK:"):
  1431. task = line[5:].strip()
  1432. current_field = "task"
  1433. elif line.startswith("CONTENT:"):
  1434. content_lines.append(line[8:].strip())
  1435. current_field = "content"
  1436. elif current_field == "task":
  1437. task += "\n" + line
  1438. elif current_field == "content":
  1439. content_lines.append(line)
  1440. if kid and content_lines:
  1441. new_entries.append({
  1442. "id": kid,
  1443. "types": types if types else ["strategy"],
  1444. "helpful": helpful,
  1445. "harmful": harmful,
  1446. "score": score,
  1447. "task": task.strip(),
  1448. "content": "\n".join(content_lines).strip()
  1449. })
  1450. if not new_entries:
  1451. raise HTTPException(status_code=500, detail="解析大模型输出失败")
  1452. # 生成向量并重建知识库
  1453. print(f"[知识瘦身] 正在为 {len(new_entries)} 条知识生成向量...")
  1454. # 批量生成向量(只基于 task)
  1455. texts = [e['task'] for e in new_entries]
  1456. embeddings = await get_embeddings_batch(texts)
  1457. # 清空并重建(PostgreSQL使用TRUNCATE)
  1458. cursor = pg_store._get_cursor()
  1459. try:
  1460. cursor.execute("TRUNCATE TABLE knowledge")
  1461. pg_store.conn.commit()
  1462. finally:
  1463. cursor.close()
  1464. knowledge_list = []
  1465. for e, embedding in zip(new_entries, embeddings):
  1466. eval_data = {
  1467. "score": e["score"],
  1468. "helpful": e["helpful"],
  1469. "harmful": e["harmful"],
  1470. "confidence": 0.9,
  1471. "helpful_history": [],
  1472. "harmful_history": []
  1473. }
  1474. source = {
  1475. "name": "slim",
  1476. "category": "exp",
  1477. "urls": [],
  1478. "agent_id": "slim",
  1479. "submitted_by": "system",
  1480. "timestamp": datetime.now(timezone.utc).isoformat()
  1481. }
  1482. knowledge_list.append({
  1483. "id": e["id"],
  1484. "task_embedding": embedding,
  1485. "message_id": "",
  1486. "task": e["task"],
  1487. "content": e["content"],
  1488. "types": e["types"],
  1489. "tags": {},
  1490. "tag_keys": [],
  1491. "scopes": ["org:cybertogether"],
  1492. "owner": "agent:slim",
  1493. "resource_ids": [],
  1494. "source": source,
  1495. "eval": eval_data,
  1496. "created_at": now,
  1497. "updated_at": now,
  1498. "status": "approved",
  1499. "relationships": json.dumps([])
  1500. })
  1501. pg_store.insert_batch(knowledge_list)
  1502. result_msg = f"瘦身完成:{len(all_knowledge)} → {len(new_entries)} 条知识"
  1503. if report_line:
  1504. result_msg += f"\n{report_line}"
  1505. print(f"[知识瘦身] {result_msg}")
  1506. return {"status": "ok", "before": len(all_knowledge), "after": len(new_entries), "report": report_line}
  1507. except HTTPException:
  1508. raise
  1509. except Exception as e:
  1510. print(f"[Slim Knowledge] 错误: {e}")
  1511. raise HTTPException(status_code=500, detail=str(e))
  1512. @app.post("/api/extract")
  1513. async def extract_knowledge_from_messages(extract_req: MessageExtractIn, background_tasks: BackgroundTasks):
  1514. """从消息历史中提取知识(LLM 分析)"""
  1515. if not extract_req.submitted_by:
  1516. raise HTTPException(status_code=400, detail="submitted_by is required")
  1517. messages = extract_req.messages
  1518. if not messages or len(messages) == 0:
  1519. return {"status": "ok", "extracted_count": 0, "knowledge_ids": []}
  1520. # 构造消息历史文本
  1521. messages_text = ""
  1522. for msg in messages:
  1523. role = msg.get("role", "unknown")
  1524. content = msg.get("content", "")
  1525. messages_text += f"[{role}]: {content}\n\n"
  1526. # LLM 提取知识
  1527. prompt = MESSAGE_EXTRACT_PROMPT_TEMPLATE.format(messages_text=messages_text)
  1528. try:
  1529. print(f"\n[Extract] 正在从 {len(messages)} 条消息中提取知识...")
  1530. response = await _dedup_llm(
  1531. messages=[{"role": "user", "content": prompt}],
  1532. )
  1533. content = response.get("content", "").strip()
  1534. # 尝试解析 JSON
  1535. # 移除可能的 markdown 代码块标记
  1536. if content.startswith("```json"):
  1537. content = content[7:]
  1538. if content.startswith("```"):
  1539. content = content[3:]
  1540. if content.endswith("```"):
  1541. content = content[:-3]
  1542. content = content.strip()
  1543. extracted_knowledge = json.loads(content)
  1544. if not isinstance(extracted_knowledge, list):
  1545. raise ValueError("LLM output is not a list")
  1546. if not extracted_knowledge:
  1547. return {"status": "ok", "extracted_count": 0, "knowledge_ids": []}
  1548. # 批量生成向量(只基于 task)
  1549. texts = [item.get('task', '') for item in extracted_knowledge]
  1550. embeddings = await get_embeddings_batch(texts)
  1551. # 保存提取的知识
  1552. knowledge_ids = []
  1553. now = int(time.time())
  1554. knowledge_list = []
  1555. for item, embedding in zip(extracted_knowledge, embeddings):
  1556. task = item.get("task", "")
  1557. knowledge_content = item.get("content", "")
  1558. types = item.get("types", ["strategy"])
  1559. score = item.get("score", 3)
  1560. if not task or not knowledge_content:
  1561. continue
  1562. # 生成 ID
  1563. timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
  1564. random_suffix = uuid.uuid4().hex[:4]
  1565. knowledge_id = f"knowledge-{timestamp}-{random_suffix}"
  1566. # 准备数据
  1567. source = {
  1568. "name": "message_extraction",
  1569. "category": "exp",
  1570. "urls": [],
  1571. "agent_id": extract_req.agent_id,
  1572. "submitted_by": extract_req.submitted_by,
  1573. "timestamp": datetime.now(timezone.utc).isoformat(),
  1574. "session_key": extract_req.session_key
  1575. }
  1576. eval_data = {
  1577. "score": score,
  1578. "helpful": 1,
  1579. "harmful": 0,
  1580. "confidence": 0.7,
  1581. "helpful_history": [],
  1582. "harmful_history": []
  1583. }
  1584. knowledge_list.append({
  1585. "id": knowledge_id,
  1586. "task_embedding": embedding,
  1587. "message_id": "",
  1588. "task": task,
  1589. "content": knowledge_content,
  1590. "types": types,
  1591. "tags": {},
  1592. "tag_keys": [],
  1593. "scopes": ["org:cybertogether"],
  1594. "owner": extract_req.submitted_by,
  1595. "resource_ids": [],
  1596. "source": source,
  1597. "eval": eval_data,
  1598. "created_at": now,
  1599. "updated_at": now,
  1600. "status": "pending",
  1601. "relationships": json.dumps([]),
  1602. })
  1603. knowledge_ids.append(knowledge_id)
  1604. # 批量插入
  1605. if knowledge_list:
  1606. pg_store.insert_batch(knowledge_list)
  1607. background_tasks.add_task(knowledge_processor.process_pending)
  1608. print(f"[Extract] 成功提取并保存 {len(knowledge_ids)} 条知识")
  1609. return {
  1610. "status": "ok",
  1611. "extracted_count": len(knowledge_ids),
  1612. "knowledge_ids": knowledge_ids
  1613. }
  1614. except json.JSONDecodeError as e:
  1615. print(f"[Extract] JSON 解析失败: {e}")
  1616. print(f"[Extract] LLM 输出: {content[:500]}")
  1617. return {"status": "error", "error": "Failed to parse LLM output", "extracted_count": 0}
  1618. except Exception as e:
  1619. print(f"[Extract] 提取失败: {e}")
  1620. return {"status": "error", "error": str(e), "extracted_count": 0}
  1621. # ===== Tool API =====
  1622. @app.post("/api/tool", status_code=201)
  1623. async def create_tool(tool: ToolIn):
  1624. """创建或更新工具"""
  1625. try:
  1626. now = int(time.time())
  1627. embedding = await get_embedding(f"{tool.name} {tool.introduction}")
  1628. pg_tool_store.insert_or_update({
  1629. 'id': tool.id,
  1630. 'name': tool.name,
  1631. 'version': tool.version,
  1632. 'introduction': tool.introduction,
  1633. 'tutorial': tool.tutorial,
  1634. 'input': tool.input,
  1635. 'output': tool.output,
  1636. 'updated_time': now,
  1637. 'status': tool.status,
  1638. 'capabilities': tool.capabilities,
  1639. 'tool_knowledge': tool.tool_knowledge,
  1640. 'case_knowledge': tool.case_knowledge,
  1641. 'process_knowledge': tool.process_knowledge,
  1642. 'embedding': embedding,
  1643. })
  1644. return {"status": "ok", "id": tool.id}
  1645. except Exception as e:
  1646. raise HTTPException(status_code=500, detail=str(e))
  1647. @app.get("/api/tool")
  1648. def list_tools(
  1649. status: Optional[str] = Query(None),
  1650. limit: int = Query(100, ge=1, le=1000),
  1651. offset: int = Query(0, ge=0),
  1652. ):
  1653. """列出工具"""
  1654. try:
  1655. results = pg_tool_store.list_all(limit=limit, offset=offset, status=status)
  1656. total = pg_tool_store.count(status=status)
  1657. return {"results": results, "total": total}
  1658. except Exception as e:
  1659. raise HTTPException(status_code=500, detail=str(e))
  1660. @app.get("/api/tool/search")
  1661. async def search_tools(
  1662. q: str = Query(..., description="查询文本"),
  1663. top_k: int = Query(5, ge=1, le=20),
  1664. status: Optional[str] = None,
  1665. ):
  1666. """向量检索工具"""
  1667. try:
  1668. query_embedding = await get_embedding(q)
  1669. results = pg_tool_store.search(query_embedding, limit=top_k, status=status)
  1670. return {"results": results, "count": len(results)}
  1671. except Exception as e:
  1672. raise HTTPException(status_code=500, detail=str(e))
  1673. @app.get("/api/tool/{tool_id:path}")
  1674. def get_tool(tool_id: str):
  1675. """获取单个工具详情"""
  1676. try:
  1677. result = pg_tool_store.get_by_id(tool_id)
  1678. if not result:
  1679. raise HTTPException(status_code=404, detail=f"Tool not found: {tool_id}")
  1680. return result
  1681. except HTTPException:
  1682. raise
  1683. except Exception as e:
  1684. raise HTTPException(status_code=500, detail=str(e))
  1685. @app.patch("/api/tool/{tool_id:path}")
  1686. async def patch_tool(tool_id: str, patch: ToolPatchIn):
  1687. """更新工具字段"""
  1688. try:
  1689. if not pg_tool_store.get_by_id(tool_id):
  1690. raise HTTPException(status_code=404, detail=f"Tool not found: {tool_id}")
  1691. updates = {}
  1692. need_reembed = False
  1693. for field in ('name', 'version', 'introduction', 'tutorial', 'input', 'output',
  1694. 'status', 'capabilities', 'tool_knowledge', 'case_knowledge', 'process_knowledge'):
  1695. value = getattr(patch, field)
  1696. if value is not None:
  1697. updates[field] = value
  1698. if field in ('name', 'introduction'):
  1699. need_reembed = True
  1700. if not updates:
  1701. return {"status": "ok", "id": tool_id}
  1702. updates['updated_time'] = int(time.time())
  1703. if need_reembed:
  1704. existing = pg_tool_store.get_by_id(tool_id)
  1705. name = updates.get('name', existing['name'])
  1706. intro = updates.get('introduction', existing['introduction'])
  1707. updates['embedding'] = await get_embedding(f"{name} {intro}")
  1708. pg_tool_store.update(tool_id, updates)
  1709. return {"status": "ok", "id": tool_id}
  1710. except HTTPException:
  1711. raise
  1712. except Exception as e:
  1713. raise HTTPException(status_code=500, detail=str(e))
  1714. @app.delete("/api/tool/{tool_id:path}")
  1715. def delete_tool(tool_id: str):
  1716. """删除工具"""
  1717. try:
  1718. if not pg_tool_store.get_by_id(tool_id):
  1719. raise HTTPException(status_code=404, detail=f"Tool not found: {tool_id}")
  1720. pg_tool_store.delete(tool_id)
  1721. return {"status": "ok", "id": tool_id}
  1722. except HTTPException:
  1723. raise
  1724. except Exception as e:
  1725. raise HTTPException(status_code=500, detail=str(e))
  1726. # ===== Capability API =====
  1727. @app.post("/api/capability", status_code=201)
  1728. async def create_capability(cap: CapabilityIn):
  1729. """创建或更新原子能力"""
  1730. try:
  1731. embedding = await get_embedding(f"{cap.name} {cap.description}")
  1732. pg_capability_store.insert_or_update({
  1733. 'id': cap.id,
  1734. 'name': cap.name,
  1735. 'criterion': cap.criterion,
  1736. 'description': cap.description,
  1737. 'requirements': cap.requirements,
  1738. 'implements': cap.implements,
  1739. 'tools': cap.tools,
  1740. 'source_knowledge': cap.source_knowledge,
  1741. 'embedding': embedding,
  1742. })
  1743. return {"status": "ok", "id": cap.id}
  1744. except Exception as e:
  1745. raise HTTPException(status_code=500, detail=str(e))
  1746. @app.get("/api/capability")
  1747. def list_capabilities(
  1748. limit: int = Query(100, ge=1, le=1000),
  1749. offset: int = Query(0, ge=0),
  1750. ):
  1751. """列出原子能力"""
  1752. try:
  1753. results = pg_capability_store.list_all(limit=limit, offset=offset)
  1754. total = pg_capability_store.count()
  1755. return {"results": results, "total": total}
  1756. except Exception as e:
  1757. raise HTTPException(status_code=500, detail=str(e))
  1758. @app.get("/api/capability/search")
  1759. async def search_capabilities(
  1760. q: str = Query(..., description="查询文本"),
  1761. top_k: int = Query(5, ge=1, le=20),
  1762. ):
  1763. """向量检索原子能力"""
  1764. try:
  1765. query_embedding = await get_embedding(q)
  1766. results = pg_capability_store.search(query_embedding, limit=top_k)
  1767. return {"results": results, "count": len(results)}
  1768. except Exception as e:
  1769. raise HTTPException(status_code=500, detail=str(e))
  1770. @app.get("/api/capability/{cap_id}")
  1771. def get_capability(cap_id: str):
  1772. """获取单个原子能力"""
  1773. try:
  1774. result = pg_capability_store.get_by_id(cap_id)
  1775. if not result:
  1776. raise HTTPException(status_code=404, detail=f"Capability not found: {cap_id}")
  1777. return result
  1778. except HTTPException:
  1779. raise
  1780. except Exception as e:
  1781. raise HTTPException(status_code=500, detail=str(e))
  1782. @app.patch("/api/capability/{cap_id}")
  1783. async def patch_capability(cap_id: str, patch: CapabilityPatchIn):
  1784. """更新原子能力字段"""
  1785. try:
  1786. existing = pg_capability_store.get_by_id(cap_id)
  1787. if not existing:
  1788. raise HTTPException(status_code=404, detail=f"Capability not found: {cap_id}")
  1789. updates = {}
  1790. need_reembed = False
  1791. for field in ('name', 'criterion', 'description', 'requirements',
  1792. 'implements', 'tools', 'source_knowledge'):
  1793. value = getattr(patch, field)
  1794. if value is not None:
  1795. updates[field] = value
  1796. if field in ('name', 'description'):
  1797. need_reembed = True
  1798. if not updates:
  1799. return {"status": "ok", "id": cap_id}
  1800. if need_reembed:
  1801. name = updates.get('name', existing['name'])
  1802. desc = updates.get('description', existing['description'])
  1803. updates['embedding'] = await get_embedding(f"{name} {desc}")
  1804. pg_capability_store.update(cap_id, updates)
  1805. return {"status": "ok", "id": cap_id}
  1806. except HTTPException:
  1807. raise
  1808. except Exception as e:
  1809. raise HTTPException(status_code=500, detail=str(e))
  1810. @app.delete("/api/capability/{cap_id}")
  1811. def delete_capability(cap_id: str):
  1812. """删除原子能力"""
  1813. try:
  1814. if not pg_capability_store.get_by_id(cap_id):
  1815. raise HTTPException(status_code=404, detail=f"Capability not found: {cap_id}")
  1816. pg_capability_store.delete(cap_id)
  1817. return {"status": "ok", "id": cap_id}
  1818. except HTTPException:
  1819. raise
  1820. except Exception as e:
  1821. raise HTTPException(status_code=500, detail=str(e))
  1822. # ===== Requirement API =====
  1823. @app.post("/api/requirement", status_code=201)
  1824. async def create_requirement(req: RequirementIn):
  1825. """创建或更新需求"""
  1826. try:
  1827. embedding = await get_embedding(req.description)
  1828. pg_requirement_store.insert_or_update({
  1829. 'id': req.id,
  1830. 'description': req.description,
  1831. 'atomics': req.atomics,
  1832. 'source_nodes': req.source_nodes,
  1833. 'status': req.status,
  1834. 'match_result': req.match_result,
  1835. 'embedding': embedding,
  1836. })
  1837. return {"status": "ok", "id": req.id}
  1838. except Exception as e:
  1839. raise HTTPException(status_code=500, detail=str(e))
  1840. @app.get("/api/requirement")
  1841. def list_requirements(
  1842. status: Optional[str] = Query(None),
  1843. limit: int = Query(100, ge=1, le=1000),
  1844. offset: int = Query(0, ge=0),
  1845. ):
  1846. """列出需求"""
  1847. try:
  1848. results = pg_requirement_store.list_all(limit=limit, offset=offset, status=status)
  1849. total = pg_requirement_store.count(status=status)
  1850. return {"results": results, "total": total}
  1851. except Exception as e:
  1852. raise HTTPException(status_code=500, detail=str(e))
  1853. @app.get("/api/requirement/search")
  1854. async def search_requirements(
  1855. q: str = Query(..., description="查询文本"),
  1856. top_k: int = Query(5, ge=1, le=20),
  1857. ):
  1858. """向量检索需求"""
  1859. try:
  1860. query_embedding = await get_embedding(q)
  1861. results = pg_requirement_store.search(query_embedding, limit=top_k)
  1862. return {"results": results, "count": len(results)}
  1863. except Exception as e:
  1864. raise HTTPException(status_code=500, detail=str(e))
  1865. @app.get("/api/requirement/{req_id}")
  1866. def get_requirement(req_id: str):
  1867. """获取单个需求"""
  1868. try:
  1869. result = pg_requirement_store.get_by_id(req_id)
  1870. if not result:
  1871. raise HTTPException(status_code=404, detail=f"Requirement not found: {req_id}")
  1872. return result
  1873. except HTTPException:
  1874. raise
  1875. except Exception as e:
  1876. raise HTTPException(status_code=500, detail=str(e))
  1877. @app.patch("/api/requirement/{req_id}")
  1878. async def patch_requirement(req_id: str, patch: RequirementPatchIn):
  1879. """更新需求字段"""
  1880. try:
  1881. existing = pg_requirement_store.get_by_id(req_id)
  1882. if not existing:
  1883. raise HTTPException(status_code=404, detail=f"Requirement not found: {req_id}")
  1884. updates = {}
  1885. need_reembed = False
  1886. for field in ('description', 'atomics', 'source_nodes', 'status', 'match_result'):
  1887. value = getattr(patch, field)
  1888. if value is not None:
  1889. updates[field] = value
  1890. if field == 'description':
  1891. need_reembed = True
  1892. if not updates:
  1893. return {"status": "ok", "id": req_id}
  1894. if need_reembed:
  1895. updates['embedding'] = await get_embedding(updates['description'])
  1896. pg_requirement_store.update(req_id, updates)
  1897. return {"status": "ok", "id": req_id}
  1898. except HTTPException:
  1899. raise
  1900. except Exception as e:
  1901. raise HTTPException(status_code=500, detail=str(e))
  1902. @app.delete("/api/requirement/{req_id}")
  1903. def delete_requirement(req_id: str):
  1904. """删除需求"""
  1905. try:
  1906. if not pg_requirement_store.get_by_id(req_id):
  1907. raise HTTPException(status_code=404, detail=f"Requirement not found: {req_id}")
  1908. pg_requirement_store.delete(req_id)
  1909. return {"status": "ok", "id": req_id}
  1910. except HTTPException:
  1911. raise
  1912. except Exception as e:
  1913. raise HTTPException(status_code=500, detail=str(e))
  1914. @app.get("/")
  1915. def frontend():
  1916. """KnowHub 管理前端"""
  1917. index_file = STATIC_DIR / "index.html"
  1918. if not index_file.exists():
  1919. return HTMLResponse("<h1>KnowHub Frontend Not Found</h1><p>Please ensure knowhub/frontend/dist/index.html exists. Run 'yarn build' in frontend directory.</p>", status_code=404)
  1920. return FileResponse(str(index_file))
  1921. @app.get("/category_tree.json")
  1922. def serve_category_tree():
  1923. """类别树JSON数据"""
  1924. tree_file = STATIC_DIR / "category_tree.json"
  1925. if not tree_file.exists():
  1926. return {"error": "Not Found"}
  1927. return FileResponse(str(tree_file))
  1928. if __name__ == "__main__":
  1929. import uvicorn
  1930. uvicorn.run(app, host="0.0.0.0", port=9999)