server.py 82 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330
  1. """
  2. KnowHub Server
  3. Agent 工具使用经验的共享平台。
  4. FastAPI + Milvus Lite(知识)+ SQLite(资源),单文件部署。
  5. """
  6. import os
  7. import re
  8. import json
  9. import asyncio
  10. import base64
  11. import time
  12. import uuid
  13. from contextlib import asynccontextmanager
  14. from datetime import datetime, timezone
  15. from typing import Optional, List, Dict
  16. from pathlib import Path
  17. from cryptography.hazmat.primitives.ciphers.aead import AESGCM
  18. from fastapi import FastAPI, HTTPException, Query, Header, Body, BackgroundTasks
  19. from fastapi.responses import HTMLResponse, FileResponse
  20. from fastapi.staticfiles import StaticFiles
  21. from pydantic import BaseModel, Field
  22. # 导入 LLM 调用(需要 agent 模块在 Python path 中)
  23. import sys
  24. sys.path.insert(0, str(Path(__file__).parent.parent))
  25. # 加载环境变量
  26. from dotenv import load_dotenv
  27. load_dotenv(Path(__file__).parent.parent / ".env")
  28. from agent.llm import create_openrouter_llm_call, create_qwen_llm_call
  29. from knowhub.kb_manage_prompts import (
  30. DEDUP_RELATION_PROMPT,
  31. TOOL_ANALYSIS_PROMPT,
  32. RERANK_PROMPT_TEMPLATE,
  33. KNOWLEDGE_EVOLVE_PROMPT_TEMPLATE,
  34. KNOWLEDGE_SLIM_PROMPT_TEMPLATE,
  35. MESSAGE_EXTRACT_PROMPT_TEMPLATE,
  36. )
  37. _dedup_llm = create_openrouter_llm_call(model="google/gemini-2.5-flash-lite")
  38. _tool_analysis_llm = create_qwen_llm_call(model="qwen3.5-plus")
  39. # 导入向量存储和 embedding
  40. from knowhub.knowhub_db.pg_store import PostgreSQLStore
  41. from knowhub.knowhub_db.pg_resource_store import PostgreSQLResourceStore
  42. from knowhub.knowhub_db.pg_tool_store import PostgreSQLToolStore
  43. from knowhub.knowhub_db.pg_capability_store import PostgreSQLCapabilityStore
  44. from knowhub.knowhub_db.pg_requirement_store import PostgreSQLRequirementStore
  45. from knowhub.embeddings import get_embedding, get_embeddings_batch
  46. BRAND_NAME = os.getenv("BRAND_NAME", "KnowHub")
  47. BRAND_API_ENV = os.getenv("BRAND_API_ENV", "KNOWHUB_API")
  48. BRAND_DB = os.getenv("BRAND_DB", "knowhub.db")
  49. # 组织密钥配置(格式:org1:key1_base64,org2:key2_base64)
  50. ORG_KEYS_RAW = os.getenv("ORG_KEYS", "")
  51. ORG_KEYS = {}
  52. if ORG_KEYS_RAW:
  53. for pair in ORG_KEYS_RAW.split(","):
  54. if ":" in pair:
  55. org, key_b64 = pair.split(":", 1)
  56. ORG_KEYS[org.strip()] = key_b64.strip()
  57. DB_PATH = Path(__file__).parent / BRAND_DB
  58. # 全局 PostgreSQL 存储实例
  59. pg_store: Optional[PostgreSQLStore] = None
  60. pg_resource_store: Optional[PostgreSQLResourceStore] = None
  61. pg_tool_store: Optional[PostgreSQLToolStore] = None
  62. pg_capability_store: Optional[PostgreSQLCapabilityStore] = None
  63. pg_requirement_store: Optional[PostgreSQLRequirementStore] = None
  64. # --- 加密/解密 ---
  65. def get_org_key(resource_id: str) -> Optional[bytes]:
  66. """从content_id提取组织前缀,返回对应密钥"""
  67. if "/" in resource_id:
  68. org = resource_id.split("/")[0]
  69. if org in ORG_KEYS:
  70. return base64.b64decode(ORG_KEYS[org])
  71. return None
  72. def encrypt_content(resource_id: str, plaintext: str) -> str:
  73. """加密内容,返回格式:encrypted:AES256-GCM:{base64_data}"""
  74. if not plaintext:
  75. return ""
  76. key = get_org_key(resource_id)
  77. if not key:
  78. # 没有配置密钥,明文存储(不推荐)
  79. return plaintext
  80. aesgcm = AESGCM(key)
  81. nonce = os.urandom(12) # 96-bit nonce
  82. ciphertext = aesgcm.encrypt(nonce, plaintext.encode("utf-8"), None)
  83. # 组合 nonce + ciphertext
  84. encrypted_data = nonce + ciphertext
  85. encoded = base64.b64encode(encrypted_data).decode("ascii")
  86. return f"encrypted:AES256-GCM:{encoded}"
  87. def decrypt_content(resource_id: str, encrypted_text: str, provided_key: Optional[str] = None) -> str:
  88. """解密内容,如果没有提供密钥或密钥错误,返回[ENCRYPTED]"""
  89. if not encrypted_text:
  90. return ""
  91. if not encrypted_text.startswith("encrypted:AES256-GCM:"):
  92. # 未加密的内容,直接返回
  93. return encrypted_text
  94. # 提取加密数据
  95. encoded = encrypted_text.split(":", 2)[2]
  96. encrypted_data = base64.b64decode(encoded)
  97. nonce = encrypted_data[:12]
  98. ciphertext = encrypted_data[12:]
  99. # 获取密钥
  100. key = None
  101. if provided_key:
  102. # 使用提供的密钥
  103. try:
  104. key = base64.b64decode(provided_key)
  105. except Exception:
  106. return "[ENCRYPTED]"
  107. else:
  108. # 从配置中获取
  109. key = get_org_key(resource_id)
  110. if not key:
  111. return "[ENCRYPTED]"
  112. try:
  113. aesgcm = AESGCM(key)
  114. plaintext = aesgcm.decrypt(nonce, ciphertext, None)
  115. return plaintext.decode("utf-8")
  116. except Exception:
  117. return "[ENCRYPTED]"
  118. def serialize_milvus_result(data):
  119. """将 Milvus 返回的数据转换为可序列化的字典"""
  120. # 基本类型直接返回
  121. if data is None or isinstance(data, (str, int, float, bool)):
  122. return data
  123. # 字典类型递归处理
  124. if isinstance(data, dict):
  125. return {k: serialize_milvus_result(v) for k, v in data.items()}
  126. # 列表/元组类型递归处理
  127. if isinstance(data, (list, tuple)):
  128. return [serialize_milvus_result(item) for item in data]
  129. # 尝试转换为字典(对于有 to_dict 方法的对象)
  130. if hasattr(data, 'to_dict') and callable(getattr(data, 'to_dict')):
  131. try:
  132. return serialize_milvus_result(data.to_dict())
  133. except:
  134. pass
  135. # 尝试转换为列表(对于可迭代对象,如 RepeatedScalarContainer)
  136. if hasattr(data, '__iter__') and not isinstance(data, (str, bytes, dict)):
  137. try:
  138. # 强制转换为列表并递归处理
  139. result = []
  140. for item in data:
  141. result.append(serialize_milvus_result(item))
  142. return result
  143. except:
  144. pass
  145. # 尝试获取对象的属性字典
  146. if hasattr(data, '__dict__'):
  147. try:
  148. return serialize_milvus_result(vars(data))
  149. except:
  150. pass
  151. # 最后的 fallback:对于无法处理的类型,返回 None 而不是字符串表示
  152. # 这样可以避免产生无法序列化的字符串
  153. return None
  154. # --- Models ---
  155. class ResourceIn(BaseModel):
  156. id: str
  157. title: str = ""
  158. body: str
  159. secure_body: str = ""
  160. content_type: str = "text" # text|code|credential|cookie
  161. metadata: dict = {}
  162. sort_order: int = 0
  163. submitted_by: str = ""
  164. class ResourcePatchIn(BaseModel):
  165. """PATCH /api/resource/{id} 请求体"""
  166. title: Optional[str] = None
  167. body: Optional[str] = None
  168. secure_body: Optional[str] = None
  169. content_type: Optional[str] = None
  170. metadata: Optional[dict] = None
  171. # Knowledge Models
  172. class KnowledgeIn(BaseModel):
  173. task: str
  174. content: str
  175. types: list[str] = ["strategy"]
  176. tags: dict = {}
  177. scopes: list[str] = ["org:cybertogether"]
  178. owner: str = ""
  179. message_id: str = ""
  180. resource_ids: list[str] = []
  181. source: dict = {} # {name, category, urls, agent_id, submitted_by, timestamp}
  182. eval: dict = {} # {score, helpful, harmful, confidence}
  183. support_capability: list[str] = []
  184. tools: list[str] = []
  185. class KnowledgeOut(BaseModel):
  186. id: str
  187. message_id: str
  188. types: list[str]
  189. task: str
  190. tags: dict
  191. scopes: list[str]
  192. owner: str
  193. content: str
  194. resource_ids: list[str]
  195. source: dict
  196. eval: dict
  197. created_at: str
  198. updated_at: str
  199. class KnowledgeUpdateIn(BaseModel):
  200. add_helpful_case: Optional[dict] = None
  201. add_harmful_case: Optional[dict] = None
  202. update_score: Optional[int] = Field(default=None, ge=1, le=5)
  203. evolve_feedback: Optional[str] = None
  204. class KnowledgePatchIn(BaseModel):
  205. """PATCH /api/knowledge/{id} 请求体(直接字段编辑)"""
  206. task: Optional[str] = None
  207. content: Optional[str] = None
  208. types: Optional[list[str]] = None
  209. tags: Optional[dict] = None
  210. scopes: Optional[list[str]] = None
  211. owner: Optional[str] = None
  212. support_capability: Optional[list[str]] = None
  213. tools: Optional[list[str]] = None
  214. class MessageExtractIn(BaseModel):
  215. """POST /api/extract 请求体(消息历史提取)"""
  216. messages: list[dict] # [{role: str, content: str}, ...]
  217. agent_id: str = "unknown"
  218. submitted_by: str # 必填,作为 owner
  219. session_key: str = ""
  220. class KnowledgeBatchUpdateIn(BaseModel):
  221. feedback_list: list[dict]
  222. class KnowledgeVerifyIn(BaseModel):
  223. action: str # "approve" | "reject"
  224. verified_by: str = "user"
  225. class KnowledgeBatchVerifyIn(BaseModel):
  226. knowledge_ids: List[str]
  227. action: str # "approve"
  228. verified_by: str
  229. class KnowledgeSearchResponse(BaseModel):
  230. results: list[dict]
  231. count: int
  232. # --- Tool Models ---
  233. class ToolIn(BaseModel):
  234. id: str
  235. name: str = ""
  236. version: Optional[str] = None
  237. introduction: str = ""
  238. tutorial: str = ""
  239. input: dict | str = ""
  240. output: dict | str = ""
  241. status: str = "未接入"
  242. capabilities: list[str] = []
  243. tool_knowledge: list[str] = []
  244. case_knowledge: list[str] = []
  245. process_knowledge: list[str] = []
  246. implemented_tool_ids: list[str] = []
  247. class ToolPatchIn(BaseModel):
  248. name: Optional[str] = None
  249. version: Optional[str] = None
  250. introduction: Optional[str] = None
  251. tutorial: Optional[str] = None
  252. input: Optional[dict | str] = None
  253. output: Optional[dict | str] = None
  254. status: Optional[str] = None
  255. capabilities: Optional[list[str]] = None
  256. tool_knowledge: Optional[list[str]] = None
  257. case_knowledge: Optional[list[str]] = None
  258. process_knowledge: Optional[list[str]] = None
  259. implemented_tool_ids: Optional[list[str]] = None
  260. # --- Capability Models ---
  261. class CapabilityIn(BaseModel):
  262. id: str
  263. name: str = ""
  264. criterion: str = ""
  265. description: str = ""
  266. requirements: list[str] = []
  267. implements: dict = {}
  268. tools: list[str] = []
  269. source_knowledge: list[str] = []
  270. class CapabilityPatchIn(BaseModel):
  271. name: Optional[str] = None
  272. criterion: Optional[str] = None
  273. description: Optional[str] = None
  274. requirements: Optional[list[str]] = None
  275. implements: Optional[dict] = None
  276. tools: Optional[list[str]] = None
  277. source_knowledge: Optional[list[str]] = None
  278. # --- Requirement Models ---
  279. class RequirementIn(BaseModel):
  280. id: str
  281. description: str = ""
  282. atomics: list[str] = []
  283. source_nodes: list[dict] = []
  284. status: str = "未满足"
  285. match_result: str = ""
  286. class RequirementPatchIn(BaseModel):
  287. description: Optional[str] = None
  288. atomics: Optional[list[str]] = None
  289. source_nodes: Optional[list[dict]] = None
  290. status: Optional[str] = None
  291. match_result: Optional[str] = None
  292. class ResourceNode(BaseModel):
  293. id: str
  294. title: str
  295. class ResourceOut(BaseModel):
  296. id: str
  297. title: str
  298. body: str
  299. secure_body: str = ""
  300. content_type: str = "text"
  301. metadata: dict = {}
  302. toc: Optional[ResourceNode] = None
  303. children: list[ResourceNode]
  304. prev: Optional[ResourceNode] = None
  305. next: Optional[ResourceNode] = None
  306. # --- Dedup: Globals & Prompt ---
  307. knowledge_processor: Optional["KnowledgeProcessor"] = None
  308. # --- Dedup: RelationCache ---
  309. class RelationCache:
  310. """关系缓存,存储在内存中"""
  311. def __init__(self):
  312. self._cache: Dict[str, List[str]] = {}
  313. def load(self) -> dict:
  314. return self._cache
  315. def save(self, cache: dict):
  316. self._cache = cache
  317. def add_relation(self, relation_type: str, knowledge_id: str):
  318. if relation_type not in self._cache:
  319. self._cache[relation_type] = []
  320. if knowledge_id not in self._cache[relation_type]:
  321. self._cache[relation_type].append(knowledge_id)
  322. # --- Dedup: KnowledgeProcessor ---
  323. class KnowledgeProcessor:
  324. def __init__(self):
  325. self._lock = asyncio.Lock()
  326. self._relation_cache = RelationCache()
  327. async def process_pending(self):
  328. """持续处理 pending 和 dedup_passed 知识直到队列为空,有锁防并发"""
  329. if self._lock.locked():
  330. return
  331. async with self._lock:
  332. # 第一阶段:处理 pending(去重)
  333. while True:
  334. try:
  335. pending = pg_store.query('status == "pending"', limit=50)
  336. except Exception as e:
  337. print(f"[KnowledgeProcessor] 查询 pending 失败: {e}")
  338. break
  339. if not pending:
  340. break
  341. for knowledge in pending:
  342. await self._process_one(knowledge)
  343. # 第二阶段:处理 dedup_passed(工具关联)
  344. while True:
  345. try:
  346. dedup_passed = pg_store.query('status == "dedup_passed"', limit=50)
  347. except Exception as e:
  348. print(f"[KnowledgeProcessor] 查询 dedup_passed 失败: {e}")
  349. break
  350. if not dedup_passed:
  351. break
  352. for knowledge in dedup_passed:
  353. await self._analyze_tool_relation(knowledge)
  354. async def _process_one(self, knowledge: dict):
  355. kid = knowledge["id"]
  356. now = int(time.time())
  357. # 乐观锁:pending → processing(时间戳存秒级)
  358. try:
  359. pg_store.update(kid, {"status": "processing", "updated_at": now})
  360. except Exception as e:
  361. print(f"[KnowledgeProcessor] 锁定 {kid} 失败: {e}")
  362. return
  363. try:
  364. # 向量召回 top-10(只召回 approved/checked)
  365. embedding = knowledge.get("task_embedding") or knowledge.get("embedding")
  366. if not embedding:
  367. embedding = await get_embedding(knowledge["task"])
  368. candidates = pg_store.search(
  369. query_embedding=embedding,
  370. filters='(status == "approved" or status == "checked")',
  371. limit=10
  372. )
  373. candidates = [c for c in candidates if c["id"] != kid]
  374. # 只保留相似度 >= 0.75 的候选,低于阈值的 task 语义差异太大,直接视为 none
  375. candidates = [c for c in candidates if c.get("score", 0) >= 0.75]
  376. if not candidates:
  377. pg_store.update(kid, {"status": "dedup_passed", "updated_at": now})
  378. return
  379. llm_result = await self._llm_judge_relations(knowledge, candidates)
  380. await self._apply_decision(knowledge, llm_result)
  381. except Exception as e:
  382. print(f"[KnowledgeProcessor] 处理 {kid} 失败: {e},回退到 pending")
  383. try:
  384. pg_store.update(kid, {"status": "pending", "updated_at": int(time.time())})
  385. except Exception:
  386. pass
  387. async def _llm_judge_relations(self, new_knowledge: dict, candidates: list) -> dict:
  388. existing_list = "\n".join([
  389. f"[{i+1}] ID: {c['id']} | Task: {c['task']} | Content: {c['content'][:300]}"
  390. for i, c in enumerate(candidates)
  391. ])
  392. prompt = DEDUP_RELATION_PROMPT.format(
  393. new_task=new_knowledge["task"],
  394. new_content=new_knowledge["content"],
  395. existing_list=existing_list
  396. )
  397. for attempt in range(3):
  398. try:
  399. response = await _dedup_llm(
  400. messages=[{"role": "user", "content": prompt}],
  401. )
  402. content = response.get("content", "").strip()
  403. # 清理 markdown 代码块
  404. if "```" in content:
  405. parts = content.split("```")
  406. for part in parts:
  407. part = part.strip()
  408. if part.startswith("json"):
  409. part = part[4:].strip()
  410. try:
  411. result = json.loads(part)
  412. if "final_decision" in result:
  413. content = part
  414. break
  415. except Exception:
  416. continue
  417. result = json.loads(content)
  418. assert result.get("final_decision") in ("approved", "rejected")
  419. return result
  420. except Exception as e:
  421. print(f"[LLM Judge] 第{attempt+1}次失败: {e}")
  422. if attempt < 2:
  423. await asyncio.sleep(1)
  424. return {"final_decision": "approved", "relations": []}
  425. async def _apply_decision(self, new_knowledge: dict, llm_result: dict):
  426. kid = new_knowledge["id"]
  427. final_decision = llm_result.get("final_decision", "approved")
  428. relations = llm_result.get("relations", [])
  429. now = int(time.time())
  430. # 强制规则:如果存在 duplicate 或 subset 关系,必须 rejected
  431. if any(rel.get("type") in ("duplicate", "subset") for rel in relations):
  432. final_decision = "rejected"
  433. if final_decision == "rejected":
  434. # 记录 rejected 知识的关系(便于溯源为什么被拒绝)
  435. rejected_relationships = []
  436. for rel in relations:
  437. old_id = rel.get("old_id")
  438. rel_type = rel.get("type", "none")
  439. if old_id and rel_type != "none":
  440. rejected_relationships.append({"type": rel_type, "target": old_id})
  441. if rel_type in ("duplicate", "subset") and old_id:
  442. try:
  443. old = pg_store.get_by_id(old_id)
  444. if not old:
  445. continue
  446. eval_data = old.get("eval") or {}
  447. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  448. helpful_history = eval_data.get("helpful_history") or []
  449. helpful_history.append({
  450. "source": "dedup",
  451. "related_id": kid,
  452. "relation_type": rel_type,
  453. "timestamp": now
  454. })
  455. eval_data["helpful_history"] = helpful_history
  456. pg_store.update(old_id, {"eval": eval_data, "updated_at": now})
  457. except Exception as e:
  458. print(f"[Apply Decision] 更新旧知识 {old_id} helpful 失败: {e}")
  459. pg_store.update(kid, {"status": "rejected", "relationships": json.dumps(rejected_relationships), "updated_at": now})
  460. else:
  461. new_relationships = []
  462. for rel in relations:
  463. rel_type = rel.get("type", "none")
  464. reverse_type = rel.get("reverse_type", "none")
  465. old_id = rel.get("old_id")
  466. if not old_id or rel_type == "none":
  467. continue
  468. new_relationships.append({"type": rel_type, "target": old_id})
  469. self._relation_cache.add_relation(rel_type, kid)
  470. self._relation_cache.add_relation(rel_type, old_id)
  471. if reverse_type and reverse_type != "none":
  472. try:
  473. old = pg_store.get_by_id(old_id)
  474. if old:
  475. old_rels = old.get("relationships") or []
  476. old_rels.append({"type": reverse_type, "target": kid})
  477. pg_store.update(old_id, {"relationships": json.dumps(old_rels), "updated_at": now})
  478. self._relation_cache.add_relation(reverse_type, old_id)
  479. self._relation_cache.add_relation(reverse_type, kid)
  480. except Exception as e:
  481. print(f"[Apply Decision] 更新旧知识关系 {old_id} 失败: {e}")
  482. pg_store.update(kid, {
  483. "status": "dedup_passed",
  484. "relationships": json.dumps(new_relationships),
  485. "updated_at": now
  486. })
  487. async def _llm_analyze_tools(self, knowledge: dict) -> dict:
  488. """使用 LLM 分析知识中涉及的工具(复用迁移脚本逻辑)"""
  489. task = (knowledge.get("task") or "")[:600]
  490. content = (knowledge.get("content") or "")[:1200]
  491. prompt = TOOL_ANALYSIS_PROMPT.format(task=task, content=content)
  492. try:
  493. response = await _tool_analysis_llm(
  494. messages=[{"role": "user", "content": prompt}],
  495. max_tokens=2048,
  496. temperature=0.1,
  497. )
  498. raw = (response.get("content") or "").strip()
  499. if raw.startswith("```"):
  500. lines = raw.split("\n")
  501. inner = []
  502. in_block = False
  503. for line in lines:
  504. if line.startswith("```"):
  505. in_block = not in_block
  506. continue
  507. if in_block:
  508. inner.append(line)
  509. raw = "\n".join(inner).strip()
  510. return json.loads(raw)
  511. except Exception as e:
  512. print(f"[Tool Analysis LLM] 调用失败: {e}")
  513. raise
  514. async def _create_or_get_tool_resource(self, tool_info: dict) -> Optional[str]:
  515. """创建或获取工具资源(存入 PostgreSQL tool_table)"""
  516. category = tool_info.get("category", "other")
  517. slug = tool_info.get("slug", "")
  518. if not slug:
  519. return None
  520. tool_id = f"tools/{category}/{slug}"
  521. now_ts = int(time.time())
  522. cursor = pg_store._get_cursor()
  523. try:
  524. cursor.execute("SELECT id FROM tool_table WHERE id = %s", (tool_id,))
  525. if cursor.fetchone():
  526. return tool_id
  527. cursor.execute("""
  528. INSERT INTO tool_table (id, name, version, introduction, tutorial, input, output,
  529. updated_time, status, tool_knowledge, case_knowledge, process_knowledge)
  530. VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
  531. """, (
  532. tool_id,
  533. tool_info.get("name", slug),
  534. tool_info.get("version") or None,
  535. tool_info.get("description", ""),
  536. tool_info.get("usage", ""),
  537. json.dumps(tool_info.get("input", "")),
  538. json.dumps(tool_info.get("output", "")),
  539. now_ts,
  540. tool_info.get("status", "未接入"),
  541. json.dumps([]),
  542. json.dumps([]),
  543. json.dumps([]),
  544. ))
  545. pg_store.conn.commit()
  546. print(f"[Tool Resource] 创建新工具: {tool_id}")
  547. return tool_id
  548. finally:
  549. cursor.close()
  550. async def _update_tool_knowledge_index(self, tool_id: str, knowledge_id: str, knowledge_types: list = None):
  551. """根据知识类型更新工具的关联索引(tool_knowledge / case_knowledge / process_knowledge)"""
  552. # 确定目标字段
  553. if knowledge_types and 'plan' in knowledge_types:
  554. target_field = 'process_knowledge'
  555. elif knowledge_types and 'usecase' in knowledge_types:
  556. target_field = 'case_knowledge'
  557. else:
  558. target_field = 'tool_knowledge'
  559. now_ts = int(time.time())
  560. cursor = pg_store._get_cursor()
  561. try:
  562. cursor.execute(f"SELECT {target_field} FROM tool_table WHERE id = %s", (tool_id,))
  563. row = cursor.fetchone()
  564. if not row:
  565. return
  566. knowledge_ids = row[target_field] if isinstance(row[target_field], list) else json.loads(row[target_field] or "[]")
  567. if knowledge_id not in knowledge_ids:
  568. knowledge_ids.append(knowledge_id)
  569. cursor.execute(
  570. f"UPDATE tool_table SET {target_field} = %s, updated_time = %s WHERE id = %s",
  571. (json.dumps(knowledge_ids), now_ts, tool_id)
  572. )
  573. pg_store.conn.commit()
  574. finally:
  575. cursor.close()
  576. async def _analyze_tool_relation(self, knowledge: dict):
  577. """分析知识与工具的关联关系"""
  578. kid = knowledge["id"]
  579. now = int(time.time())
  580. # 乐观锁:dedup_passed → analyzing
  581. try:
  582. pg_store.update(kid, {"status": "analyzing", "updated_at": now})
  583. except Exception as e:
  584. print(f"[Tool Analysis] 锁定 {kid} 失败: {e}")
  585. return
  586. try:
  587. tool_analysis = await self._llm_analyze_tools(knowledge)
  588. has_tools = bool(tool_analysis and tool_analysis.get("has_tools"))
  589. existing_tags = knowledge.get("tags") or {}
  590. has_tool_tag = existing_tags.get("tool") is True
  591. # 情况1:LLM 判定无工具,但有 tool tag → 重新分析一次
  592. if not has_tools and has_tool_tag:
  593. print(f"[Tool Analysis] {kid} LLM 判定无工具但有 tool tag,重新分析")
  594. tool_analysis = await self._llm_analyze_tools(knowledge)
  595. has_tools = bool(tool_analysis and tool_analysis.get("has_tools"))
  596. # 重新分析后仍然不一致 → 知识模糊,rejected
  597. if not has_tools:
  598. pg_store.update(kid, {"status": "rejected", "updated_at": now})
  599. print(f"[Tool Analysis] {kid} 两次判定不一致,知识模糊,rejected")
  600. return
  601. # 情况2:无工具且无 tool tag → 直接 approved
  602. if not has_tools:
  603. pg_store.update(kid, {"status": "approved", "updated_at": now})
  604. return
  605. # 情况3/4:有工具 → 创建资源并关联
  606. tool_ids = []
  607. for tool_info in (tool_analysis.get("tools") or []):
  608. tool_id = await self._create_or_get_tool_resource(tool_info)
  609. if tool_id:
  610. tool_ids.append(tool_id)
  611. existing_resource_ids = knowledge.get("resource_ids") or []
  612. updated_resource_ids = list(set(existing_resource_ids + tool_ids))
  613. updates: dict = {
  614. "status": "approved",
  615. "resource_ids": updated_resource_ids,
  616. "updated_at": now
  617. }
  618. # 有工具但无 tool tag → 添加 tag
  619. if not has_tool_tag:
  620. updated_tags = dict(existing_tags)
  621. updated_tags["tool"] = True
  622. updates["tags"] = updated_tags
  623. print(f"[Tool Analysis] {kid} 添加 tool tag")
  624. pg_store.update(kid, updates)
  625. k_types = knowledge.get("types") or []
  626. for tool_id in tool_ids:
  627. await self._update_tool_knowledge_index(tool_id, kid, k_types)
  628. print(f"[Tool Analysis] {kid} 关联了 {len(tool_ids)} 个工具")
  629. except Exception as e:
  630. print(f"[Tool Analysis] {kid} 分析失败: {e},回退到 dedup_passed")
  631. try:
  632. pg_store.update(kid, {"status": "dedup_passed", "updated_at": int(time.time())})
  633. except Exception:
  634. pass
  635. async def _periodic_processor():
  636. """每60秒检测超时条目并回滚:processing(>5min)→pending,analyzing(>10min)→dedup_passed"""
  637. while True:
  638. await asyncio.sleep(60)
  639. try:
  640. now = int(time.time())
  641. # 回滚超时的 processing(5分钟 → pending)
  642. timeout_5min = now - 300
  643. processing = pg_store.query('status == "processing"', limit=200)
  644. for item in processing:
  645. updated_at = item.get("updated_at", 0) or 0
  646. updated_at_sec = updated_at // 1000 if updated_at > 1_000_000_000_000 else updated_at
  647. if updated_at_sec < timeout_5min:
  648. print(f"[Periodic] 回滚超时 processing → pending: {item['id']}")
  649. pg_store.update(item["id"], {"status": "pending", "updated_at": int(time.time())})
  650. # 回滚超时的 analyzing(10分钟 → dedup_passed)
  651. timeout_10min = now - 600
  652. analyzing = pg_store.query('status == "analyzing"', limit=200)
  653. for item in analyzing:
  654. updated_at = item.get("updated_at", 0) or 0
  655. updated_at_sec = updated_at // 1000 if updated_at > 1_000_000_000_000 else updated_at
  656. if updated_at_sec < timeout_10min:
  657. print(f"[Periodic] 回滚超时 analyzing → dedup_passed: {item['id']}")
  658. pg_store.update(item["id"], {"status": "dedup_passed", "updated_at": int(time.time())})
  659. except Exception as e:
  660. print(f"[Periodic] 定时任务错误: {e}")
  661. # --- App ---
  662. @asynccontextmanager
  663. async def lifespan(app: FastAPI):
  664. global pg_store, pg_resource_store, pg_tool_store, pg_capability_store, pg_requirement_store, knowledge_processor
  665. # 初始化 PostgreSQL(knowledge + resources + tools + capabilities + requirements)
  666. pg_store = PostgreSQLStore()
  667. pg_resource_store = PostgreSQLResourceStore()
  668. pg_tool_store = PostgreSQLToolStore()
  669. pg_capability_store = PostgreSQLCapabilityStore()
  670. pg_requirement_store = PostgreSQLRequirementStore()
  671. # 初始化去重处理器 + 启动定时兜底任务
  672. knowledge_processor = KnowledgeProcessor()
  673. periodic_task = asyncio.create_task(_periodic_processor())
  674. yield
  675. # 清理
  676. periodic_task.cancel()
  677. try:
  678. await periodic_task
  679. except asyncio.CancelledError:
  680. pass
  681. pg_store.close()
  682. pg_resource_store.close()
  683. pg_tool_store.close()
  684. pg_capability_store.close()
  685. pg_requirement_store.close()
  686. app = FastAPI(title=BRAND_NAME, lifespan=lifespan)
  687. # 挂载静态文件
  688. STATIC_DIR = Path(__file__).parent / "frontend" / "dist"
  689. if STATIC_DIR.exists():
  690. app.mount("/assets", StaticFiles(directory=str(STATIC_DIR / "assets")), name="assets")
  691. # --- Knowledge API ---
  692. @app.post("/api/resource", status_code=201)
  693. def submit_resource(resource: ResourceIn):
  694. """提交资源(存入 PostgreSQL resources 表)"""
  695. try:
  696. # 加密敏感内容
  697. encrypted_secure_body = encrypt_content(resource.id, resource.secure_body)
  698. pg_resource_store.insert_or_update({
  699. 'id': resource.id,
  700. 'title': resource.title,
  701. 'body': resource.body,
  702. 'secure_body': encrypted_secure_body,
  703. 'content_type': resource.content_type,
  704. 'metadata': resource.metadata,
  705. 'sort_order': resource.sort_order,
  706. 'submitted_by': resource.submitted_by
  707. })
  708. return {"status": "ok", "id": resource.id}
  709. except Exception as e:
  710. raise HTTPException(status_code=500, detail=str(e))
  711. @app.get("/api/resource/{resource_id:path}", response_model=ResourceOut)
  712. def get_resource(resource_id: str, x_org_key: Optional[str] = Header(None)):
  713. """获取资源详情(从 PostgreSQL)"""
  714. try:
  715. row = pg_resource_store.get_by_id(resource_id)
  716. if not row:
  717. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  718. # 解密敏感内容
  719. secure_body = decrypt_content(resource_id, row.get("secure_body", ""), x_org_key)
  720. # 计算导航上下文
  721. root_id = resource_id.split("/")[0] if "/" in resource_id else resource_id
  722. # TOC (根节点)
  723. toc = None
  724. if "/" in resource_id:
  725. toc_row = pg_resource_store.get_by_id(root_id)
  726. if toc_row:
  727. toc = ResourceNode(id=toc_row["id"], title=toc_row["title"])
  728. # Children (子节点)
  729. children_rows = pg_resource_store.list_resources(prefix=f"{resource_id}/", limit=1000)
  730. children = [ResourceNode(id=r["id"], title=r["title"]) for r in children_rows
  731. if r["id"].count("/") == resource_id.count("/") + 1]
  732. # Prev/Next (同级节点)
  733. prev_node, next_node = pg_resource_store.get_siblings(resource_id)
  734. prev = ResourceNode(id=prev_node["id"], title=prev_node["title"]) if prev_node else None
  735. next = ResourceNode(id=next_node["id"], title=next_node["title"]) if next_node else None
  736. return ResourceOut(
  737. id=row["id"],
  738. title=row["title"],
  739. body=row["body"],
  740. secure_body=secure_body,
  741. content_type=row["content_type"],
  742. metadata=row.get("metadata", {}),
  743. toc=toc,
  744. children=children,
  745. prev=prev,
  746. next=next,
  747. )
  748. except HTTPException:
  749. raise
  750. except Exception as e:
  751. raise HTTPException(status_code=500, detail=str(e))
  752. @app.patch("/api/resource/{resource_id:path}")
  753. def patch_resource(resource_id: str, patch: ResourcePatchIn):
  754. """更新resource字段(PostgreSQL)"""
  755. try:
  756. # 检查是否存在
  757. if not pg_resource_store.get_by_id(resource_id):
  758. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  759. # 构建更新字典
  760. updates = {}
  761. if patch.title is not None:
  762. updates['title'] = patch.title
  763. if patch.body is not None:
  764. updates['body'] = patch.body
  765. if patch.secure_body is not None:
  766. updates['secure_body'] = encrypt_content(resource_id, patch.secure_body)
  767. if patch.content_type is not None:
  768. updates['content_type'] = patch.content_type
  769. if patch.metadata is not None:
  770. updates['metadata'] = patch.metadata
  771. if not updates:
  772. return {"status": "ok", "message": "No fields to update"}
  773. pg_resource_store.update(resource_id, updates)
  774. return {"status": "ok", "id": resource_id}
  775. except HTTPException:
  776. raise
  777. except Exception as e:
  778. raise HTTPException(status_code=500, detail=str(e))
  779. @app.get("/api/resource")
  780. def list_resources(
  781. content_type: Optional[str] = Query(None),
  782. limit: int = Query(100, ge=1, le=1000)
  783. ):
  784. """列出所有resource(PostgreSQL)"""
  785. try:
  786. results = pg_resource_store.list_resources(
  787. content_type=content_type,
  788. limit=limit
  789. )
  790. return {"results": results, "count": len(results)}
  791. except Exception as e:
  792. raise HTTPException(status_code=500, detail=str(e))
  793. @app.delete("/api/resource/{resource_id:path}")
  794. def delete_resource(resource_id: str):
  795. """删除单个resource(PostgreSQL)"""
  796. try:
  797. if not pg_resource_store.get_by_id(resource_id):
  798. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  799. pg_resource_store.delete(resource_id)
  800. return {"status": "ok", "id": resource_id}
  801. except HTTPException:
  802. raise
  803. except Exception as e:
  804. raise HTTPException(status_code=500, detail=str(e))
  805. # --- Knowledge API ---
  806. # ===== Knowledge API =====
  807. async def _llm_rerank(query: str, candidates: list[dict], top_k: int) -> list[str]:
  808. """
  809. 使用 LLM 对候选知识进行精排
  810. Args:
  811. query: 查询文本
  812. candidates: 候选知识列表
  813. top_k: 返回数量
  814. Returns:
  815. 排序后的知识 ID 列表
  816. """
  817. if not candidates:
  818. return []
  819. # 构造 prompt
  820. candidates_text = "\n".join([
  821. f"[{i+1}] ID: {c['id']}\nTask: {c['task']}\nContent: {c['content'][:200]}..."
  822. for i, c in enumerate(candidates)
  823. ])
  824. prompt = RERANK_PROMPT_TEMPLATE.format(
  825. top_k=top_k,
  826. query=query,
  827. candidates_text=candidates_text
  828. )
  829. try:
  830. response = await _dedup_llm(
  831. messages=[{"role": "user", "content": prompt}],
  832. )
  833. content = response.get("content", "").strip()
  834. # 解析 ID 列表
  835. selected_ids = [
  836. idx.strip()
  837. for idx in re.split(r'[,\s]+', content)
  838. if idx.strip().startswith(("knowledge-", "research-"))
  839. ]
  840. return selected_ids[:top_k]
  841. except Exception as e:
  842. print(f"[LLM Rerank] 失败: {e}")
  843. return []
  844. @app.get("/api/knowledge/search")
  845. async def search_knowledge_api(
  846. q: str = Query(..., description="查询文本"),
  847. top_k: int = Query(default=5, ge=1, le=20),
  848. min_score: int = Query(default=3, ge=1, le=5),
  849. types: Optional[str] = None,
  850. owner: Optional[str] = None
  851. ):
  852. """检索知识(向量召回 + LLM 精排)"""
  853. try:
  854. # 1. 生成查询向量
  855. query_embedding = await get_embedding(q)
  856. # 2. 构建过滤表达式
  857. filters = []
  858. if types:
  859. type_list = [t.strip() for t in types.split(',') if t.strip()]
  860. for t in type_list:
  861. filters.append(f'array_contains(types, "{t}")')
  862. if owner:
  863. owner_list = [o.strip() for o in owner.split(',') if o.strip()]
  864. if len(owner_list) == 1:
  865. filters.append(f'owner == "{owner_list[0]}"')
  866. else:
  867. # 多个owner用OR连接
  868. owner_filters = [f'owner == "{o}"' for o in owner_list]
  869. filters.append(f'({" or ".join(owner_filters)})')
  870. # 添加 min_score 过滤
  871. filters.append(f'eval["score"] >= {min_score}')
  872. # 只搜索 approved 和 checked 的知识
  873. filters.append('(status == "approved" or status == "checked")')
  874. filter_expr = ' and '.join(filters) if filters else None
  875. # 3. 向量召回(3*k 个候选)
  876. recall_limit = top_k * 3
  877. candidates = pg_store.search(
  878. query_embedding=query_embedding,
  879. filters=filter_expr,
  880. limit=recall_limit
  881. )
  882. if not candidates:
  883. return {"results": [], "count": 0, "reranked": False}
  884. # 转换为可序列化的格式
  885. serialized_candidates = [serialize_milvus_result(c) for c in candidates]
  886. # 为了保证搜索的极致速度,直接返回向量召回的 top-k(跳过缓慢的 LLM 精排)
  887. return {"results": serialized_candidates[:top_k], "count": len(serialized_candidates[:top_k]), "reranked": False}
  888. except Exception as e:
  889. print(f"[Knowledge Search] 错误: {e}")
  890. raise HTTPException(status_code=500, detail=str(e))
  891. @app.post("/api/knowledge", status_code=201)
  892. async def save_knowledge(knowledge: KnowledgeIn, background_tasks: BackgroundTasks):
  893. """保存新知识"""
  894. try:
  895. # 生成 ID
  896. timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
  897. random_suffix = uuid.uuid4().hex[:4]
  898. knowledge_id = f"knowledge-{timestamp}-{random_suffix}"
  899. now = int(time.time())
  900. # 设置默认值
  901. owner = knowledge.owner or f"agent:{knowledge.source.get('agent_id', 'unknown')}"
  902. # 准备 source
  903. source = {
  904. "name": knowledge.source.get("name", ""),
  905. "category": knowledge.source.get("category", ""),
  906. "urls": knowledge.source.get("urls", []),
  907. "agent_id": knowledge.source.get("agent_id", "unknown"),
  908. "submitted_by": knowledge.source.get("submitted_by", ""),
  909. "timestamp": datetime.now(timezone.utc).isoformat(),
  910. "message_id": knowledge.message_id
  911. }
  912. # 准备 eval
  913. eval_data = {
  914. "score": knowledge.eval.get("score", 3),
  915. "helpful": knowledge.eval.get("helpful", 1),
  916. "harmful": knowledge.eval.get("harmful", 0),
  917. "confidence": knowledge.eval.get("confidence", 0.5),
  918. "helpful_history": [],
  919. "harmful_history": []
  920. }
  921. # 生成向量(task_embedding + content_embedding 双向量)
  922. task_embedding = await get_embedding(knowledge.task)
  923. content_embedding = await get_embedding(knowledge.content)
  924. # 提取 tag keys(用于高效筛选)
  925. tag_keys = list(knowledge.tags.keys()) if isinstance(knowledge.tags, dict) else []
  926. # 准备插入数据
  927. insert_data = {
  928. "id": knowledge_id,
  929. "task_embedding": task_embedding,
  930. "content_embedding": content_embedding,
  931. "message_id": knowledge.message_id,
  932. "task": knowledge.task,
  933. "content": knowledge.content,
  934. "types": knowledge.types,
  935. "tags": knowledge.tags,
  936. "tag_keys": tag_keys,
  937. "scopes": knowledge.scopes,
  938. "owner": owner,
  939. "resource_ids": knowledge.resource_ids,
  940. "source": source,
  941. "eval": eval_data,
  942. "created_at": now,
  943. "updated_at": now,
  944. "status": "pending",
  945. "relationships": json.dumps([]),
  946. "support_capability": knowledge.support_capability,
  947. "tools": knowledge.tools,
  948. }
  949. print(f"[Save Knowledge] 插入数据: {json.dumps({k: v for k, v in insert_data.items() if k != 'embedding'}, ensure_ascii=False)}")
  950. # 插入 Milvus
  951. pg_store.insert(insert_data)
  952. # 触发后台去重处理
  953. background_tasks.add_task(knowledge_processor.process_pending)
  954. return {"status": "pending", "knowledge_id": knowledge_id, "message": "知识已入队,正在处理去重..."}
  955. except Exception as e:
  956. print(f"[Save Knowledge] 错误: {e}")
  957. raise HTTPException(status_code=500, detail=str(e))
  958. @app.get("/api/knowledge")
  959. def list_knowledge(
  960. page: int = Query(default=1, ge=1),
  961. page_size: int = Query(default=20, ge=1, le=1000),
  962. types: Optional[str] = None,
  963. scopes: Optional[str] = None,
  964. owner: Optional[str] = None,
  965. tags: Optional[str] = None,
  966. status: Optional[str] = None
  967. ):
  968. """列出知识(支持后端筛选和分页)"""
  969. try:
  970. # 构建过滤表达式
  971. filters = []
  972. # types 支持多个,用 AND 连接(交集:必须同时包含所有选中的type)
  973. if types:
  974. type_list = [t.strip() for t in types.split(',') if t.strip()]
  975. for t in type_list:
  976. filters.append(f'array_contains(types, "{t}")')
  977. if scopes:
  978. filters.append(f'array_contains(scopes, "{scopes}")')
  979. if owner:
  980. owner_list = [o.strip() for o in owner.split(',') if o.strip()]
  981. if len(owner_list) == 1:
  982. filters.append(f'owner == "{owner_list[0]}"')
  983. else:
  984. # 多个owner用OR连接
  985. owner_filters = [f'owner == "{o}"' for o in owner_list]
  986. filters.append(f'({" or ".join(owner_filters)})')
  987. # tags 支持多个,用 AND 连接(使用 tag_keys 数组进行高效筛选)
  988. if tags:
  989. tag_list = [t.strip() for t in tags.split(',') if t.strip()]
  990. for t in tag_list:
  991. filters.append(f'array_contains(tag_keys, "{t}")')
  992. # 只返回指定 status 的知识(默认 approved 和 checked)
  993. status_list = [s.strip() for s in (status or "approved,checked").split(',') if s.strip()]
  994. status_conditions = ' or '.join([f'status == "{s}"' for s in status_list])
  995. filters.append(f'({status_conditions})')
  996. # 如果没有过滤条件,查询所有
  997. filter_expr = ' and '.join(filters) if filters else 'id != ""'
  998. # 查询 Milvus(先获取所有符合条件的数据)
  999. # Milvus 的 limit 是总数限制,我们需要获取足够多的数据来支持分页
  1000. max_limit = 10000 # 设置一个合理的上限
  1001. results = pg_store.query(filter_expr, limit=max_limit)
  1002. # 转换为可序列化的格式
  1003. serialized_results = [serialize_milvus_result(r) for r in results]
  1004. # 按 created_at 降序排序(最新的在前)
  1005. serialized_results.sort(key=lambda x: x.get('created_at', 0), reverse=True)
  1006. # 计算分页
  1007. total = len(serialized_results)
  1008. total_pages = (total + page_size - 1) // page_size # 向上取整
  1009. start_idx = (page - 1) * page_size
  1010. end_idx = start_idx + page_size
  1011. page_results = serialized_results[start_idx:end_idx]
  1012. return {
  1013. "results": page_results,
  1014. "pagination": {
  1015. "page": page,
  1016. "page_size": page_size,
  1017. "total": total,
  1018. "total_pages": total_pages
  1019. }
  1020. }
  1021. except Exception as e:
  1022. print(f"[List Knowledge] 错误: {e}")
  1023. raise HTTPException(status_code=500, detail=str(e))
  1024. @app.get("/api/knowledge/meta/tags")
  1025. def get_all_tags():
  1026. """获取所有已有的 tags"""
  1027. try:
  1028. # 查询所有知识
  1029. results = pg_store.query('id != ""', limit=10000)
  1030. all_tags = set()
  1031. for item in results:
  1032. # 转换为标准字典
  1033. serialized_item = serialize_milvus_result(item)
  1034. tags_dict = serialized_item.get("tags", {})
  1035. if isinstance(tags_dict, dict):
  1036. for key in tags_dict.keys():
  1037. all_tags.add(key)
  1038. return {"tags": sorted(list(all_tags))}
  1039. except Exception as e:
  1040. print(f"[Get Tags] 错误: {e}")
  1041. raise HTTPException(status_code=500, detail=str(e))
  1042. @app.get("/api/knowledge/pending")
  1043. def get_pending_knowledge(limit: int = Query(default=50, ge=1, le=200)):
  1044. """查询待处理队列(pending + processing + dedup_passed + analyzing)"""
  1045. try:
  1046. pending = pg_store.query(
  1047. 'status == "pending" or status == "processing" or status == "dedup_passed" or status == "analyzing"',
  1048. limit=limit
  1049. )
  1050. serialized = [serialize_milvus_result(r) for r in pending]
  1051. return {"results": serialized, "count": len(serialized)}
  1052. except Exception as e:
  1053. print(f"[Pending] 错误: {e}")
  1054. raise HTTPException(status_code=500, detail=str(e))
  1055. @app.post("/api/knowledge/process")
  1056. async def trigger_process(force: bool = Query(default=False)):
  1057. """手动触发去重处理。force=true 时先回滚所有 processing → pending,analyzing → dedup_passed"""
  1058. try:
  1059. if force:
  1060. processing = pg_store.query('status == "processing"', limit=200)
  1061. for item in processing:
  1062. pg_store.update(item["id"], {"status": "pending", "updated_at": int(time.time())})
  1063. print(f"[Manual Process] 回滚 {len(processing)} 条 processing → pending")
  1064. analyzing = pg_store.query('status == "analyzing"', limit=200)
  1065. for item in analyzing:
  1066. pg_store.update(item["id"], {"status": "dedup_passed", "updated_at": int(time.time())})
  1067. print(f"[Manual Process] 回滚 {len(analyzing)} 条 analyzing → dedup_passed")
  1068. asyncio.create_task(knowledge_processor.process_pending())
  1069. return {"status": "ok", "message": "处理任务已触发"}
  1070. except Exception as e:
  1071. print(f"[Manual Process] 错误: {e}")
  1072. raise HTTPException(status_code=500, detail=str(e))
  1073. @app.post("/api/knowledge/migrate")
  1074. async def migrate_knowledge_schema():
  1075. """手动触发 schema 迁移(PostgreSQL不需要此功能)"""
  1076. return {"status": "ok", "message": "PostgreSQL不需要schema迁移"}
  1077. @app.get("/api/knowledge/status/{knowledge_id}")
  1078. def get_knowledge_status(knowledge_id: str):
  1079. """查询单条知识的处理状态和关系"""
  1080. try:
  1081. result = pg_store.get_by_id(knowledge_id)
  1082. if not result:
  1083. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1084. serialized = serialize_milvus_result(result)
  1085. return {
  1086. "id": knowledge_id,
  1087. "status": serialized.get("status", "approved"),
  1088. "relationships": serialized.get("relationships", []),
  1089. "created_at": serialized.get("created_at"),
  1090. "updated_at": serialized.get("updated_at"),
  1091. }
  1092. except HTTPException:
  1093. raise
  1094. except Exception as e:
  1095. print(f"[Knowledge Status] 错误: {e}")
  1096. raise HTTPException(status_code=500, detail=str(e))
  1097. @app.get("/api/knowledge/{knowledge_id}")
  1098. def get_knowledge(knowledge_id: str):
  1099. """获取单条知识"""
  1100. try:
  1101. result = pg_store.get_by_id(knowledge_id)
  1102. if not result:
  1103. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1104. return serialize_milvus_result(result)
  1105. except HTTPException:
  1106. raise
  1107. except Exception as e:
  1108. print(f"[Get Knowledge] 错误: {e}")
  1109. raise HTTPException(status_code=500, detail=str(e))
  1110. async def _evolve_knowledge_with_llm(old_content: str, feedback: str) -> str:
  1111. """使用 LLM 进行知识进化重写"""
  1112. prompt = KNOWLEDGE_EVOLVE_PROMPT_TEMPLATE.format(
  1113. old_content=old_content,
  1114. feedback=feedback
  1115. )
  1116. try:
  1117. response = await _dedup_llm(
  1118. messages=[{"role": "user", "content": prompt}],
  1119. )
  1120. evolved = response.get("content", "").strip()
  1121. if len(evolved) < 5:
  1122. raise ValueError("LLM output too short")
  1123. return evolved
  1124. except Exception as e:
  1125. print(f"知识进化失败,采用追加模式回退: {e}")
  1126. return f"{old_content}\n\n---\n[Update {datetime.now().strftime('%Y-%m-%d')}]: {feedback}"
  1127. @app.put("/api/knowledge/{knowledge_id}")
  1128. async def update_knowledge(knowledge_id: str, update: KnowledgeUpdateIn):
  1129. """更新知识评估,支持知识进化"""
  1130. try:
  1131. # 获取现有知识
  1132. existing = pg_store.get_by_id(knowledge_id)
  1133. if not existing:
  1134. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1135. eval_data = existing.get("eval", {})
  1136. # 更新评分
  1137. if update.update_score is not None:
  1138. eval_data["score"] = update.update_score
  1139. # 添加有效案例
  1140. if update.add_helpful_case:
  1141. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1142. if "helpful_history" not in eval_data:
  1143. eval_data["helpful_history"] = []
  1144. eval_data["helpful_history"].append(update.add_helpful_case)
  1145. # 添加有害案例
  1146. if update.add_harmful_case:
  1147. eval_data["harmful"] = eval_data.get("harmful", 0) + 1
  1148. if "harmful_history" not in eval_data:
  1149. eval_data["harmful_history"] = []
  1150. eval_data["harmful_history"].append(update.add_harmful_case)
  1151. # 知识进化
  1152. content = existing["content"]
  1153. need_reembed = False
  1154. if update.evolve_feedback:
  1155. content = await _evolve_knowledge_with_llm(content, update.evolve_feedback)
  1156. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1157. need_reembed = True
  1158. # 准备更新数据
  1159. updates = {
  1160. "content": content,
  1161. "eval": eval_data,
  1162. }
  1163. # 如果内容变化,重新生成向量
  1164. if need_reembed:
  1165. embedding = await get_embedding(existing['task'])
  1166. updates["task_embedding"] = embedding
  1167. # 更新 Milvus
  1168. pg_store.update(knowledge_id, updates)
  1169. return {"status": "ok", "knowledge_id": knowledge_id}
  1170. except HTTPException:
  1171. raise
  1172. except Exception as e:
  1173. print(f"[Update Knowledge] 错误: {e}")
  1174. raise HTTPException(status_code=500, detail=str(e))
  1175. @app.patch("/api/knowledge/{knowledge_id}")
  1176. async def patch_knowledge(knowledge_id: str, patch: KnowledgePatchIn):
  1177. """直接编辑知识字段"""
  1178. try:
  1179. # 获取现有知识
  1180. existing = pg_store.get_by_id(knowledge_id)
  1181. if not existing:
  1182. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1183. updates = {}
  1184. need_reembed = False
  1185. need_content_reembed = False
  1186. if patch.task is not None:
  1187. updates["task"] = patch.task
  1188. need_reembed = True
  1189. if patch.content is not None:
  1190. updates["content"] = patch.content
  1191. need_content_reembed = True
  1192. if patch.types is not None:
  1193. updates["types"] = patch.types
  1194. if patch.tags is not None:
  1195. updates["tags"] = patch.tags
  1196. # 同时更新 tag_keys
  1197. updates["tag_keys"] = list(patch.tags.keys()) if isinstance(patch.tags, dict) else []
  1198. if patch.scopes is not None:
  1199. updates["scopes"] = patch.scopes
  1200. if patch.owner is not None:
  1201. updates["owner"] = patch.owner
  1202. if patch.support_capability is not None:
  1203. updates["support_capability"] = patch.support_capability
  1204. if patch.tools is not None:
  1205. updates["tools"] = patch.tools
  1206. if not updates:
  1207. return {"status": "ok", "knowledge_id": knowledge_id}
  1208. # 如果 task 变化,重新生成 task_embedding
  1209. if need_reembed:
  1210. task = updates.get("task", existing["task"])
  1211. embedding = await get_embedding(task)
  1212. updates["task_embedding"] = embedding
  1213. # 如果 content 变化,重新生成 content_embedding
  1214. if need_content_reembed:
  1215. content = updates.get("content", existing["content"])
  1216. content_embedding = await get_embedding(content)
  1217. updates["content_embedding"] = content_embedding
  1218. # 更新 Milvus
  1219. pg_store.update(knowledge_id, updates)
  1220. return {"status": "ok", "knowledge_id": knowledge_id}
  1221. except HTTPException:
  1222. raise
  1223. except Exception as e:
  1224. print(f"[Patch Knowledge] 错误: {e}")
  1225. raise HTTPException(status_code=500, detail=str(e))
  1226. @app.delete("/api/knowledge/{knowledge_id}")
  1227. def delete_knowledge(knowledge_id: str):
  1228. """删除单条知识"""
  1229. try:
  1230. # 检查知识是否存在
  1231. existing = pg_store.get_by_id(knowledge_id)
  1232. if not existing:
  1233. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1234. # 从 PostgreSQL 删除
  1235. pg_store.delete(knowledge_id)
  1236. print(f"[Delete Knowledge] 已删除知识: {knowledge_id}")
  1237. return {"status": "ok", "knowledge_id": knowledge_id}
  1238. except HTTPException:
  1239. raise
  1240. except Exception as e:
  1241. print(f"[Delete Knowledge] 错误: {e}")
  1242. raise HTTPException(status_code=500, detail=str(e))
  1243. @app.post("/api/knowledge/batch_delete")
  1244. def batch_delete_knowledge(knowledge_ids: List[str] = Body(...)):
  1245. """批量删除知识"""
  1246. try:
  1247. if not knowledge_ids:
  1248. raise HTTPException(status_code=400, detail="knowledge_ids cannot be empty")
  1249. # 批量删除
  1250. cursor = pg_store._get_cursor()
  1251. try:
  1252. cursor.execute(
  1253. "DELETE FROM knowledge WHERE id = ANY(%s)",
  1254. (knowledge_ids,)
  1255. )
  1256. pg_store.conn.commit()
  1257. deleted_count = cursor.rowcount
  1258. print(f"[Batch Delete] 已删除 {deleted_count} 条知识")
  1259. return {"status": "ok", "deleted_count": deleted_count}
  1260. finally:
  1261. cursor.close()
  1262. except HTTPException:
  1263. raise
  1264. except Exception as e:
  1265. print(f"[Batch Delete] 错误: {e}")
  1266. raise HTTPException(status_code=500, detail=str(e))
  1267. @app.post("/api/knowledge/batch_verify")
  1268. async def batch_verify_knowledge(batch: KnowledgeBatchVerifyIn):
  1269. """批量验证通过(approved → checked)"""
  1270. if not batch.knowledge_ids:
  1271. return {"status": "ok", "updated": 0}
  1272. try:
  1273. now_iso = datetime.now(timezone.utc).isoformat()
  1274. updated_count = 0
  1275. for kid in batch.knowledge_ids:
  1276. existing = pg_store.get_by_id(kid)
  1277. if not existing:
  1278. continue
  1279. eval_data = existing.get("eval") or {}
  1280. eval_data["verification"] = {
  1281. "status": "checked",
  1282. "verified_by": batch.verified_by,
  1283. "verified_at": now_iso,
  1284. "note": None,
  1285. "issue_type": None,
  1286. "issue_action": None,
  1287. }
  1288. pg_store.update(kid, {"eval": eval_data, "status": "checked", "updated_at": int(time.time())})
  1289. updated_count += 1
  1290. return {"status": "ok", "updated": updated_count}
  1291. except Exception as e:
  1292. print(f"[Batch Verify] 错误: {e}")
  1293. raise HTTPException(status_code=500, detail=str(e))
  1294. @app.post("/api/knowledge/{knowledge_id}/verify")
  1295. async def verify_knowledge(knowledge_id: str, verify: KnowledgeVerifyIn):
  1296. """知识验证:approve 切换 approved↔checked,reject 设为 rejected"""
  1297. try:
  1298. existing = pg_store.get_by_id(knowledge_id)
  1299. if not existing:
  1300. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1301. current_status = existing.get("status", "approved")
  1302. if verify.action == "approve":
  1303. # checked → approved(取消验证),其他 → checked
  1304. new_status = "approved" if current_status == "checked" else "checked"
  1305. pg_store.update(knowledge_id, {
  1306. "status": new_status,
  1307. "updated_at": int(time.time())
  1308. })
  1309. return {"status": "ok", "new_status": new_status,
  1310. "message": "已取消验证" if new_status == "approved" else "验证通过"}
  1311. elif verify.action == "reject":
  1312. pg_store.update(knowledge_id, {
  1313. "status": "rejected",
  1314. "updated_at": int(time.time())
  1315. })
  1316. return {"status": "ok", "new_status": "rejected", "message": "已拒绝"}
  1317. else:
  1318. raise HTTPException(status_code=400, detail=f"Unknown action: {verify.action}")
  1319. except HTTPException:
  1320. raise
  1321. except Exception as e:
  1322. print(f"[Verify Knowledge] 错误: {e}")
  1323. raise HTTPException(status_code=500, detail=str(e))
  1324. @app.post("/api/knowledge/batch_update")
  1325. async def batch_update_knowledge(batch: KnowledgeBatchUpdateIn):
  1326. """批量反馈知识有效性"""
  1327. if not batch.feedback_list:
  1328. return {"status": "ok", "updated": 0}
  1329. try:
  1330. # 先处理无需进化的,收集需要进化的
  1331. evolution_tasks = [] # [(knowledge_id, old_content, feedback, eval_data)]
  1332. simple_updates = [] # [(knowledge_id, is_effective, eval_data)]
  1333. for item in batch.feedback_list:
  1334. knowledge_id = item.get("knowledge_id")
  1335. is_effective = item.get("is_effective")
  1336. feedback = item.get("feedback", "")
  1337. if not knowledge_id:
  1338. continue
  1339. existing = pg_store.get_by_id(knowledge_id)
  1340. if not existing:
  1341. continue
  1342. eval_data = existing.get("eval", {})
  1343. if is_effective and feedback:
  1344. evolution_tasks.append((knowledge_id, existing["content"], feedback, eval_data, existing["task"]))
  1345. else:
  1346. simple_updates.append((knowledge_id, is_effective, eval_data))
  1347. # 执行简单更新
  1348. for knowledge_id, is_effective, eval_data in simple_updates:
  1349. if is_effective:
  1350. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1351. else:
  1352. eval_data["harmful"] = eval_data.get("harmful", 0) + 1
  1353. pg_store.update(knowledge_id, {"eval": eval_data})
  1354. # 并发执行知识进化
  1355. if evolution_tasks:
  1356. print(f"🧬 并发处理 {len(evolution_tasks)} 条知识进化...")
  1357. evolved_results = await asyncio.gather(
  1358. *[_evolve_knowledge_with_llm(old, fb) for _, old, fb, _, _ in evolution_tasks]
  1359. )
  1360. for (knowledge_id, _, _, eval_data, task), evolved_content in zip(evolution_tasks, evolved_results):
  1361. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1362. # 重新生成向量(只基于 task)
  1363. embedding = await get_embedding(task)
  1364. pg_store.update(knowledge_id, {
  1365. "content": evolved_content,
  1366. "eval": eval_data,
  1367. "task_embedding": embedding
  1368. })
  1369. return {"status": "ok", "updated": len(simple_updates) + len(evolution_tasks)}
  1370. except Exception as e:
  1371. print(f"[Batch Update] 错误: {e}")
  1372. raise HTTPException(status_code=500, detail=str(e))
  1373. @app.post("/api/knowledge/slim")
  1374. async def slim_knowledge(model: str = "google/gemini-2.5-flash-lite"):
  1375. """知识库瘦身:合并语义相似知识"""
  1376. try:
  1377. # 获取所有知识
  1378. all_knowledge = pg_store.query('id != ""', limit=10000)
  1379. # 转换为可序列化的格式
  1380. all_knowledge = [serialize_milvus_result(item) for item in all_knowledge]
  1381. if len(all_knowledge) < 2:
  1382. return {"status": "ok", "message": f"知识库仅有 {len(all_knowledge)} 条,无需瘦身"}
  1383. # 构造发给大模型的内容
  1384. entries_text = ""
  1385. for item in all_knowledge:
  1386. eval_data = item.get("eval", {})
  1387. types = item.get("types", [])
  1388. entries_text += f"[ID: {item['id']}] [Types: {','.join(types)}] "
  1389. entries_text += f"[Helpful: {eval_data.get('helpful', 0)}, Harmful: {eval_data.get('harmful', 0)}] [Score: {eval_data.get('score', 3)}]\n"
  1390. entries_text += f"Task: {item['task']}\n"
  1391. entries_text += f"Content: {item['content'][:200]}...\n\n"
  1392. prompt = KNOWLEDGE_SLIM_PROMPT_TEMPLATE.format(entries_text=entries_text)
  1393. print(f"\n[知识瘦身] 正在调用 {model} 分析 {len(all_knowledge)} 条知识...")
  1394. slim_llm = create_openrouter_llm_call(model=model)
  1395. response = await slim_llm(
  1396. messages=[{"role": "user", "content": prompt}],
  1397. )
  1398. content = response.get("content", "").strip()
  1399. if not content:
  1400. raise HTTPException(status_code=500, detail="LLM 返回为空")
  1401. # 解析大模型输出
  1402. report_line = ""
  1403. new_entries = []
  1404. blocks = [b.strip() for b in content.split("===") if b.strip()]
  1405. for block in blocks:
  1406. if block.startswith("REPORT:"):
  1407. report_line = block
  1408. continue
  1409. lines = block.split("\n")
  1410. kid, types, helpful, harmful, score, task, content_lines = None, [], 0, 0, 3, "", []
  1411. current_field = None
  1412. for line in lines:
  1413. if line.startswith("ID:"):
  1414. kid = line[3:].strip()
  1415. current_field = None
  1416. elif line.startswith("TYPES:"):
  1417. types_str = line[6:].strip()
  1418. types = [t.strip() for t in types_str.split(",") if t.strip()]
  1419. current_field = None
  1420. elif line.startswith("HELPFUL:"):
  1421. try:
  1422. helpful = int(line[8:].strip())
  1423. except Exception:
  1424. helpful = 0
  1425. current_field = None
  1426. elif line.startswith("HARMFUL:"):
  1427. try:
  1428. harmful = int(line[8:].strip())
  1429. except Exception:
  1430. harmful = 0
  1431. current_field = None
  1432. elif line.startswith("SCORE:"):
  1433. try:
  1434. score = int(line[6:].strip())
  1435. except Exception:
  1436. score = 3
  1437. current_field = None
  1438. elif line.startswith("TASK:"):
  1439. task = line[5:].strip()
  1440. current_field = "task"
  1441. elif line.startswith("CONTENT:"):
  1442. content_lines.append(line[8:].strip())
  1443. current_field = "content"
  1444. elif current_field == "task":
  1445. task += "\n" + line
  1446. elif current_field == "content":
  1447. content_lines.append(line)
  1448. if kid and content_lines:
  1449. new_entries.append({
  1450. "id": kid,
  1451. "types": types if types else ["strategy"],
  1452. "helpful": helpful,
  1453. "harmful": harmful,
  1454. "score": score,
  1455. "task": task.strip(),
  1456. "content": "\n".join(content_lines).strip()
  1457. })
  1458. if not new_entries:
  1459. raise HTTPException(status_code=500, detail="解析大模型输出失败")
  1460. # 生成向量并重建知识库
  1461. print(f"[知识瘦身] 正在为 {len(new_entries)} 条知识生成向量...")
  1462. # 批量生成向量(只基于 task)
  1463. texts = [e['task'] for e in new_entries]
  1464. embeddings = await get_embeddings_batch(texts)
  1465. # 清空并重建(PostgreSQL使用TRUNCATE)
  1466. cursor = pg_store._get_cursor()
  1467. try:
  1468. cursor.execute("TRUNCATE TABLE knowledge")
  1469. pg_store.conn.commit()
  1470. finally:
  1471. cursor.close()
  1472. knowledge_list = []
  1473. for e, embedding in zip(new_entries, embeddings):
  1474. eval_data = {
  1475. "score": e["score"],
  1476. "helpful": e["helpful"],
  1477. "harmful": e["harmful"],
  1478. "confidence": 0.9,
  1479. "helpful_history": [],
  1480. "harmful_history": []
  1481. }
  1482. source = {
  1483. "name": "slim",
  1484. "category": "exp",
  1485. "urls": [],
  1486. "agent_id": "slim",
  1487. "submitted_by": "system",
  1488. "timestamp": datetime.now(timezone.utc).isoformat()
  1489. }
  1490. knowledge_list.append({
  1491. "id": e["id"],
  1492. "task_embedding": embedding,
  1493. "message_id": "",
  1494. "task": e["task"],
  1495. "content": e["content"],
  1496. "types": e["types"],
  1497. "tags": {},
  1498. "tag_keys": [],
  1499. "scopes": ["org:cybertogether"],
  1500. "owner": "agent:slim",
  1501. "resource_ids": [],
  1502. "source": source,
  1503. "eval": eval_data,
  1504. "created_at": now,
  1505. "updated_at": now,
  1506. "status": "approved",
  1507. "relationships": json.dumps([])
  1508. })
  1509. pg_store.insert_batch(knowledge_list)
  1510. result_msg = f"瘦身完成:{len(all_knowledge)} → {len(new_entries)} 条知识"
  1511. if report_line:
  1512. result_msg += f"\n{report_line}"
  1513. print(f"[知识瘦身] {result_msg}")
  1514. return {"status": "ok", "before": len(all_knowledge), "after": len(new_entries), "report": report_line}
  1515. except HTTPException:
  1516. raise
  1517. except Exception as e:
  1518. print(f"[Slim Knowledge] 错误: {e}")
  1519. raise HTTPException(status_code=500, detail=str(e))
  1520. @app.post("/api/extract")
  1521. async def extract_knowledge_from_messages(extract_req: MessageExtractIn, background_tasks: BackgroundTasks):
  1522. """从消息历史中提取知识(LLM 分析)"""
  1523. if not extract_req.submitted_by:
  1524. raise HTTPException(status_code=400, detail="submitted_by is required")
  1525. messages = extract_req.messages
  1526. if not messages or len(messages) == 0:
  1527. return {"status": "ok", "extracted_count": 0, "knowledge_ids": []}
  1528. # 构造消息历史文本
  1529. messages_text = ""
  1530. for msg in messages:
  1531. role = msg.get("role", "unknown")
  1532. content = msg.get("content", "")
  1533. messages_text += f"[{role}]: {content}\n\n"
  1534. # LLM 提取知识
  1535. prompt = MESSAGE_EXTRACT_PROMPT_TEMPLATE.format(messages_text=messages_text)
  1536. try:
  1537. print(f"\n[Extract] 正在从 {len(messages)} 条消息中提取知识...")
  1538. response = await _dedup_llm(
  1539. messages=[{"role": "user", "content": prompt}],
  1540. )
  1541. content = response.get("content", "").strip()
  1542. # 尝试解析 JSON
  1543. # 移除可能的 markdown 代码块标记
  1544. if content.startswith("```json"):
  1545. content = content[7:]
  1546. if content.startswith("```"):
  1547. content = content[3:]
  1548. if content.endswith("```"):
  1549. content = content[:-3]
  1550. content = content.strip()
  1551. extracted_knowledge = json.loads(content)
  1552. if not isinstance(extracted_knowledge, list):
  1553. raise ValueError("LLM output is not a list")
  1554. if not extracted_knowledge:
  1555. return {"status": "ok", "extracted_count": 0, "knowledge_ids": []}
  1556. # 批量生成向量(只基于 task)
  1557. texts = [item.get('task', '') for item in extracted_knowledge]
  1558. embeddings = await get_embeddings_batch(texts)
  1559. # 保存提取的知识
  1560. knowledge_ids = []
  1561. now = int(time.time())
  1562. knowledge_list = []
  1563. for item, embedding in zip(extracted_knowledge, embeddings):
  1564. task = item.get("task", "")
  1565. knowledge_content = item.get("content", "")
  1566. types = item.get("types", ["strategy"])
  1567. score = item.get("score", 3)
  1568. if not task or not knowledge_content:
  1569. continue
  1570. # 生成 ID
  1571. timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
  1572. random_suffix = uuid.uuid4().hex[:4]
  1573. knowledge_id = f"knowledge-{timestamp}-{random_suffix}"
  1574. # 准备数据
  1575. source = {
  1576. "name": "message_extraction",
  1577. "category": "exp",
  1578. "urls": [],
  1579. "agent_id": extract_req.agent_id,
  1580. "submitted_by": extract_req.submitted_by,
  1581. "timestamp": datetime.now(timezone.utc).isoformat(),
  1582. "session_key": extract_req.session_key
  1583. }
  1584. eval_data = {
  1585. "score": score,
  1586. "helpful": 1,
  1587. "harmful": 0,
  1588. "confidence": 0.7,
  1589. "helpful_history": [],
  1590. "harmful_history": []
  1591. }
  1592. knowledge_list.append({
  1593. "id": knowledge_id,
  1594. "task_embedding": embedding,
  1595. "message_id": "",
  1596. "task": task,
  1597. "content": knowledge_content,
  1598. "types": types,
  1599. "tags": {},
  1600. "tag_keys": [],
  1601. "scopes": ["org:cybertogether"],
  1602. "owner": extract_req.submitted_by,
  1603. "resource_ids": [],
  1604. "source": source,
  1605. "eval": eval_data,
  1606. "created_at": now,
  1607. "updated_at": now,
  1608. "status": "pending",
  1609. "relationships": json.dumps([]),
  1610. })
  1611. knowledge_ids.append(knowledge_id)
  1612. # 批量插入
  1613. if knowledge_list:
  1614. pg_store.insert_batch(knowledge_list)
  1615. background_tasks.add_task(knowledge_processor.process_pending)
  1616. print(f"[Extract] 成功提取并保存 {len(knowledge_ids)} 条知识")
  1617. return {
  1618. "status": "ok",
  1619. "extracted_count": len(knowledge_ids),
  1620. "knowledge_ids": knowledge_ids
  1621. }
  1622. except json.JSONDecodeError as e:
  1623. print(f"[Extract] JSON 解析失败: {e}")
  1624. print(f"[Extract] LLM 输出: {content[:500]}")
  1625. return {"status": "error", "error": "Failed to parse LLM output", "extracted_count": 0}
  1626. except Exception as e:
  1627. print(f"[Extract] 提取失败: {e}")
  1628. return {"status": "error", "error": str(e), "extracted_count": 0}
  1629. # ===== Tool API =====
  1630. @app.post("/api/tool", status_code=201)
  1631. async def create_tool(tool: ToolIn):
  1632. """创建或更新工具"""
  1633. try:
  1634. now = int(time.time())
  1635. embedding = await get_embedding(f"{tool.name} {tool.introduction}")
  1636. pg_tool_store.insert_or_update({
  1637. 'id': tool.id,
  1638. 'name': tool.name,
  1639. 'version': tool.version,
  1640. 'introduction': tool.introduction,
  1641. 'tutorial': tool.tutorial,
  1642. 'input': tool.input,
  1643. 'output': tool.output,
  1644. 'updated_time': now,
  1645. 'status': tool.status,
  1646. 'capabilities': tool.capabilities,
  1647. 'tool_knowledge': tool.tool_knowledge,
  1648. 'case_knowledge': tool.case_knowledge,
  1649. 'process_knowledge': tool.process_knowledge,
  1650. 'embedding': embedding,
  1651. })
  1652. return {"status": "ok", "id": tool.id}
  1653. except Exception as e:
  1654. raise HTTPException(status_code=500, detail=str(e))
  1655. @app.get("/api/tool")
  1656. def list_tools(
  1657. status: Optional[str] = Query(None),
  1658. limit: int = Query(100, ge=1, le=1000),
  1659. offset: int = Query(0, ge=0),
  1660. ):
  1661. """列出工具"""
  1662. try:
  1663. results = pg_tool_store.list_all(limit=limit, offset=offset, status=status)
  1664. total = pg_tool_store.count(status=status)
  1665. return {"results": results, "total": total}
  1666. except Exception as e:
  1667. raise HTTPException(status_code=500, detail=str(e))
  1668. @app.get("/api/tool/search")
  1669. async def search_tools(
  1670. q: str = Query(..., description="查询文本"),
  1671. top_k: int = Query(5, ge=1, le=100),
  1672. status: Optional[str] = None,
  1673. ):
  1674. """向量检索工具"""
  1675. try:
  1676. query_embedding = await get_embedding(q)
  1677. results = pg_tool_store.search(query_embedding, limit=top_k, status=status)
  1678. return {"results": results, "count": len(results)}
  1679. except Exception as e:
  1680. raise HTTPException(status_code=500, detail=str(e))
  1681. @app.get("/api/tool/{tool_id:path}")
  1682. def get_tool(tool_id: str):
  1683. """获取单个工具详情"""
  1684. try:
  1685. result = pg_tool_store.get_by_id(tool_id)
  1686. if not result:
  1687. raise HTTPException(status_code=404, detail=f"Tool not found: {tool_id}")
  1688. return result
  1689. except HTTPException:
  1690. raise
  1691. except Exception as e:
  1692. raise HTTPException(status_code=500, detail=str(e))
  1693. @app.patch("/api/tool/{tool_id:path}")
  1694. async def patch_tool(tool_id: str, patch: ToolPatchIn):
  1695. """更新工具字段"""
  1696. try:
  1697. if not pg_tool_store.get_by_id(tool_id):
  1698. raise HTTPException(status_code=404, detail=f"Tool not found: {tool_id}")
  1699. updates = {}
  1700. need_reembed = False
  1701. for field in ('name', 'version', 'introduction', 'tutorial', 'input', 'output',
  1702. 'status', 'capabilities', 'tool_knowledge', 'case_knowledge', 'process_knowledge'):
  1703. value = getattr(patch, field)
  1704. if value is not None:
  1705. updates[field] = value
  1706. if field in ('name', 'introduction'):
  1707. need_reembed = True
  1708. if not updates:
  1709. return {"status": "ok", "id": tool_id}
  1710. updates['updated_time'] = int(time.time())
  1711. if need_reembed:
  1712. existing = pg_tool_store.get_by_id(tool_id)
  1713. name = updates.get('name', existing['name'])
  1714. intro = updates.get('introduction', existing['introduction'])
  1715. updates['embedding'] = await get_embedding(f"{name} {intro}")
  1716. pg_tool_store.update(tool_id, updates)
  1717. return {"status": "ok", "id": tool_id}
  1718. except HTTPException:
  1719. raise
  1720. except Exception as e:
  1721. raise HTTPException(status_code=500, detail=str(e))
  1722. @app.delete("/api/tool/{tool_id:path}")
  1723. def delete_tool(tool_id: str):
  1724. """删除工具"""
  1725. try:
  1726. if not pg_tool_store.get_by_id(tool_id):
  1727. raise HTTPException(status_code=404, detail=f"Tool not found: {tool_id}")
  1728. pg_tool_store.delete(tool_id)
  1729. return {"status": "ok", "id": tool_id}
  1730. except HTTPException:
  1731. raise
  1732. except Exception as e:
  1733. raise HTTPException(status_code=500, detail=str(e))
  1734. # ===== Capability API =====
  1735. @app.post("/api/capability", status_code=201)
  1736. async def create_capability(cap: CapabilityIn):
  1737. """创建或更新原子能力"""
  1738. try:
  1739. embedding = await get_embedding(f"{cap.name} {cap.description}")
  1740. pg_capability_store.insert_or_update({
  1741. 'id': cap.id,
  1742. 'name': cap.name,
  1743. 'criterion': cap.criterion,
  1744. 'description': cap.description,
  1745. 'requirements': cap.requirements,
  1746. 'implements': cap.implements,
  1747. 'tools': cap.tools,
  1748. 'source_knowledge': cap.source_knowledge,
  1749. 'embedding': embedding,
  1750. })
  1751. return {"status": "ok", "id": cap.id}
  1752. except Exception as e:
  1753. raise HTTPException(status_code=500, detail=str(e))
  1754. @app.get("/api/capability")
  1755. def list_capabilities(
  1756. limit: int = Query(100, ge=1, le=1000),
  1757. offset: int = Query(0, ge=0),
  1758. ):
  1759. """列出原子能力"""
  1760. try:
  1761. results = pg_capability_store.list_all(limit=limit, offset=offset)
  1762. total = pg_capability_store.count()
  1763. return {"results": results, "total": total}
  1764. except Exception as e:
  1765. raise HTTPException(status_code=500, detail=str(e))
  1766. @app.get("/api/capability/search")
  1767. async def search_capabilities(
  1768. q: str = Query(..., description="查询文本"),
  1769. top_k: int = Query(5, ge=1, le=100),
  1770. ):
  1771. """向量检索原子能力"""
  1772. try:
  1773. query_embedding = await get_embedding(q)
  1774. results = pg_capability_store.search(query_embedding, limit=top_k)
  1775. return {"results": results, "count": len(results)}
  1776. except Exception as e:
  1777. raise HTTPException(status_code=500, detail=str(e))
  1778. @app.get("/api/capability/{cap_id}")
  1779. def get_capability(cap_id: str):
  1780. """获取单个原子能力"""
  1781. try:
  1782. result = pg_capability_store.get_by_id(cap_id)
  1783. if not result:
  1784. raise HTTPException(status_code=404, detail=f"Capability not found: {cap_id}")
  1785. return result
  1786. except HTTPException:
  1787. raise
  1788. except Exception as e:
  1789. raise HTTPException(status_code=500, detail=str(e))
  1790. @app.patch("/api/capability/{cap_id}")
  1791. async def patch_capability(cap_id: str, patch: CapabilityPatchIn):
  1792. """更新原子能力字段"""
  1793. try:
  1794. existing = pg_capability_store.get_by_id(cap_id)
  1795. if not existing:
  1796. raise HTTPException(status_code=404, detail=f"Capability not found: {cap_id}")
  1797. updates = {}
  1798. need_reembed = False
  1799. for field in ('name', 'criterion', 'description', 'requirements',
  1800. 'implements', 'tools', 'source_knowledge'):
  1801. value = getattr(patch, field)
  1802. if value is not None:
  1803. updates[field] = value
  1804. if field in ('name', 'description'):
  1805. need_reembed = True
  1806. if not updates:
  1807. return {"status": "ok", "id": cap_id}
  1808. if need_reembed:
  1809. name = updates.get('name', existing['name'])
  1810. desc = updates.get('description', existing['description'])
  1811. updates['embedding'] = await get_embedding(f"{name} {desc}")
  1812. pg_capability_store.update(cap_id, updates)
  1813. return {"status": "ok", "id": cap_id}
  1814. except HTTPException:
  1815. raise
  1816. except Exception as e:
  1817. raise HTTPException(status_code=500, detail=str(e))
  1818. @app.delete("/api/capability/{cap_id}")
  1819. def delete_capability(cap_id: str):
  1820. """删除原子能力"""
  1821. try:
  1822. if not pg_capability_store.get_by_id(cap_id):
  1823. raise HTTPException(status_code=404, detail=f"Capability not found: {cap_id}")
  1824. pg_capability_store.delete(cap_id)
  1825. return {"status": "ok", "id": cap_id}
  1826. except HTTPException:
  1827. raise
  1828. except Exception as e:
  1829. raise HTTPException(status_code=500, detail=str(e))
  1830. # ===== Requirement API =====
  1831. @app.post("/api/requirement", status_code=201)
  1832. async def create_requirement(req: RequirementIn):
  1833. """创建或更新需求"""
  1834. try:
  1835. embedding = await get_embedding(req.description)
  1836. pg_requirement_store.insert_or_update({
  1837. 'id': req.id,
  1838. 'description': req.description,
  1839. 'atomics': req.atomics,
  1840. 'source_nodes': req.source_nodes,
  1841. 'status': req.status,
  1842. 'match_result': req.match_result,
  1843. 'embedding': embedding,
  1844. })
  1845. return {"status": "ok", "id": req.id}
  1846. except Exception as e:
  1847. raise HTTPException(status_code=500, detail=str(e))
  1848. @app.get("/api/requirement")
  1849. def list_requirements(
  1850. status: Optional[str] = Query(None),
  1851. limit: int = Query(100, ge=1, le=1000),
  1852. offset: int = Query(0, ge=0),
  1853. ):
  1854. """列出需求"""
  1855. try:
  1856. results = pg_requirement_store.list_all(limit=limit, offset=offset, status=status)
  1857. total = pg_requirement_store.count(status=status)
  1858. return {"results": results, "total": total}
  1859. except Exception as e:
  1860. raise HTTPException(status_code=500, detail=str(e))
  1861. @app.get("/api/requirement/search")
  1862. async def search_requirements(
  1863. q: str = Query(..., description="查询文本"),
  1864. top_k: int = Query(5, ge=1, le=100),
  1865. ):
  1866. """向量检索需求"""
  1867. try:
  1868. query_embedding = await get_embedding(q)
  1869. results = pg_requirement_store.search(query_embedding, limit=top_k)
  1870. return {"results": results, "count": len(results)}
  1871. except Exception as e:
  1872. raise HTTPException(status_code=500, detail=str(e))
  1873. @app.get("/api/requirement/{req_id}")
  1874. def get_requirement(req_id: str):
  1875. """获取单个需求"""
  1876. try:
  1877. result = pg_requirement_store.get_by_id(req_id)
  1878. if not result:
  1879. raise HTTPException(status_code=404, detail=f"Requirement not found: {req_id}")
  1880. return result
  1881. except HTTPException:
  1882. raise
  1883. except Exception as e:
  1884. raise HTTPException(status_code=500, detail=str(e))
  1885. @app.patch("/api/requirement/{req_id}")
  1886. async def patch_requirement(req_id: str, patch: RequirementPatchIn):
  1887. """更新需求字段"""
  1888. try:
  1889. existing = pg_requirement_store.get_by_id(req_id)
  1890. if not existing:
  1891. raise HTTPException(status_code=404, detail=f"Requirement not found: {req_id}")
  1892. updates = {}
  1893. need_reembed = False
  1894. for field in ('description', 'atomics', 'source_nodes', 'status', 'match_result'):
  1895. value = getattr(patch, field)
  1896. if value is not None:
  1897. updates[field] = value
  1898. if field == 'description':
  1899. need_reembed = True
  1900. if not updates:
  1901. return {"status": "ok", "id": req_id}
  1902. if need_reembed:
  1903. updates['embedding'] = await get_embedding(updates['description'])
  1904. pg_requirement_store.update(req_id, updates)
  1905. return {"status": "ok", "id": req_id}
  1906. except HTTPException:
  1907. raise
  1908. except Exception as e:
  1909. raise HTTPException(status_code=500, detail=str(e))
  1910. @app.delete("/api/requirement/{req_id}")
  1911. def delete_requirement(req_id: str):
  1912. """删除需求"""
  1913. try:
  1914. if not pg_requirement_store.get_by_id(req_id):
  1915. raise HTTPException(status_code=404, detail=f"Requirement not found: {req_id}")
  1916. pg_requirement_store.delete(req_id)
  1917. return {"status": "ok", "id": req_id}
  1918. except HTTPException:
  1919. raise
  1920. except Exception as e:
  1921. raise HTTPException(status_code=500, detail=str(e))
  1922. @app.get("/")
  1923. def frontend():
  1924. """KnowHub 管理前端"""
  1925. index_file = STATIC_DIR / "index.html"
  1926. if not index_file.exists():
  1927. return HTMLResponse("<h1>KnowHub Frontend Not Found</h1><p>Please ensure knowhub/frontend/dist/index.html exists. Run 'yarn build' in frontend directory.</p>", status_code=404)
  1928. return FileResponse(str(index_file))
  1929. @app.get("/category_tree.json")
  1930. def serve_category_tree():
  1931. """类别树JSON数据"""
  1932. tree_file = STATIC_DIR / "category_tree.json"
  1933. if not tree_file.exists():
  1934. return {"error": "Not Found"}
  1935. return FileResponse(str(tree_file))
  1936. if __name__ == "__main__":
  1937. import uvicorn
  1938. uvicorn.run(app, host="0.0.0.0", port=9999)