server.py 82 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329
  1. """
  2. KnowHub Server
  3. Agent 工具使用经验的共享平台。
  4. FastAPI + Milvus Lite(知识)+ SQLite(资源),单文件部署。
  5. """
  6. import os
  7. import re
  8. import json
  9. import asyncio
  10. import base64
  11. import time
  12. import uuid
  13. from contextlib import asynccontextmanager
  14. from datetime import datetime, timezone
  15. from typing import Optional, List, Dict
  16. from pathlib import Path
  17. from cryptography.hazmat.primitives.ciphers.aead import AESGCM
  18. from fastapi import FastAPI, HTTPException, Query, Header, Body, BackgroundTasks
  19. from fastapi.responses import HTMLResponse, FileResponse
  20. from fastapi.staticfiles import StaticFiles
  21. from pydantic import BaseModel, Field
  22. # 导入 LLM 调用(需要 agent 模块在 Python path 中)
  23. import sys
  24. sys.path.insert(0, str(Path(__file__).parent.parent))
  25. # 加载环境变量
  26. from dotenv import load_dotenv
  27. load_dotenv(Path(__file__).parent.parent / ".env")
  28. from agent.llm import create_openrouter_llm_call, create_qwen_llm_call
  29. from knowhub.kb_manage_prompts import (
  30. DEDUP_RELATION_PROMPT,
  31. TOOL_ANALYSIS_PROMPT,
  32. RERANK_PROMPT_TEMPLATE,
  33. KNOWLEDGE_EVOLVE_PROMPT_TEMPLATE,
  34. KNOWLEDGE_SLIM_PROMPT_TEMPLATE,
  35. MESSAGE_EXTRACT_PROMPT_TEMPLATE,
  36. )
  37. _dedup_llm = create_openrouter_llm_call(model="google/gemini-2.5-flash-lite")
  38. _tool_analysis_llm = create_qwen_llm_call(model="qwen3.5-plus")
  39. # 导入向量存储和 embedding
  40. from knowhub.knowhub_db.pg_store import PostgreSQLStore
  41. from knowhub.knowhub_db.pg_resource_store import PostgreSQLResourceStore
  42. from knowhub.knowhub_db.pg_tool_store import PostgreSQLToolStore
  43. from knowhub.knowhub_db.pg_capability_store import PostgreSQLCapabilityStore
  44. from knowhub.knowhub_db.pg_requirement_store import PostgreSQLRequirementStore
  45. from knowhub.embeddings import get_embedding, get_embeddings_batch
  46. BRAND_NAME = os.getenv("BRAND_NAME", "KnowHub")
  47. BRAND_API_ENV = os.getenv("BRAND_API_ENV", "KNOWHUB_API")
  48. BRAND_DB = os.getenv("BRAND_DB", "knowhub.db")
  49. # 组织密钥配置(格式:org1:key1_base64,org2:key2_base64)
  50. ORG_KEYS_RAW = os.getenv("ORG_KEYS", "")
  51. ORG_KEYS = {}
  52. if ORG_KEYS_RAW:
  53. for pair in ORG_KEYS_RAW.split(","):
  54. if ":" in pair:
  55. org, key_b64 = pair.split(":", 1)
  56. ORG_KEYS[org.strip()] = key_b64.strip()
  57. DB_PATH = Path(__file__).parent / BRAND_DB
  58. # 全局 PostgreSQL 存储实例
  59. pg_store: Optional[PostgreSQLStore] = None
  60. pg_resource_store: Optional[PostgreSQLResourceStore] = None
  61. pg_tool_store: Optional[PostgreSQLToolStore] = None
  62. pg_capability_store: Optional[PostgreSQLCapabilityStore] = None
  63. pg_requirement_store: Optional[PostgreSQLRequirementStore] = None
  64. # --- 加密/解密 ---
  65. def get_org_key(resource_id: str) -> Optional[bytes]:
  66. """从content_id提取组织前缀,返回对应密钥"""
  67. if "/" in resource_id:
  68. org = resource_id.split("/")[0]
  69. if org in ORG_KEYS:
  70. return base64.b64decode(ORG_KEYS[org])
  71. return None
  72. def encrypt_content(resource_id: str, plaintext: str) -> str:
  73. """加密内容,返回格式:encrypted:AES256-GCM:{base64_data}"""
  74. if not plaintext:
  75. return ""
  76. key = get_org_key(resource_id)
  77. if not key:
  78. # 没有配置密钥,明文存储(不推荐)
  79. return plaintext
  80. aesgcm = AESGCM(key)
  81. nonce = os.urandom(12) # 96-bit nonce
  82. ciphertext = aesgcm.encrypt(nonce, plaintext.encode("utf-8"), None)
  83. # 组合 nonce + ciphertext
  84. encrypted_data = nonce + ciphertext
  85. encoded = base64.b64encode(encrypted_data).decode("ascii")
  86. return f"encrypted:AES256-GCM:{encoded}"
  87. def decrypt_content(resource_id: str, encrypted_text: str, provided_key: Optional[str] = None) -> str:
  88. """解密内容,如果没有提供密钥或密钥错误,返回[ENCRYPTED]"""
  89. if not encrypted_text:
  90. return ""
  91. if not encrypted_text.startswith("encrypted:AES256-GCM:"):
  92. # 未加密的内容,直接返回
  93. return encrypted_text
  94. # 提取加密数据
  95. encoded = encrypted_text.split(":", 2)[2]
  96. encrypted_data = base64.b64decode(encoded)
  97. nonce = encrypted_data[:12]
  98. ciphertext = encrypted_data[12:]
  99. # 获取密钥
  100. key = None
  101. if provided_key:
  102. # 使用提供的密钥
  103. try:
  104. key = base64.b64decode(provided_key)
  105. except Exception:
  106. return "[ENCRYPTED]"
  107. else:
  108. # 从配置中获取
  109. key = get_org_key(resource_id)
  110. if not key:
  111. return "[ENCRYPTED]"
  112. try:
  113. aesgcm = AESGCM(key)
  114. plaintext = aesgcm.decrypt(nonce, ciphertext, None)
  115. return plaintext.decode("utf-8")
  116. except Exception:
  117. return "[ENCRYPTED]"
  118. def serialize_milvus_result(data):
  119. """将 Milvus 返回的数据转换为可序列化的字典"""
  120. # 基本类型直接返回
  121. if data is None or isinstance(data, (str, int, float, bool)):
  122. return data
  123. # 字典类型递归处理
  124. if isinstance(data, dict):
  125. return {k: serialize_milvus_result(v) for k, v in data.items()}
  126. # 列表/元组类型递归处理
  127. if isinstance(data, (list, tuple)):
  128. return [serialize_milvus_result(item) for item in data]
  129. # 尝试转换为字典(对于有 to_dict 方法的对象)
  130. if hasattr(data, 'to_dict') and callable(getattr(data, 'to_dict')):
  131. try:
  132. return serialize_milvus_result(data.to_dict())
  133. except:
  134. pass
  135. # 尝试转换为列表(对于可迭代对象,如 RepeatedScalarContainer)
  136. if hasattr(data, '__iter__') and not isinstance(data, (str, bytes, dict)):
  137. try:
  138. # 强制转换为列表并递归处理
  139. result = []
  140. for item in data:
  141. result.append(serialize_milvus_result(item))
  142. return result
  143. except:
  144. pass
  145. # 尝试获取对象的属性字典
  146. if hasattr(data, '__dict__'):
  147. try:
  148. return serialize_milvus_result(vars(data))
  149. except:
  150. pass
  151. # 最后的 fallback:对于无法处理的类型,返回 None 而不是字符串表示
  152. # 这样可以避免产生无法序列化的字符串
  153. return None
  154. # --- Models ---
  155. class ResourceIn(BaseModel):
  156. id: str
  157. title: str = ""
  158. body: str
  159. secure_body: str = ""
  160. content_type: str = "text" # text|code|credential|cookie
  161. metadata: dict = {}
  162. sort_order: int = 0
  163. submitted_by: str = ""
  164. class ResourcePatchIn(BaseModel):
  165. """PATCH /api/resource/{id} 请求体"""
  166. title: Optional[str] = None
  167. body: Optional[str] = None
  168. secure_body: Optional[str] = None
  169. content_type: Optional[str] = None
  170. metadata: Optional[dict] = None
  171. # Knowledge Models
  172. class KnowledgeIn(BaseModel):
  173. task: str
  174. content: str
  175. types: list[str] = ["strategy"]
  176. tags: dict = {}
  177. scopes: list[str] = ["org:cybertogether"]
  178. owner: str = ""
  179. message_id: str = ""
  180. resource_ids: list[str] = []
  181. source: dict = {} # {name, category, urls, agent_id, submitted_by, timestamp}
  182. eval: dict = {} # {score, helpful, harmful, confidence}
  183. support_capability: list[str] = []
  184. tools: list[str] = []
  185. class KnowledgeOut(BaseModel):
  186. id: str
  187. message_id: str
  188. types: list[str]
  189. task: str
  190. tags: dict
  191. scopes: list[str]
  192. owner: str
  193. content: str
  194. resource_ids: list[str]
  195. source: dict
  196. eval: dict
  197. created_at: str
  198. updated_at: str
  199. class KnowledgeUpdateIn(BaseModel):
  200. add_helpful_case: Optional[dict] = None
  201. add_harmful_case: Optional[dict] = None
  202. update_score: Optional[int] = Field(default=None, ge=1, le=5)
  203. evolve_feedback: Optional[str] = None
  204. class KnowledgePatchIn(BaseModel):
  205. """PATCH /api/knowledge/{id} 请求体(直接字段编辑)"""
  206. task: Optional[str] = None
  207. content: Optional[str] = None
  208. types: Optional[list[str]] = None
  209. tags: Optional[dict] = None
  210. scopes: Optional[list[str]] = None
  211. owner: Optional[str] = None
  212. support_capability: Optional[list[str]] = None
  213. tools: Optional[list[str]] = None
  214. class MessageExtractIn(BaseModel):
  215. """POST /api/extract 请求体(消息历史提取)"""
  216. messages: list[dict] # [{role: str, content: str}, ...]
  217. agent_id: str = "unknown"
  218. submitted_by: str # 必填,作为 owner
  219. session_key: str = ""
  220. class KnowledgeBatchUpdateIn(BaseModel):
  221. feedback_list: list[dict]
  222. class KnowledgeVerifyIn(BaseModel):
  223. action: str # "approve" | "reject"
  224. verified_by: str = "user"
  225. class KnowledgeBatchVerifyIn(BaseModel):
  226. knowledge_ids: List[str]
  227. action: str # "approve"
  228. verified_by: str
  229. class KnowledgeSearchResponse(BaseModel):
  230. results: list[dict]
  231. count: int
  232. # --- Tool Models ---
  233. class ToolIn(BaseModel):
  234. id: str
  235. name: str = ""
  236. version: Optional[str] = None
  237. introduction: str = ""
  238. tutorial: str = ""
  239. input: dict | str = ""
  240. output: dict | str = ""
  241. status: str = "未接入"
  242. capabilities: list[str] = []
  243. tool_knowledge: list[str] = []
  244. case_knowledge: list[str] = []
  245. process_knowledge: list[str] = []
  246. class ToolPatchIn(BaseModel):
  247. name: Optional[str] = None
  248. version: Optional[str] = None
  249. introduction: Optional[str] = None
  250. tutorial: Optional[str] = None
  251. input: Optional[dict | str] = None
  252. output: Optional[dict | str] = None
  253. status: Optional[str] = None
  254. capabilities: Optional[list[str]] = None
  255. tool_knowledge: Optional[list[str]] = None
  256. case_knowledge: Optional[list[str]] = None
  257. process_knowledge: Optional[list[str]] = None
  258. # --- Capability Models ---
  259. class CapabilityIn(BaseModel):
  260. id: str
  261. name: str = ""
  262. criterion: str = ""
  263. description: str = ""
  264. requirements: list[str] = []
  265. implements: dict = {}
  266. tools: list[str] = []
  267. source_knowledge: list[str] = []
  268. class CapabilityPatchIn(BaseModel):
  269. name: Optional[str] = None
  270. criterion: Optional[str] = None
  271. description: Optional[str] = None
  272. requirements: Optional[list[str]] = None
  273. implements: Optional[dict] = None
  274. tools: Optional[list[str]] = None
  275. source_knowledge: Optional[list[str]] = None
  276. # --- Requirement Models ---
  277. class RequirementIn(BaseModel):
  278. id: str
  279. description: str = ""
  280. atomics: list[str] = []
  281. source_nodes: list[dict] = []
  282. status: str = "未满足"
  283. match_result: str = ""
  284. class RequirementPatchIn(BaseModel):
  285. description: Optional[str] = None
  286. atomics: Optional[list[str]] = None
  287. source_nodes: Optional[list[dict]] = None
  288. status: Optional[str] = None
  289. match_result: Optional[str] = None
  290. class ResourceNode(BaseModel):
  291. id: str
  292. title: str
  293. class ResourceOut(BaseModel):
  294. id: str
  295. title: str
  296. body: str
  297. secure_body: str = ""
  298. content_type: str = "text"
  299. metadata: dict = {}
  300. toc: Optional[ResourceNode] = None
  301. children: list[ResourceNode]
  302. prev: Optional[ResourceNode] = None
  303. next: Optional[ResourceNode] = None
  304. # --- Dedup: Globals & Prompt ---
  305. knowledge_processor: Optional["KnowledgeProcessor"] = None
  306. # --- Dedup: RelationCache ---
  307. class RelationCache:
  308. """关系缓存,存储在内存中"""
  309. def __init__(self):
  310. self._cache: Dict[str, List[str]] = {}
  311. def load(self) -> dict:
  312. return self._cache
  313. def save(self, cache: dict):
  314. self._cache = cache
  315. def add_relation(self, relation_type: str, knowledge_id: str):
  316. if relation_type not in self._cache:
  317. self._cache[relation_type] = []
  318. if knowledge_id not in self._cache[relation_type]:
  319. self._cache[relation_type].append(knowledge_id)
  320. # --- Dedup: KnowledgeProcessor ---
  321. class KnowledgeProcessor:
  322. def __init__(self):
  323. self._lock = asyncio.Lock()
  324. self._relation_cache = RelationCache()
  325. async def process_pending(self):
  326. """持续处理 pending 和 dedup_passed 知识直到队列为空,有锁防并发"""
  327. if self._lock.locked():
  328. return
  329. async with self._lock:
  330. # 第一阶段:处理 pending(去重)
  331. while True:
  332. try:
  333. pending = pg_store.query('status == "pending"', limit=50)
  334. except Exception as e:
  335. print(f"[KnowledgeProcessor] 查询 pending 失败: {e}")
  336. break
  337. if not pending:
  338. break
  339. for knowledge in pending:
  340. await self._process_one(knowledge)
  341. # 第二阶段:处理 dedup_passed(工具关联)
  342. while True:
  343. try:
  344. dedup_passed = pg_store.query('status == "dedup_passed"', limit=50)
  345. except Exception as e:
  346. print(f"[KnowledgeProcessor] 查询 dedup_passed 失败: {e}")
  347. break
  348. if not dedup_passed:
  349. break
  350. for knowledge in dedup_passed:
  351. await self._analyze_tool_relation(knowledge)
  352. async def _process_one(self, knowledge: dict):
  353. kid = knowledge["id"]
  354. now = int(time.time())
  355. # 乐观锁:pending → processing(时间戳存秒级)
  356. try:
  357. pg_store.update(kid, {"status": "processing", "updated_at": now})
  358. except Exception as e:
  359. print(f"[KnowledgeProcessor] 锁定 {kid} 失败: {e}")
  360. return
  361. try:
  362. # 向量召回 top-10(只召回 approved/checked)
  363. embedding = knowledge.get("task_embedding") or knowledge.get("embedding")
  364. if not embedding:
  365. embedding = await get_embedding(knowledge["task"])
  366. candidates = pg_store.search(
  367. query_embedding=embedding,
  368. filters='(status == "approved" or status == "checked")',
  369. limit=10
  370. )
  371. candidates = [c for c in candidates if c["id"] != kid]
  372. # 只保留相似度 >= 0.75 的候选,低于阈值的 task 语义差异太大,直接视为 none
  373. candidates = [c for c in candidates if c.get("score", 0) >= 0.75]
  374. if not candidates:
  375. pg_store.update(kid, {"status": "dedup_passed", "updated_at": now})
  376. return
  377. llm_result = await self._llm_judge_relations(knowledge, candidates)
  378. await self._apply_decision(knowledge, llm_result)
  379. except Exception as e:
  380. print(f"[KnowledgeProcessor] 处理 {kid} 失败: {e},回退到 pending")
  381. try:
  382. pg_store.update(kid, {"status": "pending", "updated_at": int(time.time())})
  383. except Exception:
  384. pass
  385. async def _llm_judge_relations(self, new_knowledge: dict, candidates: list) -> dict:
  386. existing_list = "\n".join([
  387. f"[{i+1}] ID: {c['id']} | Task: {c['task']} | Content: {c['content'][:300]}"
  388. for i, c in enumerate(candidates)
  389. ])
  390. prompt = DEDUP_RELATION_PROMPT.format(
  391. new_task=new_knowledge["task"],
  392. new_content=new_knowledge["content"],
  393. existing_list=existing_list
  394. )
  395. for attempt in range(3):
  396. try:
  397. response = await _dedup_llm(
  398. messages=[{"role": "user", "content": prompt}],
  399. )
  400. content = response.get("content", "").strip()
  401. # 清理 markdown 代码块
  402. if "```" in content:
  403. parts = content.split("```")
  404. for part in parts:
  405. part = part.strip()
  406. if part.startswith("json"):
  407. part = part[4:].strip()
  408. try:
  409. result = json.loads(part)
  410. if "final_decision" in result:
  411. content = part
  412. break
  413. except Exception:
  414. continue
  415. result = json.loads(content)
  416. assert result.get("final_decision") in ("approved", "rejected")
  417. return result
  418. except Exception as e:
  419. print(f"[LLM Judge] 第{attempt+1}次失败: {e}")
  420. if attempt < 2:
  421. await asyncio.sleep(1)
  422. return {"final_decision": "approved", "relations": []}
  423. async def _apply_decision(self, new_knowledge: dict, llm_result: dict):
  424. kid = new_knowledge["id"]
  425. final_decision = llm_result.get("final_decision", "approved")
  426. relations = llm_result.get("relations", [])
  427. now = int(time.time())
  428. # 强制规则:如果存在 duplicate 或 subset 关系,必须 rejected
  429. if any(rel.get("type") in ("duplicate", "subset") for rel in relations):
  430. final_decision = "rejected"
  431. if final_decision == "rejected":
  432. # 记录 rejected 知识的关系(便于溯源为什么被拒绝)
  433. rejected_relationships = []
  434. for rel in relations:
  435. old_id = rel.get("old_id")
  436. rel_type = rel.get("type", "none")
  437. if old_id and rel_type != "none":
  438. rejected_relationships.append({"type": rel_type, "target": old_id})
  439. if rel_type in ("duplicate", "subset") and old_id:
  440. try:
  441. old = pg_store.get_by_id(old_id)
  442. if not old:
  443. continue
  444. eval_data = old.get("eval") or {}
  445. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  446. helpful_history = eval_data.get("helpful_history") or []
  447. helpful_history.append({
  448. "source": "dedup",
  449. "related_id": kid,
  450. "relation_type": rel_type,
  451. "timestamp": now
  452. })
  453. eval_data["helpful_history"] = helpful_history
  454. pg_store.update(old_id, {"eval": eval_data, "updated_at": now})
  455. except Exception as e:
  456. print(f"[Apply Decision] 更新旧知识 {old_id} helpful 失败: {e}")
  457. pg_store.update(kid, {"status": "rejected", "relationships": json.dumps(rejected_relationships), "updated_at": now})
  458. else:
  459. new_relationships = []
  460. for rel in relations:
  461. rel_type = rel.get("type", "none")
  462. reverse_type = rel.get("reverse_type", "none")
  463. old_id = rel.get("old_id")
  464. if not old_id or rel_type == "none":
  465. continue
  466. new_relationships.append({"type": rel_type, "target": old_id})
  467. self._relation_cache.add_relation(rel_type, kid)
  468. self._relation_cache.add_relation(rel_type, old_id)
  469. if reverse_type and reverse_type != "none":
  470. try:
  471. old = pg_store.get_by_id(old_id)
  472. if old:
  473. old_rels = old.get("relationships") or []
  474. old_rels.append({"type": reverse_type, "target": kid})
  475. pg_store.update(old_id, {"relationships": json.dumps(old_rels), "updated_at": now})
  476. self._relation_cache.add_relation(reverse_type, old_id)
  477. self._relation_cache.add_relation(reverse_type, kid)
  478. except Exception as e:
  479. print(f"[Apply Decision] 更新旧知识关系 {old_id} 失败: {e}")
  480. pg_store.update(kid, {
  481. "status": "dedup_passed",
  482. "relationships": json.dumps(new_relationships),
  483. "updated_at": now
  484. })
  485. async def _llm_analyze_tools(self, knowledge: dict) -> dict:
  486. """使用 LLM 分析知识中涉及的工具(复用迁移脚本逻辑)"""
  487. task = (knowledge.get("task") or "")[:600]
  488. content = (knowledge.get("content") or "")[:1200]
  489. prompt = TOOL_ANALYSIS_PROMPT.format(task=task, content=content)
  490. try:
  491. response = await _tool_analysis_llm(
  492. messages=[{"role": "user", "content": prompt}],
  493. max_tokens=2048,
  494. temperature=0.1,
  495. )
  496. raw = (response.get("content") or "").strip()
  497. if raw.startswith("```"):
  498. lines = raw.split("\n")
  499. inner = []
  500. in_block = False
  501. for line in lines:
  502. if line.startswith("```"):
  503. in_block = not in_block
  504. continue
  505. if in_block:
  506. inner.append(line)
  507. raw = "\n".join(inner).strip()
  508. return json.loads(raw)
  509. except Exception as e:
  510. print(f"[Tool Analysis LLM] 调用失败: {e}")
  511. raise
  512. async def _create_or_get_tool_resource(self, tool_info: dict) -> Optional[str]:
  513. """创建或获取工具资源(存入 PostgreSQL tool_table)"""
  514. category = tool_info.get("category", "other")
  515. slug = tool_info.get("slug", "")
  516. if not slug:
  517. return None
  518. tool_id = f"tools/{category}/{slug}"
  519. now_ts = int(time.time())
  520. cursor = pg_store._get_cursor()
  521. try:
  522. cursor.execute("SELECT id FROM tool_table WHERE id = %s", (tool_id,))
  523. if cursor.fetchone():
  524. return tool_id
  525. cursor.execute("""
  526. INSERT INTO tool_table (id, name, version, introduction, tutorial, input, output,
  527. updated_time, status, tool_knowledge, case_knowledge, process_knowledge)
  528. VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
  529. """, (
  530. tool_id,
  531. tool_info.get("name", slug),
  532. tool_info.get("version") or None,
  533. tool_info.get("description", ""),
  534. tool_info.get("usage", ""),
  535. json.dumps(tool_info.get("input", "")),
  536. json.dumps(tool_info.get("output", "")),
  537. now_ts,
  538. tool_info.get("status", "未接入"),
  539. json.dumps([]),
  540. json.dumps([]),
  541. json.dumps([]),
  542. ))
  543. pg_store.conn.commit()
  544. print(f"[Tool Resource] 创建新工具: {tool_id}")
  545. return tool_id
  546. finally:
  547. cursor.close()
  548. async def _update_tool_knowledge_index(self, tool_id: str, knowledge_id: str):
  549. """更新工具的 tool_knowledge 关联索引(PostgreSQL tool_table)"""
  550. now_ts = int(time.time())
  551. cursor = pg_store._get_cursor()
  552. try:
  553. cursor.execute("SELECT tool_knowledge FROM tool_table WHERE id = %s", (tool_id,))
  554. row = cursor.fetchone()
  555. if not row:
  556. return
  557. knowledge_ids = row["tool_knowledge"] if isinstance(row["tool_knowledge"], list) else json.loads(row["tool_knowledge"] or "[]")
  558. if knowledge_id not in knowledge_ids:
  559. knowledge_ids.append(knowledge_id)
  560. cursor.execute(
  561. "UPDATE tool_table SET tool_knowledge = %s, updated_time = %s WHERE id = %s",
  562. (json.dumps(knowledge_ids), now_ts, tool_id)
  563. )
  564. pg_store.conn.commit()
  565. finally:
  566. cursor.close()
  567. async def _analyze_tool_relation(self, knowledge: dict):
  568. """分析知识与工具的关联关系"""
  569. kid = knowledge["id"]
  570. now = int(time.time())
  571. # 乐观锁:dedup_passed → analyzing
  572. try:
  573. pg_store.update(kid, {"status": "analyzing", "updated_at": now})
  574. except Exception as e:
  575. print(f"[Tool Analysis] 锁定 {kid} 失败: {e}")
  576. return
  577. try:
  578. tool_analysis = await self._llm_analyze_tools(knowledge)
  579. has_tools = bool(tool_analysis and tool_analysis.get("has_tools"))
  580. existing_tags = knowledge.get("tags") or {}
  581. has_tool_tag = existing_tags.get("tool") is True
  582. # 情况1:LLM 判定无工具,但有 tool tag → 重新分析一次
  583. if not has_tools and has_tool_tag:
  584. print(f"[Tool Analysis] {kid} LLM 判定无工具但有 tool tag,重新分析")
  585. tool_analysis = await self._llm_analyze_tools(knowledge)
  586. has_tools = bool(tool_analysis and tool_analysis.get("has_tools"))
  587. # 重新分析后仍然不一致 → 知识模糊,rejected
  588. if not has_tools:
  589. pg_store.update(kid, {"status": "rejected", "updated_at": now})
  590. print(f"[Tool Analysis] {kid} 两次判定不一致,知识模糊,rejected")
  591. return
  592. # 情况2:无工具且无 tool tag → 直接 approved
  593. if not has_tools:
  594. pg_store.update(kid, {"status": "approved", "updated_at": now})
  595. return
  596. # 情况3/4:有工具 → 创建资源并关联
  597. tool_ids = []
  598. for tool_info in (tool_analysis.get("tools") or []):
  599. tool_id = await self._create_or_get_tool_resource(tool_info)
  600. if tool_id:
  601. tool_ids.append(tool_id)
  602. existing_resource_ids = knowledge.get("resource_ids") or []
  603. updated_resource_ids = list(set(existing_resource_ids + tool_ids))
  604. updates: dict = {
  605. "status": "approved",
  606. "resource_ids": updated_resource_ids,
  607. "updated_at": now
  608. }
  609. # 有工具但无 tool tag → 添加 tag
  610. if not has_tool_tag:
  611. updated_tags = dict(existing_tags)
  612. updated_tags["tool"] = True
  613. updates["tags"] = updated_tags
  614. print(f"[Tool Analysis] {kid} 添加 tool tag")
  615. pg_store.update(kid, updates)
  616. for tool_id in tool_ids:
  617. await self._update_tool_knowledge_index(tool_id, kid)
  618. print(f"[Tool Analysis] {kid} 关联了 {len(tool_ids)} 个工具")
  619. except Exception as e:
  620. print(f"[Tool Analysis] {kid} 分析失败: {e},回退到 dedup_passed")
  621. try:
  622. pg_store.update(kid, {"status": "dedup_passed", "updated_at": int(time.time())})
  623. except Exception:
  624. pass
  625. async def _periodic_processor():
  626. """每60秒检测超时条目并回滚:processing(>5min)→pending,analyzing(>10min)→dedup_passed"""
  627. while True:
  628. await asyncio.sleep(60)
  629. try:
  630. now = int(time.time())
  631. # 回滚超时的 processing(5分钟 → pending)
  632. timeout_5min = now - 300
  633. processing = pg_store.query('status == "processing"', limit=200)
  634. for item in processing:
  635. updated_at = item.get("updated_at", 0) or 0
  636. updated_at_sec = updated_at // 1000 if updated_at > 1_000_000_000_000 else updated_at
  637. if updated_at_sec < timeout_5min:
  638. print(f"[Periodic] 回滚超时 processing → pending: {item['id']}")
  639. pg_store.update(item["id"], {"status": "pending", "updated_at": int(time.time())})
  640. # 回滚超时的 analyzing(10分钟 → dedup_passed)
  641. timeout_10min = now - 600
  642. analyzing = pg_store.query('status == "analyzing"', limit=200)
  643. for item in analyzing:
  644. updated_at = item.get("updated_at", 0) or 0
  645. updated_at_sec = updated_at // 1000 if updated_at > 1_000_000_000_000 else updated_at
  646. if updated_at_sec < timeout_10min:
  647. print(f"[Periodic] 回滚超时 analyzing → dedup_passed: {item['id']}")
  648. pg_store.update(item["id"], {"status": "dedup_passed", "updated_at": int(time.time())})
  649. except Exception as e:
  650. print(f"[Periodic] 定时任务错误: {e}")
  651. # --- App ---
  652. @asynccontextmanager
  653. async def lifespan(app: FastAPI):
  654. global pg_store, pg_resource_store, pg_tool_store, pg_capability_store, pg_requirement_store, knowledge_processor
  655. # 初始化 PostgreSQL(knowledge + resources + tools + capabilities + requirements)
  656. pg_store = PostgreSQLStore()
  657. pg_resource_store = PostgreSQLResourceStore()
  658. pg_tool_store = PostgreSQLToolStore()
  659. pg_capability_store = PostgreSQLCapabilityStore()
  660. pg_requirement_store = PostgreSQLRequirementStore()
  661. # 初始化去重处理器 + 启动定时兜底任务
  662. knowledge_processor = KnowledgeProcessor()
  663. periodic_task = asyncio.create_task(_periodic_processor())
  664. yield
  665. # 清理
  666. periodic_task.cancel()
  667. try:
  668. await periodic_task
  669. except asyncio.CancelledError:
  670. pass
  671. pg_store.close()
  672. pg_resource_store.close()
  673. pg_tool_store.close()
  674. pg_capability_store.close()
  675. pg_requirement_store.close()
  676. app = FastAPI(title=BRAND_NAME, lifespan=lifespan)
  677. # 挂载静态文件
  678. STATIC_DIR = Path(__file__).parent / "frontend" / "dist"
  679. if STATIC_DIR.exists():
  680. app.mount("/assets", StaticFiles(directory=str(STATIC_DIR / "assets")), name="assets")
  681. # --- Knowledge API ---
  682. @app.post("/api/resource", status_code=201)
  683. def submit_resource(resource: ResourceIn):
  684. """提交资源(存入 PostgreSQL resources 表)"""
  685. try:
  686. # 加密敏感内容
  687. encrypted_secure_body = encrypt_content(resource.id, resource.secure_body)
  688. pg_resource_store.insert_or_update({
  689. 'id': resource.id,
  690. 'title': resource.title,
  691. 'body': resource.body,
  692. 'secure_body': encrypted_secure_body,
  693. 'content_type': resource.content_type,
  694. 'metadata': resource.metadata,
  695. 'sort_order': resource.sort_order,
  696. 'submitted_by': resource.submitted_by
  697. })
  698. return {"status": "ok", "id": resource.id}
  699. except Exception as e:
  700. raise HTTPException(status_code=500, detail=str(e))
  701. @app.get("/api/resource/{resource_id:path}", response_model=ResourceOut)
  702. def get_resource(resource_id: str, x_org_key: Optional[str] = Header(None)):
  703. """获取资源详情(从 PostgreSQL)"""
  704. try:
  705. row = pg_resource_store.get_by_id(resource_id)
  706. if not row:
  707. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  708. # 解密敏感内容
  709. secure_body = decrypt_content(resource_id, row.get("secure_body", ""), x_org_key)
  710. # 计算导航上下文
  711. root_id = resource_id.split("/")[0] if "/" in resource_id else resource_id
  712. # TOC (根节点)
  713. toc = None
  714. if "/" in resource_id:
  715. toc_row = pg_resource_store.get_by_id(root_id)
  716. if toc_row:
  717. toc = ResourceNode(id=toc_row["id"], title=toc_row["title"])
  718. # Children (子节点)
  719. children_rows = pg_resource_store.list_resources(prefix=f"{resource_id}/", limit=1000)
  720. children = [ResourceNode(id=r["id"], title=r["title"]) for r in children_rows
  721. if r["id"].count("/") == resource_id.count("/") + 1]
  722. # Prev/Next (同级节点)
  723. prev_node, next_node = pg_resource_store.get_siblings(resource_id)
  724. prev = ResourceNode(id=prev_node["id"], title=prev_node["title"]) if prev_node else None
  725. next = ResourceNode(id=next_node["id"], title=next_node["title"]) if next_node else None
  726. return ResourceOut(
  727. id=row["id"],
  728. title=row["title"],
  729. body=row["body"],
  730. secure_body=secure_body,
  731. content_type=row["content_type"],
  732. metadata=row.get("metadata", {}),
  733. toc=toc,
  734. children=children,
  735. prev=prev,
  736. next=next,
  737. )
  738. except HTTPException:
  739. raise
  740. except Exception as e:
  741. raise HTTPException(status_code=500, detail=str(e))
  742. @app.patch("/api/resource/{resource_id:path}")
  743. def patch_resource(resource_id: str, patch: ResourcePatchIn):
  744. """更新resource字段(PostgreSQL)"""
  745. try:
  746. # 检查是否存在
  747. if not pg_resource_store.get_by_id(resource_id):
  748. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  749. # 构建更新字典
  750. updates = {}
  751. if patch.title is not None:
  752. updates['title'] = patch.title
  753. if patch.body is not None:
  754. updates['body'] = patch.body
  755. if patch.secure_body is not None:
  756. updates['secure_body'] = encrypt_content(resource_id, patch.secure_body)
  757. if patch.content_type is not None:
  758. updates['content_type'] = patch.content_type
  759. if patch.metadata is not None:
  760. updates['metadata'] = patch.metadata
  761. if not updates:
  762. return {"status": "ok", "message": "No fields to update"}
  763. pg_resource_store.update(resource_id, updates)
  764. return {"status": "ok", "id": resource_id}
  765. except HTTPException:
  766. raise
  767. except Exception as e:
  768. raise HTTPException(status_code=500, detail=str(e))
  769. @app.get("/api/resource")
  770. def list_resources(
  771. content_type: Optional[str] = Query(None),
  772. limit: int = Query(100, ge=1, le=1000)
  773. ):
  774. """列出所有resource(PostgreSQL)"""
  775. try:
  776. results = pg_resource_store.list_resources(
  777. content_type=content_type,
  778. limit=limit
  779. )
  780. return {"results": results, "count": len(results)}
  781. except Exception as e:
  782. raise HTTPException(status_code=500, detail=str(e))
  783. @app.delete("/api/resource/{resource_id:path}")
  784. def delete_resource(resource_id: str):
  785. """删除单个resource(PostgreSQL)"""
  786. try:
  787. if not pg_resource_store.get_by_id(resource_id):
  788. raise HTTPException(status_code=404, detail=f"Resource not found: {resource_id}")
  789. pg_resource_store.delete(resource_id)
  790. return {"status": "ok", "id": resource_id}
  791. except HTTPException:
  792. raise
  793. except Exception as e:
  794. raise HTTPException(status_code=500, detail=str(e))
  795. # --- Knowledge API ---
  796. # ===== Knowledge API =====
  797. async def _llm_rerank(query: str, candidates: list[dict], top_k: int) -> list[str]:
  798. """
  799. 使用 LLM 对候选知识进行精排
  800. Args:
  801. query: 查询文本
  802. candidates: 候选知识列表
  803. top_k: 返回数量
  804. Returns:
  805. 排序后的知识 ID 列表
  806. """
  807. if not candidates:
  808. return []
  809. # 构造 prompt
  810. candidates_text = "\n".join([
  811. f"[{i+1}] ID: {c['id']}\nTask: {c['task']}\nContent: {c['content'][:200]}..."
  812. for i, c in enumerate(candidates)
  813. ])
  814. prompt = RERANK_PROMPT_TEMPLATE.format(
  815. top_k=top_k,
  816. query=query,
  817. candidates_text=candidates_text
  818. )
  819. try:
  820. response = await _dedup_llm(
  821. messages=[{"role": "user", "content": prompt}],
  822. )
  823. content = response.get("content", "").strip()
  824. # 解析 ID 列表
  825. selected_ids = [
  826. idx.strip()
  827. for idx in re.split(r'[,\s]+', content)
  828. if idx.strip().startswith(("knowledge-", "research-"))
  829. ]
  830. return selected_ids[:top_k]
  831. except Exception as e:
  832. print(f"[LLM Rerank] 失败: {e}")
  833. return []
  834. @app.get("/api/knowledge/search")
  835. async def search_knowledge_api(
  836. q: str = Query(..., description="查询文本"),
  837. top_k: int = Query(default=5, ge=1, le=20),
  838. min_score: int = Query(default=3, ge=1, le=5),
  839. types: Optional[str] = None,
  840. owner: Optional[str] = None
  841. ):
  842. """检索知识(向量召回 + LLM 精排)"""
  843. try:
  844. # 1. 生成查询向量
  845. query_embedding = await get_embedding(q)
  846. # 2. 构建过滤表达式
  847. filters = []
  848. if types:
  849. type_list = [t.strip() for t in types.split(',') if t.strip()]
  850. for t in type_list:
  851. filters.append(f'array_contains(types, "{t}")')
  852. if owner:
  853. owner_list = [o.strip() for o in owner.split(',') if o.strip()]
  854. if len(owner_list) == 1:
  855. filters.append(f'owner == "{owner_list[0]}"')
  856. else:
  857. # 多个owner用OR连接
  858. owner_filters = [f'owner == "{o}"' for o in owner_list]
  859. filters.append(f'({" or ".join(owner_filters)})')
  860. # 添加 min_score 过滤
  861. filters.append(f'eval["score"] >= {min_score}')
  862. # 只搜索 approved 和 checked 的知识
  863. filters.append('(status == "approved" or status == "checked")')
  864. filter_expr = ' and '.join(filters) if filters else None
  865. # 3. 向量召回(3*k 个候选)
  866. recall_limit = top_k * 3
  867. candidates = pg_store.search(
  868. query_embedding=query_embedding,
  869. filters=filter_expr,
  870. limit=recall_limit
  871. )
  872. if not candidates:
  873. return {"results": [], "count": 0, "reranked": False}
  874. # 转换为可序列化的格式
  875. serialized_candidates = [serialize_milvus_result(c) for c in candidates]
  876. # 4. LLM 精排
  877. reranked_ids = await _llm_rerank(q, serialized_candidates, top_k)
  878. if reranked_ids:
  879. # 按 LLM 排序返回
  880. id_to_candidate = {c["id"]: c for c in serialized_candidates}
  881. results = [id_to_candidate[id] for id in reranked_ids if id in id_to_candidate]
  882. return {"results": results, "count": len(results), "reranked": True}
  883. else:
  884. # Fallback:直接返回向量召回的 top k
  885. print(f"[Knowledge Search] LLM 精排失败,fallback 到向量 top-{top_k}")
  886. return {"results": serialized_candidates[:top_k], "count": len(serialized_candidates[:top_k]), "reranked": False}
  887. except Exception as e:
  888. print(f"[Knowledge Search] 错误: {e}")
  889. raise HTTPException(status_code=500, detail=str(e))
  890. @app.post("/api/knowledge", status_code=201)
  891. async def save_knowledge(knowledge: KnowledgeIn, background_tasks: BackgroundTasks):
  892. """保存新知识"""
  893. try:
  894. # 生成 ID
  895. timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
  896. random_suffix = uuid.uuid4().hex[:4]
  897. knowledge_id = f"knowledge-{timestamp}-{random_suffix}"
  898. now = int(time.time())
  899. # 设置默认值
  900. owner = knowledge.owner or f"agent:{knowledge.source.get('agent_id', 'unknown')}"
  901. # 准备 source
  902. source = {
  903. "name": knowledge.source.get("name", ""),
  904. "category": knowledge.source.get("category", ""),
  905. "urls": knowledge.source.get("urls", []),
  906. "agent_id": knowledge.source.get("agent_id", "unknown"),
  907. "submitted_by": knowledge.source.get("submitted_by", ""),
  908. "timestamp": datetime.now(timezone.utc).isoformat(),
  909. "message_id": knowledge.message_id
  910. }
  911. # 准备 eval
  912. eval_data = {
  913. "score": knowledge.eval.get("score", 3),
  914. "helpful": knowledge.eval.get("helpful", 1),
  915. "harmful": knowledge.eval.get("harmful", 0),
  916. "confidence": knowledge.eval.get("confidence", 0.5),
  917. "helpful_history": [],
  918. "harmful_history": []
  919. }
  920. # 生成向量(task_embedding + content_embedding 双向量)
  921. task_embedding = await get_embedding(knowledge.task)
  922. content_embedding = await get_embedding(knowledge.content)
  923. # 提取 tag keys(用于高效筛选)
  924. tag_keys = list(knowledge.tags.keys()) if isinstance(knowledge.tags, dict) else []
  925. # 准备插入数据
  926. insert_data = {
  927. "id": knowledge_id,
  928. "task_embedding": task_embedding,
  929. "content_embedding": content_embedding,
  930. "message_id": knowledge.message_id,
  931. "task": knowledge.task,
  932. "content": knowledge.content,
  933. "types": knowledge.types,
  934. "tags": knowledge.tags,
  935. "tag_keys": tag_keys,
  936. "scopes": knowledge.scopes,
  937. "owner": owner,
  938. "resource_ids": knowledge.resource_ids,
  939. "source": source,
  940. "eval": eval_data,
  941. "created_at": now,
  942. "updated_at": now,
  943. "status": "pending",
  944. "relationships": json.dumps([]),
  945. "support_capability": knowledge.support_capability,
  946. "tools": knowledge.tools,
  947. }
  948. print(f"[Save Knowledge] 插入数据: {json.dumps({k: v for k, v in insert_data.items() if k != 'embedding'}, ensure_ascii=False)}")
  949. # 插入 Milvus
  950. pg_store.insert(insert_data)
  951. # 触发后台去重处理
  952. background_tasks.add_task(knowledge_processor.process_pending)
  953. return {"status": "pending", "knowledge_id": knowledge_id, "message": "知识已入队,正在处理去重..."}
  954. except Exception as e:
  955. print(f"[Save Knowledge] 错误: {e}")
  956. raise HTTPException(status_code=500, detail=str(e))
  957. @app.get("/api/knowledge")
  958. def list_knowledge(
  959. page: int = Query(default=1, ge=1),
  960. page_size: int = Query(default=200, ge=1, le=500),
  961. types: Optional[str] = None,
  962. scopes: Optional[str] = None,
  963. owner: Optional[str] = None,
  964. tags: Optional[str] = None,
  965. status: Optional[str] = None
  966. ):
  967. """列出知识(支持后端筛选和分页)"""
  968. try:
  969. # 构建过滤表达式
  970. filters = []
  971. # types 支持多个,用 AND 连接(交集:必须同时包含所有选中的type)
  972. if types:
  973. type_list = [t.strip() for t in types.split(',') if t.strip()]
  974. for t in type_list:
  975. filters.append(f'array_contains(types, "{t}")')
  976. if scopes:
  977. filters.append(f'array_contains(scopes, "{scopes}")')
  978. if owner:
  979. owner_list = [o.strip() for o in owner.split(',') if o.strip()]
  980. if len(owner_list) == 1:
  981. filters.append(f'owner == "{owner_list[0]}"')
  982. else:
  983. # 多个owner用OR连接
  984. owner_filters = [f'owner == "{o}"' for o in owner_list]
  985. filters.append(f'({" or ".join(owner_filters)})')
  986. # tags 支持多个,用 AND 连接(使用 tag_keys 数组进行高效筛选)
  987. if tags:
  988. tag_list = [t.strip() for t in tags.split(',') if t.strip()]
  989. for t in tag_list:
  990. filters.append(f'array_contains(tag_keys, "{t}")')
  991. # 只返回指定 status 的知识(默认 approved 和 checked)
  992. status_list = [s.strip() for s in (status or "approved,checked").split(',') if s.strip()]
  993. status_conditions = ' or '.join([f'status == "{s}"' for s in status_list])
  994. filters.append(f'({status_conditions})')
  995. # 如果没有过滤条件,查询所有
  996. filter_expr = ' and '.join(filters) if filters else 'id != ""'
  997. # 查询 Milvus(先获取所有符合条件的数据)
  998. # Milvus 的 limit 是总数限制,我们需要获取足够多的数据来支持分页
  999. max_limit = 10000 # 设置一个合理的上限
  1000. results = pg_store.query(filter_expr, limit=max_limit)
  1001. # 转换为可序列化的格式
  1002. serialized_results = [serialize_milvus_result(r) for r in results]
  1003. # 按 created_at 降序排序(最新的在前)
  1004. serialized_results.sort(key=lambda x: x.get('created_at', 0), reverse=True)
  1005. # 计算分页
  1006. total = len(serialized_results)
  1007. total_pages = (total + page_size - 1) // page_size # 向上取整
  1008. start_idx = (page - 1) * page_size
  1009. end_idx = start_idx + page_size
  1010. page_results = serialized_results[start_idx:end_idx]
  1011. return {
  1012. "results": page_results,
  1013. "pagination": {
  1014. "page": page,
  1015. "page_size": page_size,
  1016. "total": total,
  1017. "total_pages": total_pages
  1018. }
  1019. }
  1020. except Exception as e:
  1021. print(f"[List Knowledge] 错误: {e}")
  1022. raise HTTPException(status_code=500, detail=str(e))
  1023. @app.get("/api/knowledge/meta/tags")
  1024. def get_all_tags():
  1025. """获取所有已有的 tags"""
  1026. try:
  1027. # 查询所有知识
  1028. results = pg_store.query('id != ""', limit=10000)
  1029. all_tags = set()
  1030. for item in results:
  1031. # 转换为标准字典
  1032. serialized_item = serialize_milvus_result(item)
  1033. tags_dict = serialized_item.get("tags", {})
  1034. if isinstance(tags_dict, dict):
  1035. for key in tags_dict.keys():
  1036. all_tags.add(key)
  1037. return {"tags": sorted(list(all_tags))}
  1038. except Exception as e:
  1039. print(f"[Get Tags] 错误: {e}")
  1040. raise HTTPException(status_code=500, detail=str(e))
  1041. @app.get("/api/knowledge/pending")
  1042. def get_pending_knowledge(limit: int = Query(default=50, ge=1, le=200)):
  1043. """查询待处理队列(pending + processing + dedup_passed + analyzing)"""
  1044. try:
  1045. pending = pg_store.query(
  1046. 'status == "pending" or status == "processing" or status == "dedup_passed" or status == "analyzing"',
  1047. limit=limit
  1048. )
  1049. serialized = [serialize_milvus_result(r) for r in pending]
  1050. return {"results": serialized, "count": len(serialized)}
  1051. except Exception as e:
  1052. print(f"[Pending] 错误: {e}")
  1053. raise HTTPException(status_code=500, detail=str(e))
  1054. @app.post("/api/knowledge/process")
  1055. async def trigger_process(force: bool = Query(default=False)):
  1056. """手动触发去重处理。force=true 时先回滚所有 processing → pending,analyzing → dedup_passed"""
  1057. try:
  1058. if force:
  1059. processing = pg_store.query('status == "processing"', limit=200)
  1060. for item in processing:
  1061. pg_store.update(item["id"], {"status": "pending", "updated_at": int(time.time())})
  1062. print(f"[Manual Process] 回滚 {len(processing)} 条 processing → pending")
  1063. analyzing = pg_store.query('status == "analyzing"', limit=200)
  1064. for item in analyzing:
  1065. pg_store.update(item["id"], {"status": "dedup_passed", "updated_at": int(time.time())})
  1066. print(f"[Manual Process] 回滚 {len(analyzing)} 条 analyzing → dedup_passed")
  1067. asyncio.create_task(knowledge_processor.process_pending())
  1068. return {"status": "ok", "message": "处理任务已触发"}
  1069. except Exception as e:
  1070. print(f"[Manual Process] 错误: {e}")
  1071. raise HTTPException(status_code=500, detail=str(e))
  1072. @app.post("/api/knowledge/migrate")
  1073. async def migrate_knowledge_schema():
  1074. """手动触发 schema 迁移(PostgreSQL不需要此功能)"""
  1075. return {"status": "ok", "message": "PostgreSQL不需要schema迁移"}
  1076. @app.get("/api/knowledge/status/{knowledge_id}")
  1077. def get_knowledge_status(knowledge_id: str):
  1078. """查询单条知识的处理状态和关系"""
  1079. try:
  1080. result = pg_store.get_by_id(knowledge_id)
  1081. if not result:
  1082. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1083. serialized = serialize_milvus_result(result)
  1084. return {
  1085. "id": knowledge_id,
  1086. "status": serialized.get("status", "approved"),
  1087. "relationships": serialized.get("relationships", []),
  1088. "created_at": serialized.get("created_at"),
  1089. "updated_at": serialized.get("updated_at"),
  1090. }
  1091. except HTTPException:
  1092. raise
  1093. except Exception as e:
  1094. print(f"[Knowledge Status] 错误: {e}")
  1095. raise HTTPException(status_code=500, detail=str(e))
  1096. @app.get("/api/knowledge/{knowledge_id}")
  1097. def get_knowledge(knowledge_id: str):
  1098. """获取单条知识"""
  1099. try:
  1100. result = pg_store.get_by_id(knowledge_id)
  1101. if not result:
  1102. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1103. return serialize_milvus_result(result)
  1104. except HTTPException:
  1105. raise
  1106. except Exception as e:
  1107. print(f"[Get Knowledge] 错误: {e}")
  1108. raise HTTPException(status_code=500, detail=str(e))
  1109. async def _evolve_knowledge_with_llm(old_content: str, feedback: str) -> str:
  1110. """使用 LLM 进行知识进化重写"""
  1111. prompt = KNOWLEDGE_EVOLVE_PROMPT_TEMPLATE.format(
  1112. old_content=old_content,
  1113. feedback=feedback
  1114. )
  1115. try:
  1116. response = await _dedup_llm(
  1117. messages=[{"role": "user", "content": prompt}],
  1118. )
  1119. evolved = response.get("content", "").strip()
  1120. if len(evolved) < 5:
  1121. raise ValueError("LLM output too short")
  1122. return evolved
  1123. except Exception as e:
  1124. print(f"知识进化失败,采用追加模式回退: {e}")
  1125. return f"{old_content}\n\n---\n[Update {datetime.now().strftime('%Y-%m-%d')}]: {feedback}"
  1126. @app.put("/api/knowledge/{knowledge_id}")
  1127. async def update_knowledge(knowledge_id: str, update: KnowledgeUpdateIn):
  1128. """更新知识评估,支持知识进化"""
  1129. try:
  1130. # 获取现有知识
  1131. existing = pg_store.get_by_id(knowledge_id)
  1132. if not existing:
  1133. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1134. eval_data = existing.get("eval", {})
  1135. # 更新评分
  1136. if update.update_score is not None:
  1137. eval_data["score"] = update.update_score
  1138. # 添加有效案例
  1139. if update.add_helpful_case:
  1140. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1141. if "helpful_history" not in eval_data:
  1142. eval_data["helpful_history"] = []
  1143. eval_data["helpful_history"].append(update.add_helpful_case)
  1144. # 添加有害案例
  1145. if update.add_harmful_case:
  1146. eval_data["harmful"] = eval_data.get("harmful", 0) + 1
  1147. if "harmful_history" not in eval_data:
  1148. eval_data["harmful_history"] = []
  1149. eval_data["harmful_history"].append(update.add_harmful_case)
  1150. # 知识进化
  1151. content = existing["content"]
  1152. need_reembed = False
  1153. if update.evolve_feedback:
  1154. content = await _evolve_knowledge_with_llm(content, update.evolve_feedback)
  1155. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1156. need_reembed = True
  1157. # 准备更新数据
  1158. updates = {
  1159. "content": content,
  1160. "eval": eval_data,
  1161. }
  1162. # 如果内容变化,重新生成向量
  1163. if need_reembed:
  1164. embedding = await get_embedding(existing['task'])
  1165. updates["task_embedding"] = embedding
  1166. # 更新 Milvus
  1167. pg_store.update(knowledge_id, updates)
  1168. return {"status": "ok", "knowledge_id": knowledge_id}
  1169. except HTTPException:
  1170. raise
  1171. except Exception as e:
  1172. print(f"[Update Knowledge] 错误: {e}")
  1173. raise HTTPException(status_code=500, detail=str(e))
  1174. @app.patch("/api/knowledge/{knowledge_id}")
  1175. async def patch_knowledge(knowledge_id: str, patch: KnowledgePatchIn):
  1176. """直接编辑知识字段"""
  1177. try:
  1178. # 获取现有知识
  1179. existing = pg_store.get_by_id(knowledge_id)
  1180. if not existing:
  1181. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1182. updates = {}
  1183. need_reembed = False
  1184. need_content_reembed = False
  1185. if patch.task is not None:
  1186. updates["task"] = patch.task
  1187. need_reembed = True
  1188. if patch.content is not None:
  1189. updates["content"] = patch.content
  1190. need_content_reembed = True
  1191. if patch.types is not None:
  1192. updates["types"] = patch.types
  1193. if patch.tags is not None:
  1194. updates["tags"] = patch.tags
  1195. # 同时更新 tag_keys
  1196. updates["tag_keys"] = list(patch.tags.keys()) if isinstance(patch.tags, dict) else []
  1197. if patch.scopes is not None:
  1198. updates["scopes"] = patch.scopes
  1199. if patch.owner is not None:
  1200. updates["owner"] = patch.owner
  1201. if patch.support_capability is not None:
  1202. updates["support_capability"] = patch.support_capability
  1203. if patch.tools is not None:
  1204. updates["tools"] = patch.tools
  1205. if not updates:
  1206. return {"status": "ok", "knowledge_id": knowledge_id}
  1207. # 如果 task 变化,重新生成 task_embedding
  1208. if need_reembed:
  1209. task = updates.get("task", existing["task"])
  1210. embedding = await get_embedding(task)
  1211. updates["task_embedding"] = embedding
  1212. # 如果 content 变化,重新生成 content_embedding
  1213. if need_content_reembed:
  1214. content = updates.get("content", existing["content"])
  1215. content_embedding = await get_embedding(content)
  1216. updates["content_embedding"] = content_embedding
  1217. # 更新 Milvus
  1218. pg_store.update(knowledge_id, updates)
  1219. return {"status": "ok", "knowledge_id": knowledge_id}
  1220. except HTTPException:
  1221. raise
  1222. except Exception as e:
  1223. print(f"[Patch Knowledge] 错误: {e}")
  1224. raise HTTPException(status_code=500, detail=str(e))
  1225. @app.delete("/api/knowledge/{knowledge_id}")
  1226. def delete_knowledge(knowledge_id: str):
  1227. """删除单条知识"""
  1228. try:
  1229. # 检查知识是否存在
  1230. existing = pg_store.get_by_id(knowledge_id)
  1231. if not existing:
  1232. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1233. # 从 PostgreSQL 删除
  1234. pg_store.delete(knowledge_id)
  1235. print(f"[Delete Knowledge] 已删除知识: {knowledge_id}")
  1236. return {"status": "ok", "knowledge_id": knowledge_id}
  1237. except HTTPException:
  1238. raise
  1239. except Exception as e:
  1240. print(f"[Delete Knowledge] 错误: {e}")
  1241. raise HTTPException(status_code=500, detail=str(e))
  1242. @app.post("/api/knowledge/batch_delete")
  1243. def batch_delete_knowledge(knowledge_ids: List[str] = Body(...)):
  1244. """批量删除知识"""
  1245. try:
  1246. if not knowledge_ids:
  1247. raise HTTPException(status_code=400, detail="knowledge_ids cannot be empty")
  1248. # 批量删除
  1249. cursor = pg_store._get_cursor()
  1250. try:
  1251. cursor.execute(
  1252. "DELETE FROM knowledge WHERE id = ANY(%s)",
  1253. (knowledge_ids,)
  1254. )
  1255. pg_store.conn.commit()
  1256. deleted_count = cursor.rowcount
  1257. print(f"[Batch Delete] 已删除 {deleted_count} 条知识")
  1258. return {"status": "ok", "deleted_count": deleted_count}
  1259. finally:
  1260. cursor.close()
  1261. except HTTPException:
  1262. raise
  1263. except Exception as e:
  1264. print(f"[Batch Delete] 错误: {e}")
  1265. raise HTTPException(status_code=500, detail=str(e))
  1266. @app.post("/api/knowledge/batch_verify")
  1267. async def batch_verify_knowledge(batch: KnowledgeBatchVerifyIn):
  1268. """批量验证通过(approved → checked)"""
  1269. if not batch.knowledge_ids:
  1270. return {"status": "ok", "updated": 0}
  1271. try:
  1272. now_iso = datetime.now(timezone.utc).isoformat()
  1273. updated_count = 0
  1274. for kid in batch.knowledge_ids:
  1275. existing = pg_store.get_by_id(kid)
  1276. if not existing:
  1277. continue
  1278. eval_data = existing.get("eval") or {}
  1279. eval_data["verification"] = {
  1280. "status": "checked",
  1281. "verified_by": batch.verified_by,
  1282. "verified_at": now_iso,
  1283. "note": None,
  1284. "issue_type": None,
  1285. "issue_action": None,
  1286. }
  1287. pg_store.update(kid, {"eval": eval_data, "status": "checked", "updated_at": int(time.time())})
  1288. updated_count += 1
  1289. return {"status": "ok", "updated": updated_count}
  1290. except Exception as e:
  1291. print(f"[Batch Verify] 错误: {e}")
  1292. raise HTTPException(status_code=500, detail=str(e))
  1293. @app.post("/api/knowledge/{knowledge_id}/verify")
  1294. async def verify_knowledge(knowledge_id: str, verify: KnowledgeVerifyIn):
  1295. """知识验证:approve 切换 approved↔checked,reject 设为 rejected"""
  1296. try:
  1297. existing = pg_store.get_by_id(knowledge_id)
  1298. if not existing:
  1299. raise HTTPException(status_code=404, detail=f"Knowledge not found: {knowledge_id}")
  1300. current_status = existing.get("status", "approved")
  1301. if verify.action == "approve":
  1302. # checked → approved(取消验证),其他 → checked
  1303. new_status = "approved" if current_status == "checked" else "checked"
  1304. pg_store.update(knowledge_id, {
  1305. "status": new_status,
  1306. "updated_at": int(time.time())
  1307. })
  1308. return {"status": "ok", "new_status": new_status,
  1309. "message": "已取消验证" if new_status == "approved" else "验证通过"}
  1310. elif verify.action == "reject":
  1311. pg_store.update(knowledge_id, {
  1312. "status": "rejected",
  1313. "updated_at": int(time.time())
  1314. })
  1315. return {"status": "ok", "new_status": "rejected", "message": "已拒绝"}
  1316. else:
  1317. raise HTTPException(status_code=400, detail=f"Unknown action: {verify.action}")
  1318. except HTTPException:
  1319. raise
  1320. except Exception as e:
  1321. print(f"[Verify Knowledge] 错误: {e}")
  1322. raise HTTPException(status_code=500, detail=str(e))
  1323. @app.post("/api/knowledge/batch_update")
  1324. async def batch_update_knowledge(batch: KnowledgeBatchUpdateIn):
  1325. """批量反馈知识有效性"""
  1326. if not batch.feedback_list:
  1327. return {"status": "ok", "updated": 0}
  1328. try:
  1329. # 先处理无需进化的,收集需要进化的
  1330. evolution_tasks = [] # [(knowledge_id, old_content, feedback, eval_data)]
  1331. simple_updates = [] # [(knowledge_id, is_effective, eval_data)]
  1332. for item in batch.feedback_list:
  1333. knowledge_id = item.get("knowledge_id")
  1334. is_effective = item.get("is_effective")
  1335. feedback = item.get("feedback", "")
  1336. if not knowledge_id:
  1337. continue
  1338. existing = pg_store.get_by_id(knowledge_id)
  1339. if not existing:
  1340. continue
  1341. eval_data = existing.get("eval", {})
  1342. if is_effective and feedback:
  1343. evolution_tasks.append((knowledge_id, existing["content"], feedback, eval_data, existing["task"]))
  1344. else:
  1345. simple_updates.append((knowledge_id, is_effective, eval_data))
  1346. # 执行简单更新
  1347. for knowledge_id, is_effective, eval_data in simple_updates:
  1348. if is_effective:
  1349. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1350. else:
  1351. eval_data["harmful"] = eval_data.get("harmful", 0) + 1
  1352. pg_store.update(knowledge_id, {"eval": eval_data})
  1353. # 并发执行知识进化
  1354. if evolution_tasks:
  1355. print(f"🧬 并发处理 {len(evolution_tasks)} 条知识进化...")
  1356. evolved_results = await asyncio.gather(
  1357. *[_evolve_knowledge_with_llm(old, fb) for _, old, fb, _, _ in evolution_tasks]
  1358. )
  1359. for (knowledge_id, _, _, eval_data, task), evolved_content in zip(evolution_tasks, evolved_results):
  1360. eval_data["helpful"] = eval_data.get("helpful", 0) + 1
  1361. # 重新生成向量(只基于 task)
  1362. embedding = await get_embedding(task)
  1363. pg_store.update(knowledge_id, {
  1364. "content": evolved_content,
  1365. "eval": eval_data,
  1366. "task_embedding": embedding
  1367. })
  1368. return {"status": "ok", "updated": len(simple_updates) + len(evolution_tasks)}
  1369. except Exception as e:
  1370. print(f"[Batch Update] 错误: {e}")
  1371. raise HTTPException(status_code=500, detail=str(e))
  1372. @app.post("/api/knowledge/slim")
  1373. async def slim_knowledge(model: str = "google/gemini-2.5-flash-lite"):
  1374. """知识库瘦身:合并语义相似知识"""
  1375. try:
  1376. # 获取所有知识
  1377. all_knowledge = pg_store.query('id != ""', limit=10000)
  1378. # 转换为可序列化的格式
  1379. all_knowledge = [serialize_milvus_result(item) for item in all_knowledge]
  1380. if len(all_knowledge) < 2:
  1381. return {"status": "ok", "message": f"知识库仅有 {len(all_knowledge)} 条,无需瘦身"}
  1382. # 构造发给大模型的内容
  1383. entries_text = ""
  1384. for item in all_knowledge:
  1385. eval_data = item.get("eval", {})
  1386. types = item.get("types", [])
  1387. entries_text += f"[ID: {item['id']}] [Types: {','.join(types)}] "
  1388. entries_text += f"[Helpful: {eval_data.get('helpful', 0)}, Harmful: {eval_data.get('harmful', 0)}] [Score: {eval_data.get('score', 3)}]\n"
  1389. entries_text += f"Task: {item['task']}\n"
  1390. entries_text += f"Content: {item['content'][:200]}...\n\n"
  1391. prompt = KNOWLEDGE_SLIM_PROMPT_TEMPLATE.format(entries_text=entries_text)
  1392. print(f"\n[知识瘦身] 正在调用 {model} 分析 {len(all_knowledge)} 条知识...")
  1393. slim_llm = create_openrouter_llm_call(model=model)
  1394. response = await slim_llm(
  1395. messages=[{"role": "user", "content": prompt}],
  1396. )
  1397. content = response.get("content", "").strip()
  1398. if not content:
  1399. raise HTTPException(status_code=500, detail="LLM 返回为空")
  1400. # 解析大模型输出
  1401. report_line = ""
  1402. new_entries = []
  1403. blocks = [b.strip() for b in content.split("===") if b.strip()]
  1404. for block in blocks:
  1405. if block.startswith("REPORT:"):
  1406. report_line = block
  1407. continue
  1408. lines = block.split("\n")
  1409. kid, types, helpful, harmful, score, task, content_lines = None, [], 0, 0, 3, "", []
  1410. current_field = None
  1411. for line in lines:
  1412. if line.startswith("ID:"):
  1413. kid = line[3:].strip()
  1414. current_field = None
  1415. elif line.startswith("TYPES:"):
  1416. types_str = line[6:].strip()
  1417. types = [t.strip() for t in types_str.split(",") if t.strip()]
  1418. current_field = None
  1419. elif line.startswith("HELPFUL:"):
  1420. try:
  1421. helpful = int(line[8:].strip())
  1422. except Exception:
  1423. helpful = 0
  1424. current_field = None
  1425. elif line.startswith("HARMFUL:"):
  1426. try:
  1427. harmful = int(line[8:].strip())
  1428. except Exception:
  1429. harmful = 0
  1430. current_field = None
  1431. elif line.startswith("SCORE:"):
  1432. try:
  1433. score = int(line[6:].strip())
  1434. except Exception:
  1435. score = 3
  1436. current_field = None
  1437. elif line.startswith("TASK:"):
  1438. task = line[5:].strip()
  1439. current_field = "task"
  1440. elif line.startswith("CONTENT:"):
  1441. content_lines.append(line[8:].strip())
  1442. current_field = "content"
  1443. elif current_field == "task":
  1444. task += "\n" + line
  1445. elif current_field == "content":
  1446. content_lines.append(line)
  1447. if kid and content_lines:
  1448. new_entries.append({
  1449. "id": kid,
  1450. "types": types if types else ["strategy"],
  1451. "helpful": helpful,
  1452. "harmful": harmful,
  1453. "score": score,
  1454. "task": task.strip(),
  1455. "content": "\n".join(content_lines).strip()
  1456. })
  1457. if not new_entries:
  1458. raise HTTPException(status_code=500, detail="解析大模型输出失败")
  1459. # 生成向量并重建知识库
  1460. print(f"[知识瘦身] 正在为 {len(new_entries)} 条知识生成向量...")
  1461. # 批量生成向量(只基于 task)
  1462. texts = [e['task'] for e in new_entries]
  1463. embeddings = await get_embeddings_batch(texts)
  1464. # 清空并重建(PostgreSQL使用TRUNCATE)
  1465. cursor = pg_store._get_cursor()
  1466. try:
  1467. cursor.execute("TRUNCATE TABLE knowledge")
  1468. pg_store.conn.commit()
  1469. finally:
  1470. cursor.close()
  1471. knowledge_list = []
  1472. for e, embedding in zip(new_entries, embeddings):
  1473. eval_data = {
  1474. "score": e["score"],
  1475. "helpful": e["helpful"],
  1476. "harmful": e["harmful"],
  1477. "confidence": 0.9,
  1478. "helpful_history": [],
  1479. "harmful_history": []
  1480. }
  1481. source = {
  1482. "name": "slim",
  1483. "category": "exp",
  1484. "urls": [],
  1485. "agent_id": "slim",
  1486. "submitted_by": "system",
  1487. "timestamp": datetime.now(timezone.utc).isoformat()
  1488. }
  1489. knowledge_list.append({
  1490. "id": e["id"],
  1491. "task_embedding": embedding,
  1492. "message_id": "",
  1493. "task": e["task"],
  1494. "content": e["content"],
  1495. "types": e["types"],
  1496. "tags": {},
  1497. "tag_keys": [],
  1498. "scopes": ["org:cybertogether"],
  1499. "owner": "agent:slim",
  1500. "resource_ids": [],
  1501. "source": source,
  1502. "eval": eval_data,
  1503. "created_at": now,
  1504. "updated_at": now,
  1505. "status": "approved",
  1506. "relationships": json.dumps([])
  1507. })
  1508. pg_store.insert_batch(knowledge_list)
  1509. result_msg = f"瘦身完成:{len(all_knowledge)} → {len(new_entries)} 条知识"
  1510. if report_line:
  1511. result_msg += f"\n{report_line}"
  1512. print(f"[知识瘦身] {result_msg}")
  1513. return {"status": "ok", "before": len(all_knowledge), "after": len(new_entries), "report": report_line}
  1514. except HTTPException:
  1515. raise
  1516. except Exception as e:
  1517. print(f"[Slim Knowledge] 错误: {e}")
  1518. raise HTTPException(status_code=500, detail=str(e))
  1519. @app.post("/api/extract")
  1520. async def extract_knowledge_from_messages(extract_req: MessageExtractIn, background_tasks: BackgroundTasks):
  1521. """从消息历史中提取知识(LLM 分析)"""
  1522. if not extract_req.submitted_by:
  1523. raise HTTPException(status_code=400, detail="submitted_by is required")
  1524. messages = extract_req.messages
  1525. if not messages or len(messages) == 0:
  1526. return {"status": "ok", "extracted_count": 0, "knowledge_ids": []}
  1527. # 构造消息历史文本
  1528. messages_text = ""
  1529. for msg in messages:
  1530. role = msg.get("role", "unknown")
  1531. content = msg.get("content", "")
  1532. messages_text += f"[{role}]: {content}\n\n"
  1533. # LLM 提取知识
  1534. prompt = MESSAGE_EXTRACT_PROMPT_TEMPLATE.format(messages_text=messages_text)
  1535. try:
  1536. print(f"\n[Extract] 正在从 {len(messages)} 条消息中提取知识...")
  1537. response = await _dedup_llm(
  1538. messages=[{"role": "user", "content": prompt}],
  1539. )
  1540. content = response.get("content", "").strip()
  1541. # 尝试解析 JSON
  1542. # 移除可能的 markdown 代码块标记
  1543. if content.startswith("```json"):
  1544. content = content[7:]
  1545. if content.startswith("```"):
  1546. content = content[3:]
  1547. if content.endswith("```"):
  1548. content = content[:-3]
  1549. content = content.strip()
  1550. extracted_knowledge = json.loads(content)
  1551. if not isinstance(extracted_knowledge, list):
  1552. raise ValueError("LLM output is not a list")
  1553. if not extracted_knowledge:
  1554. return {"status": "ok", "extracted_count": 0, "knowledge_ids": []}
  1555. # 批量生成向量(只基于 task)
  1556. texts = [item.get('task', '') for item in extracted_knowledge]
  1557. embeddings = await get_embeddings_batch(texts)
  1558. # 保存提取的知识
  1559. knowledge_ids = []
  1560. now = int(time.time())
  1561. knowledge_list = []
  1562. for item, embedding in zip(extracted_knowledge, embeddings):
  1563. task = item.get("task", "")
  1564. knowledge_content = item.get("content", "")
  1565. types = item.get("types", ["strategy"])
  1566. score = item.get("score", 3)
  1567. if not task or not knowledge_content:
  1568. continue
  1569. # 生成 ID
  1570. timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
  1571. random_suffix = uuid.uuid4().hex[:4]
  1572. knowledge_id = f"knowledge-{timestamp}-{random_suffix}"
  1573. # 准备数据
  1574. source = {
  1575. "name": "message_extraction",
  1576. "category": "exp",
  1577. "urls": [],
  1578. "agent_id": extract_req.agent_id,
  1579. "submitted_by": extract_req.submitted_by,
  1580. "timestamp": datetime.now(timezone.utc).isoformat(),
  1581. "session_key": extract_req.session_key
  1582. }
  1583. eval_data = {
  1584. "score": score,
  1585. "helpful": 1,
  1586. "harmful": 0,
  1587. "confidence": 0.7,
  1588. "helpful_history": [],
  1589. "harmful_history": []
  1590. }
  1591. knowledge_list.append({
  1592. "id": knowledge_id,
  1593. "task_embedding": embedding,
  1594. "message_id": "",
  1595. "task": task,
  1596. "content": knowledge_content,
  1597. "types": types,
  1598. "tags": {},
  1599. "tag_keys": [],
  1600. "scopes": ["org:cybertogether"],
  1601. "owner": extract_req.submitted_by,
  1602. "resource_ids": [],
  1603. "source": source,
  1604. "eval": eval_data,
  1605. "created_at": now,
  1606. "updated_at": now,
  1607. "status": "pending",
  1608. "relationships": json.dumps([]),
  1609. })
  1610. knowledge_ids.append(knowledge_id)
  1611. # 批量插入
  1612. if knowledge_list:
  1613. pg_store.insert_batch(knowledge_list)
  1614. background_tasks.add_task(knowledge_processor.process_pending)
  1615. print(f"[Extract] 成功提取并保存 {len(knowledge_ids)} 条知识")
  1616. return {
  1617. "status": "ok",
  1618. "extracted_count": len(knowledge_ids),
  1619. "knowledge_ids": knowledge_ids
  1620. }
  1621. except json.JSONDecodeError as e:
  1622. print(f"[Extract] JSON 解析失败: {e}")
  1623. print(f"[Extract] LLM 输出: {content[:500]}")
  1624. return {"status": "error", "error": "Failed to parse LLM output", "extracted_count": 0}
  1625. except Exception as e:
  1626. print(f"[Extract] 提取失败: {e}")
  1627. return {"status": "error", "error": str(e), "extracted_count": 0}
  1628. # ===== Tool API =====
  1629. @app.post("/api/tool", status_code=201)
  1630. async def create_tool(tool: ToolIn):
  1631. """创建或更新工具"""
  1632. try:
  1633. now = int(time.time())
  1634. embedding = await get_embedding(f"{tool.name} {tool.introduction}")
  1635. pg_tool_store.insert_or_update({
  1636. 'id': tool.id,
  1637. 'name': tool.name,
  1638. 'version': tool.version,
  1639. 'introduction': tool.introduction,
  1640. 'tutorial': tool.tutorial,
  1641. 'input': tool.input,
  1642. 'output': tool.output,
  1643. 'updated_time': now,
  1644. 'status': tool.status,
  1645. 'capabilities': tool.capabilities,
  1646. 'tool_knowledge': tool.tool_knowledge,
  1647. 'case_knowledge': tool.case_knowledge,
  1648. 'process_knowledge': tool.process_knowledge,
  1649. 'embedding': embedding,
  1650. })
  1651. return {"status": "ok", "id": tool.id}
  1652. except Exception as e:
  1653. raise HTTPException(status_code=500, detail=str(e))
  1654. @app.get("/api/tool")
  1655. def list_tools(
  1656. status: Optional[str] = Query(None),
  1657. limit: int = Query(100, ge=1, le=1000),
  1658. offset: int = Query(0, ge=0),
  1659. ):
  1660. """列出工具"""
  1661. try:
  1662. results = pg_tool_store.list_all(limit=limit, offset=offset, status=status)
  1663. total = pg_tool_store.count(status=status)
  1664. return {"results": results, "total": total}
  1665. except Exception as e:
  1666. raise HTTPException(status_code=500, detail=str(e))
  1667. @app.get("/api/tool/search")
  1668. async def search_tools(
  1669. q: str = Query(..., description="查询文本"),
  1670. top_k: int = Query(5, ge=1, le=20),
  1671. status: Optional[str] = None,
  1672. ):
  1673. """向量检索工具"""
  1674. try:
  1675. query_embedding = await get_embedding(q)
  1676. results = pg_tool_store.search(query_embedding, limit=top_k, status=status)
  1677. return {"results": results, "count": len(results)}
  1678. except Exception as e:
  1679. raise HTTPException(status_code=500, detail=str(e))
  1680. @app.get("/api/tool/{tool_id:path}")
  1681. def get_tool(tool_id: str):
  1682. """获取单个工具详情"""
  1683. try:
  1684. result = pg_tool_store.get_by_id(tool_id)
  1685. if not result:
  1686. raise HTTPException(status_code=404, detail=f"Tool not found: {tool_id}")
  1687. return result
  1688. except HTTPException:
  1689. raise
  1690. except Exception as e:
  1691. raise HTTPException(status_code=500, detail=str(e))
  1692. @app.patch("/api/tool/{tool_id:path}")
  1693. async def patch_tool(tool_id: str, patch: ToolPatchIn):
  1694. """更新工具字段"""
  1695. try:
  1696. if not pg_tool_store.get_by_id(tool_id):
  1697. raise HTTPException(status_code=404, detail=f"Tool not found: {tool_id}")
  1698. updates = {}
  1699. need_reembed = False
  1700. for field in ('name', 'version', 'introduction', 'tutorial', 'input', 'output',
  1701. 'status', 'capabilities', 'tool_knowledge', 'case_knowledge', 'process_knowledge'):
  1702. value = getattr(patch, field)
  1703. if value is not None:
  1704. updates[field] = value
  1705. if field in ('name', 'introduction'):
  1706. need_reembed = True
  1707. if not updates:
  1708. return {"status": "ok", "id": tool_id}
  1709. updates['updated_time'] = int(time.time())
  1710. if need_reembed:
  1711. existing = pg_tool_store.get_by_id(tool_id)
  1712. name = updates.get('name', existing['name'])
  1713. intro = updates.get('introduction', existing['introduction'])
  1714. updates['embedding'] = await get_embedding(f"{name} {intro}")
  1715. pg_tool_store.update(tool_id, updates)
  1716. return {"status": "ok", "id": tool_id}
  1717. except HTTPException:
  1718. raise
  1719. except Exception as e:
  1720. raise HTTPException(status_code=500, detail=str(e))
  1721. @app.delete("/api/tool/{tool_id:path}")
  1722. def delete_tool(tool_id: str):
  1723. """删除工具"""
  1724. try:
  1725. if not pg_tool_store.get_by_id(tool_id):
  1726. raise HTTPException(status_code=404, detail=f"Tool not found: {tool_id}")
  1727. pg_tool_store.delete(tool_id)
  1728. return {"status": "ok", "id": tool_id}
  1729. except HTTPException:
  1730. raise
  1731. except Exception as e:
  1732. raise HTTPException(status_code=500, detail=str(e))
  1733. # ===== Capability API =====
  1734. @app.post("/api/capability", status_code=201)
  1735. async def create_capability(cap: CapabilityIn):
  1736. """创建或更新原子能力"""
  1737. try:
  1738. embedding = await get_embedding(f"{cap.name} {cap.description}")
  1739. pg_capability_store.insert_or_update({
  1740. 'id': cap.id,
  1741. 'name': cap.name,
  1742. 'criterion': cap.criterion,
  1743. 'description': cap.description,
  1744. 'requirements': cap.requirements,
  1745. 'implements': cap.implements,
  1746. 'tools': cap.tools,
  1747. 'source_knowledge': cap.source_knowledge,
  1748. 'embedding': embedding,
  1749. })
  1750. return {"status": "ok", "id": cap.id}
  1751. except Exception as e:
  1752. raise HTTPException(status_code=500, detail=str(e))
  1753. @app.get("/api/capability")
  1754. def list_capabilities(
  1755. limit: int = Query(100, ge=1, le=1000),
  1756. offset: int = Query(0, ge=0),
  1757. ):
  1758. """列出原子能力"""
  1759. try:
  1760. results = pg_capability_store.list_all(limit=limit, offset=offset)
  1761. total = pg_capability_store.count()
  1762. return {"results": results, "total": total}
  1763. except Exception as e:
  1764. raise HTTPException(status_code=500, detail=str(e))
  1765. @app.get("/api/capability/search")
  1766. async def search_capabilities(
  1767. q: str = Query(..., description="查询文本"),
  1768. top_k: int = Query(5, ge=1, le=20),
  1769. ):
  1770. """向量检索原子能力"""
  1771. try:
  1772. query_embedding = await get_embedding(q)
  1773. results = pg_capability_store.search(query_embedding, limit=top_k)
  1774. return {"results": results, "count": len(results)}
  1775. except Exception as e:
  1776. raise HTTPException(status_code=500, detail=str(e))
  1777. @app.get("/api/capability/{cap_id}")
  1778. def get_capability(cap_id: str):
  1779. """获取单个原子能力"""
  1780. try:
  1781. result = pg_capability_store.get_by_id(cap_id)
  1782. if not result:
  1783. raise HTTPException(status_code=404, detail=f"Capability not found: {cap_id}")
  1784. return result
  1785. except HTTPException:
  1786. raise
  1787. except Exception as e:
  1788. raise HTTPException(status_code=500, detail=str(e))
  1789. @app.patch("/api/capability/{cap_id}")
  1790. async def patch_capability(cap_id: str, patch: CapabilityPatchIn):
  1791. """更新原子能力字段"""
  1792. try:
  1793. existing = pg_capability_store.get_by_id(cap_id)
  1794. if not existing:
  1795. raise HTTPException(status_code=404, detail=f"Capability not found: {cap_id}")
  1796. updates = {}
  1797. need_reembed = False
  1798. for field in ('name', 'criterion', 'description', 'requirements',
  1799. 'implements', 'tools', 'source_knowledge'):
  1800. value = getattr(patch, field)
  1801. if value is not None:
  1802. updates[field] = value
  1803. if field in ('name', 'description'):
  1804. need_reembed = True
  1805. if not updates:
  1806. return {"status": "ok", "id": cap_id}
  1807. if need_reembed:
  1808. name = updates.get('name', existing['name'])
  1809. desc = updates.get('description', existing['description'])
  1810. updates['embedding'] = await get_embedding(f"{name} {desc}")
  1811. pg_capability_store.update(cap_id, updates)
  1812. return {"status": "ok", "id": cap_id}
  1813. except HTTPException:
  1814. raise
  1815. except Exception as e:
  1816. raise HTTPException(status_code=500, detail=str(e))
  1817. @app.delete("/api/capability/{cap_id}")
  1818. def delete_capability(cap_id: str):
  1819. """删除原子能力"""
  1820. try:
  1821. if not pg_capability_store.get_by_id(cap_id):
  1822. raise HTTPException(status_code=404, detail=f"Capability not found: {cap_id}")
  1823. pg_capability_store.delete(cap_id)
  1824. return {"status": "ok", "id": cap_id}
  1825. except HTTPException:
  1826. raise
  1827. except Exception as e:
  1828. raise HTTPException(status_code=500, detail=str(e))
  1829. # ===== Requirement API =====
  1830. @app.post("/api/requirement", status_code=201)
  1831. async def create_requirement(req: RequirementIn):
  1832. """创建或更新需求"""
  1833. try:
  1834. embedding = await get_embedding(req.description)
  1835. pg_requirement_store.insert_or_update({
  1836. 'id': req.id,
  1837. 'description': req.description,
  1838. 'atomics': req.atomics,
  1839. 'source_nodes': req.source_nodes,
  1840. 'status': req.status,
  1841. 'match_result': req.match_result,
  1842. 'embedding': embedding,
  1843. })
  1844. return {"status": "ok", "id": req.id}
  1845. except Exception as e:
  1846. raise HTTPException(status_code=500, detail=str(e))
  1847. @app.get("/api/requirement")
  1848. def list_requirements(
  1849. status: Optional[str] = Query(None),
  1850. limit: int = Query(100, ge=1, le=1000),
  1851. offset: int = Query(0, ge=0),
  1852. ):
  1853. """列出需求"""
  1854. try:
  1855. results = pg_requirement_store.list_all(limit=limit, offset=offset, status=status)
  1856. total = pg_requirement_store.count(status=status)
  1857. return {"results": results, "total": total}
  1858. except Exception as e:
  1859. raise HTTPException(status_code=500, detail=str(e))
  1860. @app.get("/api/requirement/search")
  1861. async def search_requirements(
  1862. q: str = Query(..., description="查询文本"),
  1863. top_k: int = Query(5, ge=1, le=20),
  1864. ):
  1865. """向量检索需求"""
  1866. try:
  1867. query_embedding = await get_embedding(q)
  1868. results = pg_requirement_store.search(query_embedding, limit=top_k)
  1869. return {"results": results, "count": len(results)}
  1870. except Exception as e:
  1871. raise HTTPException(status_code=500, detail=str(e))
  1872. @app.get("/api/requirement/{req_id}")
  1873. def get_requirement(req_id: str):
  1874. """获取单个需求"""
  1875. try:
  1876. result = pg_requirement_store.get_by_id(req_id)
  1877. if not result:
  1878. raise HTTPException(status_code=404, detail=f"Requirement not found: {req_id}")
  1879. return result
  1880. except HTTPException:
  1881. raise
  1882. except Exception as e:
  1883. raise HTTPException(status_code=500, detail=str(e))
  1884. @app.patch("/api/requirement/{req_id}")
  1885. async def patch_requirement(req_id: str, patch: RequirementPatchIn):
  1886. """更新需求字段"""
  1887. try:
  1888. existing = pg_requirement_store.get_by_id(req_id)
  1889. if not existing:
  1890. raise HTTPException(status_code=404, detail=f"Requirement not found: {req_id}")
  1891. updates = {}
  1892. need_reembed = False
  1893. for field in ('description', 'atomics', 'source_nodes', 'status', 'match_result'):
  1894. value = getattr(patch, field)
  1895. if value is not None:
  1896. updates[field] = value
  1897. if field == 'description':
  1898. need_reembed = True
  1899. if not updates:
  1900. return {"status": "ok", "id": req_id}
  1901. if need_reembed:
  1902. updates['embedding'] = await get_embedding(updates['description'])
  1903. pg_requirement_store.update(req_id, updates)
  1904. return {"status": "ok", "id": req_id}
  1905. except HTTPException:
  1906. raise
  1907. except Exception as e:
  1908. raise HTTPException(status_code=500, detail=str(e))
  1909. @app.delete("/api/requirement/{req_id}")
  1910. def delete_requirement(req_id: str):
  1911. """删除需求"""
  1912. try:
  1913. if not pg_requirement_store.get_by_id(req_id):
  1914. raise HTTPException(status_code=404, detail=f"Requirement not found: {req_id}")
  1915. pg_requirement_store.delete(req_id)
  1916. return {"status": "ok", "id": req_id}
  1917. except HTTPException:
  1918. raise
  1919. except Exception as e:
  1920. raise HTTPException(status_code=500, detail=str(e))
  1921. @app.get("/")
  1922. def frontend():
  1923. """KnowHub 管理前端"""
  1924. index_file = STATIC_DIR / "index.html"
  1925. if not index_file.exists():
  1926. return HTMLResponse("<h1>KnowHub Frontend Not Found</h1><p>Please ensure knowhub/frontend/dist/index.html exists. Run 'yarn build' in frontend directory.</p>", status_code=404)
  1927. return FileResponse(str(index_file))
  1928. @app.get("/category_tree.json")
  1929. def serve_category_tree():
  1930. """类别树JSON数据"""
  1931. tree_file = STATIC_DIR / "category_tree.json"
  1932. if not tree_file.exists():
  1933. return {"error": "Not Found"}
  1934. return FileResponse(str(tree_file))
  1935. if __name__ == "__main__":
  1936. import uvicorn
  1937. uvicorn.run(app, host="0.0.0.0", port=9999)