luojunhui 1 tydzień temu
rodzic
commit
d4b4c350a6

+ 1 - 1
.gitignore

@@ -9,4 +9,4 @@ build/
 .idea/
 .vscode/
 .claude
-
+CLAUDE.md

+ 2 - 1
app.py

@@ -7,5 +7,6 @@
 from supply.server import create_app, server_routes, xxl_routes
 
 app = create_app()
+
 app.register_blueprint(server_routes())
-app.register_blueprint(xxl_routes())  # XXL-JOB 端点挂在根路径 /beat /run 等
+app.register_blueprint(xxl_routes())

+ 27 - 8
supply/__init__.py

@@ -66,17 +66,36 @@ from supply.server import create_app
 __all__ = [
     # Config
     "GlobalConfig",
-    "DatabaseConfig", "AliyunLogConfig", "AliyunOssConfig",
-    "PgConfig", "MilvusConfig", "RedisConfig",
+    "DatabaseConfig",
+    "AliyunLogConfig",
+    "AliyunOssConfig",
+    "PgConfig",
+    "MilvusConfig",
+    "RedisConfig",
     # Infra
-    "DatabaseManager", "create_backend", "MySQLBackend",
-    "LogService", "LogRecord", "LogLevel", "LogCategory", "LogStatus",
+    "DatabaseManager",
+    "create_backend",
+    "MySQLBackend",
+    "LogService",
+    "LogRecord",
+    "LogLevel",
+    "LogCategory",
+    "LogStatus",
     "AsyncHttpClient",
     # Core - Agent
-    "Agent", "AgentContext", "AgentResult", "AgentStatus",
-    "AgentOrchestrator", "ChainOrchestrator", "ParallelOrchestrator",
-    "Tool", "ToolResult", "ToolRegistry",
-    "Memory", "ConversationMemory", "VectorMemory",
+    "Agent",
+    "AgentContext",
+    "AgentResult",
+    "AgentStatus",
+    "AgentOrchestrator",
+    "ChainOrchestrator",
+    "ParallelOrchestrator",
+    "Tool",
+    "ToolResult",
+    "ToolRegistry",
+    "Memory",
+    "ConversationMemory",
+    "VectorMemory",
     # Server
     "create_app",
 ]

+ 1 - 1
supply/config/__init__.py

@@ -31,4 +31,4 @@ __all__ = [
     "AliyunOssConfig",
     "XxlJobConfig",
     "GlobalConfig",
-]
+]

+ 17 - 5
supply/config/database.py

@@ -65,7 +65,10 @@ class LongVideoDatabaseConfig(DatabaseConfig):
     db: str = "longvideo"
 
     model_config = SettingsConfigDict(
-        env_prefix="LONG_VIDEO_DB_", env_file=".env", case_sensitive=False, extra="ignore"
+        env_prefix="LONG_VIDEO_DB_",
+        env_file=".env",
+        case_sensitive=False,
+        extra="ignore",
     )
 
 
@@ -76,7 +79,10 @@ class LongArticlesDatabaseConfig(DatabaseConfig):
     db: str = "long_articles"
 
     model_config = SettingsConfigDict(
-        env_prefix="LONG_ARTICLES_DB_", env_file=".env", case_sensitive=False, extra="ignore"
+        env_prefix="LONG_ARTICLES_DB_",
+        env_file=".env",
+        case_sensitive=False,
+        extra="ignore",
     )
 
 
@@ -87,7 +93,10 @@ class PiaoquanCrawlerDatabaseConfig(DatabaseConfig):
     db: str = "piaoquan-crawler"
 
     model_config = SettingsConfigDict(
-        env_prefix="PIAOQUAN_CRAWLER_DB_", env_file=".env", case_sensitive=False, extra="ignore"
+        env_prefix="PIAOQUAN_CRAWLER_DB_",
+        env_file=".env",
+        case_sensitive=False,
+        extra="ignore",
     )
 
 
@@ -109,7 +118,10 @@ class ContentDeconstructionSupplyConfig(DatabaseConfig):
     db: str = "content-deconstruction-supply"
 
     model_config = SettingsConfigDict(
-        env_prefix="CONTENT_DECONSREUCTION_DB_", env_file=".env", case_sensitive=False, extra="ignore"
+        env_prefix="CONTENT_DECONSREUCTION_DB_",
+        env_file=".env",
+        case_sensitive=False,
+        extra="ignore",
     )
 
 
@@ -158,4 +170,4 @@ class RedisConfig(BaseSettings):
 
     model_config = SettingsConfigDict(
         env_prefix="REDIS_", env_file=".env", case_sensitive=False, extra="ignore"
-    )
+    )

+ 7 - 1
supply/core/__init__.py

@@ -24,7 +24,13 @@ from supply.config import (
 from supply.infra.database import DatabaseManager, create_backend
 
 # 可观测性
-from supply.infra.observability import LogService, LogRecord, LogLevel, LogCategory, LogStatus
+from supply.infra.observability import (
+    LogService,
+    LogRecord,
+    LogLevel,
+    LogCategory,
+    LogStatus,
+)
 
 # 生命周期
 from supply.core.bootstrap import AppContext

+ 3 - 9
supply/handlers/_discovery.py

@@ -25,15 +25,11 @@ def discover_and_register(package: str = "supply.handlers") -> None:
     try:
         pkg = importlib.import_module(package)
     except ImportError:
-        logger.warning(
-            "XXL-JOB handler 包 %r 不存在,无 handler 注册", package
-        )
+        logger.warning("XXL-JOB handler 包 %r 不存在,无 handler 注册", package)
         return
 
     registered = 0
-    for finder, name, is_pkg in pkgutil.iter_modules(
-        pkg.__path__, pkg.__name__ + "."
-    ):
+    for finder, name, is_pkg in pkgutil.iter_modules(pkg.__path__, pkg.__name__ + "."):
         if name.split(".")[-1].startswith("_"):
             continue
 
@@ -45,8 +41,6 @@ def discover_and_register(package: str = "supply.handlers") -> None:
             logger.exception("XXL-JOB handler 模块 %r 导入失败,跳过", name)
 
     if registered:
-        logger.info(
-            "XXL-JOB 自动发现完成: 已导入 %d 个 handler 模块", registered
-        )
+        logger.info("XXL-JOB 自动发现完成: 已导入 %d 个 handler 模块", registered)
     else:
         logger.info("XXL-JOB 自动发现: 在 %r 中未找到 handler 模块", package)

+ 2 - 6
supply/infra/database/manager.py

@@ -92,9 +92,7 @@ class DatabaseManager:
     async def init_pools(self) -> None:
         for name, backend in self._backends.items():
             await backend.init()
-            logger.info(
-                "存储后端 [%s] 初始化完成 → %s", name, type(backend).__name__
-            )
+            logger.info("存储后端 [%s] 初始化完成 → %s", name, type(backend).__name__)
 
     async def close_pools(self) -> None:
         for name, backend in self._backends.items():
@@ -167,9 +165,7 @@ class DatabaseManager:
         name = db_name or self._default_db
         backend = self._backends.get(name)
         if backend is None:
-            raise KeyError(
-                f"存储后端 '{name}' 未注册,可用: {self.list_databases()}"
-            )
+            raise KeyError(f"存储后端 '{name}' 未注册,可用: {self.list_databases()}")
         return backend
 
     def _get_sql(self, db_name: Optional[str] = None) -> SqlBackend:

+ 1 - 3
supply/infra/database/mysql.py

@@ -103,6 +103,4 @@ class MySQLBackend(SqlBackend):
 
     def _ensure_ready(self):
         if not self._initialized or self._pool is None:
-            raise RuntimeError(
-                f"MySQL backend [{self.name}] 未初始化,请先调用 init()"
-            )
+            raise RuntimeError(f"MySQL backend [{self.name}] 未初始化,请先调用 init()")

+ 19 - 10
supply/infra/observability/log_schema.py

@@ -96,12 +96,12 @@ class LogLevel(str, Enum):
 class LogCategory(str, Enum):
     """日志分类 —— 按系统边界划分,用于快速过滤"""
 
-    SYSTEM = "system"    # 启动、关闭、心跳
-    TASK = "task"        # 任务调度生命周期
-    AGENT = "agent"      # Agent 执行(规划、推理、反思)
-    TOOL = "tool"        # Tool 调用(HTTP、DB 查询等)
-    HTTP = "http"        # 入站 HTTP 请求
-    DB = "db"            # 数据库操作
+    SYSTEM = "system"  # 启动、关闭、心跳
+    TASK = "task"  # 任务调度生命周期
+    AGENT = "agent"  # Agent 执行(规划、推理、反思)
+    TOOL = "tool"  # Tool 调用(HTTP、DB 查询等)
+    HTTP = "http"  # 入站 HTTP 请求
+    DB = "db"  # 数据库操作
     SECURITY = "security"  # 安全相关(认证、鉴权、告警)
 
 
@@ -137,7 +137,9 @@ class LogRecord(BaseModel):
     # ── 核心字段 ──
     level: LogLevel = Field(default=LogLevel.INFO, description="日志级别")
     category: LogCategory = Field(default=LogCategory.SYSTEM, description="日志分类")
-    event: str = Field(default="", description="事件名,如 task_started / agent_think / tool_error")
+    event: str = Field(
+        default="", description="事件名,如 task_started / agent_think / tool_error"
+    )
     message: str = Field(default="", description="人类可读描述,用于全文检索")
 
     # ── 追踪字段 ──
@@ -150,7 +152,9 @@ class LogRecord(BaseModel):
     tool_name: str = Field(default="", description="Tool 名,如 WebSearchTool")
 
     # ── 性能字段 ──
-    duration_ms: int = Field(default=0, description="执行耗时(毫秒),用于 P50/P99 分析")
+    duration_ms: int = Field(
+        default=0, description="执行耗时(毫秒),用于 P50/P99 分析"
+    )
 
     # ── 状态与异常 ──
     status: LogStatus = Field(default=LogStatus.SUCCESS, description="执行状态")
@@ -158,7 +162,9 @@ class LogRecord(BaseModel):
     error_message: str = Field(default="", description="异常消息全文,用于全文检索")
 
     # ── 扩展字段 ──
-    extra: Dict[str, Any] = Field(default_factory=dict, description="业务自定义 KV,JSON 索引可查询子字段")
+    extra: Dict[str, Any] = Field(
+        default_factory=dict, description="业务自定义 KV,JSON 索引可查询子字段"
+    )
 
     def to_sls_items(self) -> list[tuple[str, str]]:
         """转为阿里云 SLS LogItem 的 contents 格式 [(key, value), ...]
@@ -180,7 +186,10 @@ class LogRecord(BaseModel):
             ("error_type", self.error_type),
             ("error_message", self.error_message),
             # extra 存 JSON 字符串,SLS json 类型索引可解析
-            ("extra", json.dumps(self.extra, ensure_ascii=False) if self.extra else "{}"),
+            (
+                "extra",
+                json.dumps(self.extra, ensure_ascii=False) if self.extra else "{}",
+            ),
         ]
 
 

+ 14 - 4
supply/infra/observability/logging.py

@@ -66,7 +66,9 @@ class LogService:
         self.queue = asyncio.Queue(maxsize=10000)
         self._running = True
         self._worker_task = asyncio.create_task(self._worker())
-        logger.info("LogService 已启动 → %s/%s", self.config.project, self.config.logstore)
+        logger.info(
+            "LogService 已启动 → %s/%s", self.config.project, self.config.logstore
+        )
 
     async def stop(self):
         if not self._running:
@@ -106,8 +108,16 @@ class LogService:
             try:
                 from supply.infra.observability.log_schema import LogLevel, LogCategory
 
-                level = LogLevel(level_str) if level_str in LogLevel._value2member_map_ else LogLevel.INFO
-                category = LogCategory(category_str) if category_str in LogCategory._value2member_map_ else LogCategory.SYSTEM
+                level = (
+                    LogLevel(level_str)
+                    if level_str in LogLevel._value2member_map_
+                    else LogLevel.INFO
+                )
+                category = (
+                    LogCategory(category_str)
+                    if category_str in LogCategory._value2member_map_
+                    else LogCategory.SYSTEM
+                )
             except (KeyError, ValueError):
                 level = LogLevel.INFO
                 category = LogCategory.SYSTEM
@@ -158,7 +168,7 @@ class LogService:
         req = PutLogsRequest(
             self.config.project,
             self.config.logstore,
-            topic=record.category.value,     # topic 用 category,方便消费端路由
+            topic=record.category.value,  # topic 用 category,方便消费端路由
             source="",
             logitems=[log_item],
         )

+ 1 - 1
supply/infra/shared/__init__.py

@@ -10,4 +10,4 @@ from .tools import (
     yield_batch,
     timestamp_to_str,
     days_remaining_in_month,
-)
+)

+ 2 - 0
supply/infra/shared/async_tasks.py

@@ -41,6 +41,7 @@ async def run_tasks_with_async_worker_group(
     if show_progress:
         try:
             from tqdm.asyncio import tqdm as async_tqdm
+
             pbar = async_tqdm(total=total_task, unit=unit, desc=description)
         except ImportError:
             pbar = None
@@ -139,6 +140,7 @@ async def run_tasks_with_asyncio_task_group(
     if show_progress:
         try:
             from tqdm.asyncio import tqdm as async_tqdm
+
             pbar = async_tqdm(total=total_task, unit=unit, desc=description)
         except ImportError:
             pbar = None

+ 0 - 1
supply/infra/shared/tools.py

@@ -117,4 +117,3 @@ def days_remaining_in_month() -> int:
         next_month = today.replace(month=today.month + 1, day=1)
     last_day_of_month = next_month - timedelta(days=1)
     return (last_day_of_month - today).days
-

+ 1 - 3
supply/infra/xxl_jobs/xxl_executor.py

@@ -217,9 +217,7 @@ class XxlJobExecutor:
             return {"code": 200, "msg": None}
         return {"code": 500, "msg": f"job {job_id} not found or already finished"}
 
-    def log(
-        self, log_date_time: int, log_id: int, from_line_num: int
-    ) -> dict:
+    def log(self, log_date_time: int, log_id: int, from_line_num: int) -> dict:
         """读取执行日志 —— Admin 调用 POST /log"""
         log_file = os.path.join(
             self.config.log_path,

+ 6 - 4
supply/server/app.py

@@ -32,10 +32,12 @@ def create_app(global_config: GlobalConfig = None) -> Quart:
 
     # 初始化各层依赖 —— 全部从 global_config 取值
     log_service = LogService(global_config.aliyun_log)
-    db = DatabaseManager({
-        name: create_backend(name, config)
-        for name, config in global_config.active_db_configs().items()
-    })
+    db = DatabaseManager(
+        {
+            name: create_backend(name, config)
+            for name, config in global_config.active_db_configs().items()
+        }
+    )
     discover_and_register()
     xxl_executor = XxlJobExecutor(global_config.xxl_job)