AutoScraperX/utils/config_loader.py

import yaml
import os
from urllib.parse import urljoin
from utils.project_paths import config_spiders_path


class ConfigLoader:
    def __init__(self, config_path=config_spiders_path):
        if not os.path.exists(config_path):
            raise FileNotFoundError(f"[配置错误] 找不到配置文件: {config_path}")
        self.config_path = config_path
        self.config = self._load_yaml()

    def _load_yaml(self):
        with open(self.config_path, "r", encoding="utf-8") as f:
            return yaml.safe_load(f)

    def get_platform_config(self, platform: str) -> dict:
        """获取平台配置，并拼接完整 URL"""
        if platform not in self.config:
            raise ValueError(f"[配置错误] 未找到平台配置: {platform}")

        platform_config = self.config.get(platform, {})
        base_config = self.config.get("default", {})

        # 合并配置：平台配置覆盖默认配置
        merged = {**base_config, **platform_config}

        # 自动拼接完整 url（优先用完整 url）
        if "url" not in merged and "base_url" in merged and "path" in merged:
            merged["url"] = urljoin(merged["base_url"], merged["path"])

        return merged

if __name__ == '__main__':
    config = ConfigLoader().get_platform_config("benshanzhufu")
    print(config)