AutoScraperX/core/base/async_request_client.py

import asyncio
from typing import Optional, Dict

import aiohttp


class AsyncRequestClient:
    """
    可独立复用的异步请求客户端，支持重试、日志结构化和限流预留
    """
    def __init__(self, logger=None, max_retries=3, timeout=30):
        self.logger = logger
        self.max_retries = max_retries
        self.timeout = timeout

    async def request(self, session: aiohttp.ClientSession, method: str, url: str, **kwargs) -> Optional[Dict]:
        retries = 0

        while retries < self.max_retries:
            try:
                if self.logger:
                    self.logger.info(f"请求 {method} {url}, 尝试 {retries+1}/{self.max_retries}")
                async with session.request(method, url, **kwargs) as response:
                    response.raise_for_status()
                    return await response.json()
            except Exception as e:
                retries += 1
                if retries >= self.max_retries:
                    if self.logger:
                        self.logger.error(f"请求失败达到最大重试次数: {e}")
                    raise
                if self.logger:
                    self.logger.warning(f"请求失败 {e}, 重试 {retries}/{self.max_retries}")
                await asyncio.sleep(1)