AutoScraperX/core/base/async_request_client.py

import asyncio
from typing import Optional, Dict
from pprint import pformat

import aiohttp


class AsyncRequestClient:
    """
    可独立复用的异步请求客户端，支持重试、日志结构化和限流预留
    """
    def __init__(self, logger=None, aliyun_log=None,max_retries=3, timeout=30):
        self.logger = logger
        self.aliyun_log = aliyun_log
        self.max_retries = max_retries
        self.timeout = timeout

    async def request(self, session: aiohttp.ClientSession, method: str, url: str, **kwargs) -> Optional[Dict]:
        retries = 0

        while retries < self.max_retries:
            try:
                if self.logger:
                    self.logger.info(f"请求 {method} {url}, 请求参数{kwargs}")
                async with session.request(method, url, **kwargs) as response:
                    response.raise_for_status()
                    resp = await response.json()
                    # self.logger.info(f"{url}响应：\n {pformat(resp)}")
                    if resp.get('code') != 0:
                        retries += 1
                        if self.logger:
                            self.logger.warning(f"请求失败code不等于0 {resp}, 重试 {retries}/{self.max_retries}")
                            await asyncio.sleep(5)
                            continue
                    return resp
            except Exception as e:
                retries += 1
                if retries >= self.max_retries:
                    if self.logger:
                        self.logger.error(f"请求失败达到最大重试次数: {e}")
                    if self.aliyun_log:
                        self.aliyun_log.logging(
                            code="9006",
                            message=f"请求异常达到最大重试次数",
                            data=f"{url}"
                        )
                    return
                if self.logger:
                    self.logger.warning(f"请求失败 {e}, 重试 {retries}/{self.max_retries}")
                await asyncio.sleep(5)