AutoScraperX/spiders/recommendspider.py

# spiders/recommendspider.py
from typing import List, Dict, Optional
from spiders.basespider import BaseSpider
from core.utils.extractors import safe_extract


class RecommendSpider(BaseSpider):
    """推荐模式爬虫 - 重新封装版本"""

    def __init__(self, rule_dict: Dict, user_list: List, env: str = "prod",
                 request_client=None, db_service=None, mq_producer=None):
        super().__init__(rule_dict, user_list, env, request_client, db_service, mq_producer)
        self.last_response = None

    async def execute(self):
        """执行核心逻辑 - 使用 make_request 方法"""
        if not await self.is_video_count_sufficient():
            self.logger.info("视频数量已达到上限，跳过执行")
            return

        iteration = 0
        while iteration < self.loop_times and await self.is_video_count_sufficient():
            self.logger.info(f"执行第 {iteration + 1} 轮")

            # 准备请求体
            request_body = self.request_preparer.prepare(
                self.request_body_template,
                self.last_response or {}
            )

            # 发送请求 - 使用 make_request 方法
            response = await self.make_request(request_body)

            if not response:
                self.logger.info("未获取到响应数据")
                iteration += 1
                await self.wait_between_iterations()
                continue

            self.last_response = response

            # 提取数据
            data_list = safe_extract(response, self.data_path)
            if not data_list:
                self.logger.info("未获取到数据")
                iteration += 1
                await self.wait_between_iterations()
                continue

            # 处理数据
            await self.process_data(data_list)

            iteration += 1
            await self.wait_between_iterations()