automatic_crawler/scheduler/search_spider.py

"""
获取热点日历
"""
import schedule
import requests
import json
import datetime


class PQuanCalendar(object):
    """
    热点日历
    """

    def __init__(self):
        self.url = "https://www.adguider.com/sv1/calendar/getCalendarAjax"
        self.c_dict = {}

    def get_calendar(self):
        """
        请求日历
        :return:
        """
        today = datetime.datetime.today()
        tomorrow = today + datetime.timedelta(days=1)
        tomorrow = tomorrow.strftime("%Y-%m-%d")
        payload = json.dumps({
            "startTime": tomorrow,
            "endTime": tomorrow,
            "fdIdList": [
                13,
                4,
                3,
                10,
                5,
                8,
            ]
        })
        headers = {
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'Accept-Language': 'en,zh;q=0.9,zh-CN;q=0.8',
            'Connection': 'keep-alive',
            'Content-Type': 'application/json',
            'Origin': 'https://www.adguider.com',
            'Referer': 'https://www.adguider.com/sv1/calendar/getCalendar?mode=1&startDate=2024/03/13',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
        }
        response = requests.post(self.url, headers=headers, data=payload)
        result = response.json()
        # print(json.dumps(result, ensure_ascii=False, indent=4))
        return result

    def process_response(self, response_json):
        """
        对获取的数据进行处理， 处理结构为
        date_info: [festival_obj1, festival_obj2, festival_obj3, festival_obj4 ......]
        :param response_json:
        :return: calender_dict
        """
        if response_json['data']:
            for cat_obj in response_json['data']:
                if cat_obj.get("adFestivalFixedVos"):
                    festival_list = cat_obj["adFestivalFixedVos"]
                    category = cat_obj["ftName"]
                    self.c_dict[category] = [item['ffName'] for item in festival_list]
            print(json.dumps(self.c_dict, ensure_ascii=False, indent=4))
        return self.c_dict


class SearchSpider(object):
    """
    定时从日历中获取明天的节日，通过节日去搜索视频，并且把视频发送至 ETL 下载
    """
    def __init__(self, festival_dict):
        self.key_list = festival_dict


if __name__ == '__main__':
    P = PQuanCalendar()
    res = P.get_calendar()
    P.process_response(res)