automatic_crawler/app/off_line_controler.py

import os
import sys
import time
import schedule
import multiprocessing

sys.path.append(os.getcwd())

from scheduler import SpiderHome


class SpiderScheduler(object):
    """
    线下爬虫调度器
    """
    SH = SpiderHome()

    @classmethod
    def protect_spider_timeout(cls, function, hour):
        """
        守护进程，在程序启动后的某一个时段内守护爬虫进程
        :param function: 被守护的函数
        :param hour: 守护时长 / hour
        """
        run_time_limit = hour * 3600
        start_time = time.time()
        process = multiprocessing.Process(target=function)
        process.start()
        while True:
            if time.time() - start_time >= run_time_limit:
                process.terminate()
                break
            if not process.is_alive():
                process.terminate()
                time.sleep(60)
                os.system("adb forward --remove-all")
                process = multiprocessing.Process(target=function)
                process.start()
            time.sleep(60)

    @classmethod
    def run_xng_plus(cls, hour):
        """
        “小年糕+”推荐爬虫
        :param hour:
        """
        cls.protect_spider_timeout(function=cls.SH.run_xng_plus, hour=hour)

    @classmethod
    def run_zhuFuQuanZi(cls):
        print("hello")

    @classmethod
    def run_spss(cls, hour):
        """
        "视频刷刷"推荐爬虫
        :param hour:
        """
        cls.protect_spider_timeout(function=cls.SH.run_spss, hour=hour)

    @classmethod
    def run_zfhybf(cls, hour):
        """
        "祝福好运暴富"推荐爬虫
        :param hour:
        """
        cls.protect_spider_timeout(function=cls.SH.run_zfhybf, hour=hour)

    @classmethod
    def run_spss_id(cls, hour):
        cls.protect_spider_timeout(function=cls.SH.run_spss_id, hour=hour)

    @classmethod
    def run_xng_id(cls, hour):
        cls.protect_spider_timeout(function=cls.SH.run_xng_id, hour=hour)


if __name__ == "__main__":
    SC = SpiderScheduler()
    SC.run_xng_plus(hour=100)
    # SC.run_spss(hour=5)
    # # schedule.every().day.at("20:06").do(SC.run_xng_plus, hour=1)
    # schedule.every().day.at("20:30").do(SC.run_spss, hour=1)
    #
    # schedule.every().day.at("18:30").do(SC.run_spss_id, hour=1)
    #
    # while True:
    #     schedule.run_pending()