LongArticlesJob/run_video_account_crawler.py

"""
@author: luojunhui
执行视频&&账号的抓取
"""
import traceback

from datetime import datetime
from argparse import ArgumentParser

from applications import bot
from coldStartTasks.crawler import WeixinAccountCrawler, WeixinVideoCrawler

account_crawler = WeixinAccountCrawler()
video_crawler = WeixinVideoCrawler()


def main():
    """
    主函数
    :return:
    """
    parser = ArgumentParser()
    parser.add_argument("--run-date",
                        help="Run only once for date in format of %Y%m%d. \
                            If no specified, run as daily jobs.")
    args = parser.parse_args()

    if args.run_date:
        run_date = datetime.strptime(args.run_date, "%Y-%m-%d")
        print("Run in manual mode. Date: {}".format(args.run_date))
    else:
        run_date = datetime.today()
    # # 先执行账号抓取
    # try:
    #     account_crawler.run(run_date)
    # except Exception as e:
    #     error_msg = traceback.format_exc()
    #     bot(
    #         title='账号抓取v1执行失败',
    #         detail={
    #             "error": str(e),
    #             "traceback": error_msg
    #         }
    #     )
    # 再执行文章抓取
    try:
        video_crawler.run()
    except Exception as e:
        error_msg = traceback.format_exc()
        bot(
            title='视频抓取执行失败',
            detail={
                "error": str(e),
                "traceback": error_msg
            }
        )
    # 再执行账号抓取v2
    try:
        account_crawler.run_v2()
    except Exception as e:
        error_msg = traceback.format_exc()
        bot(
            title='账号抓取V2执行失败',
            detail={
                "error": str(e),
                "traceback": error_msg
            }
        )


if __name__ == '__main__':
    main()