LongArticlesJob/spider/weixinAssociationCrawler.py

"""
@author: luojunhui
微信联想抓取
"""
import json

from tqdm import tqdm

from applications import PQMySQL
from applications.spiderTool import SpiderTools


class weixinAssociation(object):
    """
    微信联想方法
    """
    pq_mysql_client = PQMySQL()
    spider_tool = SpiderTools()

    @classmethod
    def getAssociationAccounts(cls):
        """
        获取已经联想过的账号
        :return:
        """
        select_sql = f"""
        SELECT distinct(gh_id)
        FROM long_articles_accounts
        where is_using = 1 and account_category = 'association';"""
        account_id_tuple = cls.pq_mysql_client.select(select_sql)
        account_id_list = [list(i) for i in account_id_tuple]
        return account_id_list

    @classmethod
    def deal(cls):
        """
        main function
        :return:
        """
        account_info_list = cls.getAssociationAccounts()
        for line in tqdm(account_info_list[1:]):
            gh_id = line[0]
            cls.spider_tool.searchEachAccountArticlesSinglePage(
                gh_id=gh_id,
                category="association"
            )




w = weixinAssociation()
w.deal()