LongArticlesJob-fork/spider/weixinCategoryCrawler.py

"""
@author: luojunhui
抓取全局品类文章
"""

import time

from tqdm import tqdm

from applications import WeixinSpider, Functions, DeNetMysql


class weixinCategory(object):
    """
    微信全局品类账号抓取
    """

    def __init__(self):
        self.spider_client = DeNetMysql()
        self.spider = WeixinSpider()
        self.function = Functions()

    def getAccountList(self):
        """
        获取账号
        :return:
        """
        now_time = int(time.time())
        twenty_hours_ago = now_time - 3600 * 20
        sql = f"""select * from long_article_accounts_outside where latest_article_timestamp < {twenty_hours_ago};"""
        account_tuple = self.spider_client.select(sql)
        result = [
            {
                "gh_id": i[0],
                "platform": i[1],
                "account_name": i[2],
                "category": i[3],
                "latest_timestamp": i[4],
            }
            for i in account_tuple
        ]
        return result

    def updateDataIntoMysql(self, gh_id, category, article_list):
        """
        将数据更新到数据库
        :return:
        """
        for article_obj in article_list:
            detail_article_list = article_obj["AppMsg"]["DetailInfo"]
            for obj in detail_article_list:
                try:
                    show_stat = self.function.show_desc_to_sta(obj["ShowDesc"])
                    show_view_count = show_stat.get("show_view_count", 0)
                    show_like_count = show_stat.get("show_like_count", 0)
                    insert_sql = f"""
                        insert into crawler_meta_article
                        (platform, mode, category, out_account_id, article_index, title, link, read_cnt, like_cnt, description, publish_time, crawler_time, status, unique_index)
                        VALUES 
                        (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
                    """
                    self.spider_client.update(
                        sql=insert_sql,
                        params=(
                            "weixin",
                            "account",
                            category,
                            gh_id,
                            obj['ItemIndex'],
                            obj["Title"],
                            obj["ContentUrl"],
                            show_view_count,
                            show_like_count,
                            obj["Digest"],
                            obj["send_time"],
                            int(time.time()),
                            1,
                            self.function.generateGzhId(obj["ContentUrl"]),
                        ),
                    )
                except Exception as e:
                    print(e)

    def updateLatestAccountTimeStamp(self, gh_id):
        """
        更新账号的最新时间戳
        :return:
        """
        select_sql = f"""
        SELECT publish_time 
        From crawler_meta_article 
        WHERE out_account_id = '{gh_id}'
        ORDER BY publish_time DESC LIMIT 1;
        """
        result = self.spider_client.select(select_sql)
        time_stamp = result[0][0]
        update_sql = f"""
            update long_article_accounts_outside
            set latest_article_timestamp = %s
            where account_id = %s;
        """
        self.spider_client.update(sql=update_sql, params=(time_stamp, gh_id))

    def updateEachAccountArticles(self, gh_id, category, latest_time_stamp, index=None):
        """
        更新账号文章
        :return:
        """
        response = self.spider.update_msg_list(ghId=gh_id, index=index)
        msg_list = response.get("data", {}).get("data")
        if msg_list:
            last_article_in_this_msg = msg_list[-1]
            self.updateDataIntoMysql(
                gh_id=gh_id, category=category, article_list=msg_list
            )
            last_time_stamp_in_this_msg = last_article_in_this_msg["AppMsg"]["BaseInfo"]["UpdateTime"]
            if latest_time_stamp < last_time_stamp_in_this_msg:
                next_cursor = response["data"]["next_cursor"]
                return self.updateEachAccountArticles(
                    gh_id=gh_id,
                    latest_time_stamp=latest_time_stamp,
                    category=category,
                    index=next_cursor,
                )
            else:
                # 更新最近抓取时间
                self.updateLatestAccountTimeStamp(gh_id=gh_id)
        else:
            print("No more data")


if __name__ == "__main__":
    wxCategory = weixinCategory()
    account_list = wxCategory.getAccountList()
    for account in tqdm(account_list):
        try:
            wxCategory.updateEachAccountArticles(
                gh_id=account["gh_id"],
                category=account["category"],
                latest_time_stamp=account["latest_timestamp"],
            )
        except Exception as e:
            print("fail because of {}".format(e))