piaoquan_crawler/weixinzhishu/weixinzhishu_main/weixinzhishu.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2023/2/10
import os
import sys
import time
from datetime import date, timedelta
import requests
import json
sys.path.append(os.getcwd())
from common.feishu import Feishu
from common.common import Common


class Weixinzhishu:
    pageNum = 1

    # 获取微信 key / openid
    @classmethod
    def get_wechat_key(cls, log_type, crawler):
        """
        获取微信 key / openid
        https://w42nne6hzg.feishu.cn/sheets/shtcnqhMRUGunIfGnGXMOBYiy4K?sheet=sVL74k
        :param log_type: 日志名
        :param crawler: 哪款爬虫，填写:weixinzhishu
        :return: search_key, openid
        """
        try:
            sheet = Feishu.get_values_batch(log_type, crawler, 'sVL74k')
            for i in range(len(sheet)):
                search_key = sheet[1][1]
                openid = sheet[1][2]
                return search_key, openid
        except Exception as e:
            Common.logger(log_type, crawler).error(f"wechat_key:{e}\n")

    # 获取热词
    @classmethod
    def get_word(cls):
        url = '/hot/word/getAllWords'
        params = {
            'pageNum': cls.pageNum,  # 第几页，默认1，int
            'pageSize': 100  # 请求条目数，默认为100，int
        }
        response = requests.post(url=url, json=params)
        cls.pageNum += 1
        word_list = []
        print(response.text)
        return word_list

    # 获取热词分数
    @classmethod
    def get_word_score(cls, log_type, crawler, word_id, word):
        """
        获取热词分数
        :param log_type: 日志名
        :param crawler: 哪款爬虫，填写:weixinzhishu
        :param word_id: 热词 ID
        :param word: 热词
        :return: 热词 7 天指数，例如：
        {'id': 1,
        'word': '消息',
        'wechatScores': [
        {'score': 95521022, 'scoreDate': '2023-02-07'},
        {'score': 97315283, 'scoreDate': '2023-02-08'},
        {'score': 109845849, 'scoreDate': '2023-02-09'},
        {'score': 107089560, 'scoreDate': '2023-02-10'},
        {'score': 102658391, 'scoreDate': '2023-02-11'},
        {'score': 93843701, 'scoreDate': '2023-02-12'},
        {'score': 100211894, 'scoreDate': '2023-02-13'}]}
        """
        try:
            wechat_key = cls.get_wechat_key(log_type, crawler)
            search_key = wechat_key[0]
            openid = wechat_key[-1]
            end_ymd = (date.today() + timedelta(days=0)).strftime("%Y%m%d")
            start_ymd = (date.today() + timedelta(days=-7)).strftime("%Y%m%d")
            url = "https://search.weixin.qq.com/cgi-bin/wxaweb/wxindex"
            payload = json.dumps({
                "openid": openid,
                "search_key": search_key,
                "cgi_name": "GetDefaultIndex",
                "start_ymd": start_ymd,
                "end_ymd": end_ymd,
                "query": word
            })
            headers = {
                'Host': 'search.weixin.qq.com',
                'content-type': 'application/json',
                'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.32(0x1800202a) NetType/WIFI Language/zh_CN',
                'Referer': 'https://servicewechat.com/wxc026e7662ec26a3a/42/page-frame.html'
            }
            response = requests.request("POST", url, headers=headers, data=payload)
            wechat_score_list = []
            word_wechat_score_dict = {
                "id": word_id,
                "word": word,
                "wechatScores": wechat_score_list,
            }
            if response.json()['code'] == -10000:
                # Common.logger(log_type, crawler).warning(f"response:{response.json()['msg']} 休眠 10 秒，重新获取\n")
                # time.sleep(10)
                # cls.get_word_score(log_type, crawler, word_id, word)
                return None
            elif response.json()['code'] == -10002:
                # Common.logger(log_type, crawler).info(f'{word}:该词暂未收录')
                # # 数据写入飞书
                # now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time())))
                # values = [[now, word, "该词暂未收录"]]
                # Feishu.insert_columns(log_type, crawler, "5011a2", "ROWS", 1, 2)
                # time.sleep(0.5)
                # Feishu.update_values(log_type, crawler, "5011a2", "F2:Z2", values)
                # Common.logger(log_type, crawler).info(f'热词"{word}"微信指数数据写入飞书成功\n')
                return word_wechat_score_dict
            elif response.json()['code'] != 0:
                Common.logger(log_type, crawler).info(f'response:{response.text}\n')
                return word_wechat_score_dict
            else:
                time_index = response.json()['content']['resp_list'][0]['indexes'][0]['time_indexes']
                for i in range(len(time_index)):
                    score_time = time_index[i]['time']
                    score_time_str = f"{str(score_time)[:4]}-{str(score_time)[4:6]}-{str(score_time)[6:]}"
                    score = time_index[i]['score']
                    wechat_score_dict = {"score": score, "scoreDate": score_time_str}
                    wechat_score_list.append(wechat_score_dict)

                    # # 数据写入飞书
                    # now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time())))
                    # values = [[now, word, score_time_str, score]]
                    # Feishu.insert_columns(log_type, crawler, "5011a2", "ROWS", 1, 2)
                    # time.sleep(0.5)
                    # Feishu.update_values(log_type, crawler, "5011a2", "F2:Z2", values)
                    # Common.logger(log_type, crawler).info(f'热词"{word}"微信指数数据写入飞书成功\n')

                return word_wechat_score_dict
        except Exception as e:
            Common.logger(log_type, crawler).error(f"weixinzhishu异常:{e}\n")

    # 获取微信指数
    @classmethod
    def get_wechat_score(cls, log_type, crawler):
        """
        获取微信指数
        :param log_type: 日志名
        :param crawler: 哪款爬虫
        :return: 热词指数列表
        """
        while True:
            word_list = cls.get_word()
            if len(word_list) == 0:
                Common.logger(log_type, crawler).info(f"热词更新完毕\n")
                cls.pageNum = 1
                return []
            else:
                wechat_score_data = []
                for i in range(len(word_list)):
                    word_id = word_list[i]['Id']
                    word = word_list[i]['word']
                    word_score_dict = cls.get_word_score(log_type, crawler, word_id, word)
                    wechat_score_data.append(word_score_dict)
                    return wechat_score_data

    # 更新微信指数
    @classmethod
    def update_wechat_score(cls, log_type, crawler):
        """
        更新热词微信指数
        :param log_type: 日志名
        :param crawler: 哪款爬虫
        :return: {"code":200, "message":"success"}
        """
        data = {
            'data': cls.get_wechat_score(log_type, crawler)
        }

        url = '/hot/word/updateWechatScore'
        params = {
            'data': data
        }
        response = requests.post(url=url, json=params)
        print(response.text)

    @classmethod
    def update_wechat_score_test(cls, log_type, crawler):
        our_word_list = []
        out_word_list = []
        our_word_sheet = Feishu.get_values_batch(log_type, 'weixinzhishu_search_word', "nCudsM")
        out_word_sheet = Feishu.get_values_batch(log_type, 'weixinzhishu_search_word', "D80uEf")
        for x in our_word_sheet:
            for y in x:
                if y is None:
                    pass
                else:
                    our_word_list.append(y)
        for x in out_word_sheet:
            for y in x:
                if y is None:
                    pass
                else:
                    out_word_list.append(y)
        word_list = our_word_list+out_word_list
        word_score_list = []
        # for i in range(len(word_list)):
        for i in range(100):
            while True:
                Common.logger(log_type, crawler).info(f"word_id:{i + 1}, word:{word_list[i]}")
                word_score = cls.get_word_score(log_type, crawler, int(i + 1), word_list[i])
                if word_score is None:
                    Common.logger(log_type, crawler).info("微信key过期，10秒钟后重试")
                    time.sleep(10)
                else:
                    word_score_list.append(word_score)
                    Common.logger(log_type, crawler).info(f'微信指数：{word_score}\n')
                    break

        word_dict = {
            "data": word_score_list
        }
        return word_dict

    @classmethod
    def get_score_test(cls, log_type, crawler, word_id, word):
        wechat_key = cls.get_wechat_key(log_type, crawler)
        search_key = wechat_key[0]
        openid = wechat_key[-1]
        end_ymd = (date.today() + timedelta(days=0)).strftime("%Y%m%d")
        start_ymd = (date.today() + timedelta(days=-7)).strftime("%Y%m%d")
        url = "https://search.weixin.qq.com/cgi-bin/wxaweb/wxindex"
        payload = json.dumps({
            "openid": openid,
            "search_key": search_key,
            "cgi_name": "GetDefaultIndex",
            "start_ymd": start_ymd,
            "end_ymd": end_ymd,
            "query": word
        })
        headers = {
            'Host': 'search.weixin.qq.com',
            'content-type': 'application/json',
            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.32(0x1800202a) NetType/WIFI Language/zh_CN',
            'Referer': 'https://servicewechat.com/wxc026e7662ec26a3a/42/page-frame.html'
        }
        response = requests.request("POST", url, headers=headers, data=payload)
        wechat_score_list = []
        word_wechat_score_dict = {
            "id": word_id,
            "word": word,
            "wechatScores": wechat_score_list,
        }
        if response.json()['code'] == -10000:
            print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(time.time())))} response:{response.json()['msg']} 休眠 10 秒，重新获取\n")
            time.sleep(10)
            cls.get_score_test(log_type, crawler, word_id, word)
        elif response.json()['code'] == -10002:
            print("该词暂未收录")
            print(f"{word_wechat_score_dict}")
        elif response.json()['code'] != 0:
            print(f"{word_wechat_score_dict}")
        else:
            time_index = response.json()['content']['resp_list'][0]['indexes'][0]['time_indexes']
            for i in range(len(time_index)):
                score_time = time_index[i]['time']
                score_time_str = f"{str(score_time)[:4]}-{str(score_time)[4:6]}-{str(score_time)[6:]}"
                score = time_index[i]['score']
                wechat_score_dict = {"score": score, "scoreDate": score_time_str}
                wechat_score_list.append(wechat_score_dict)
                print(f"wechat_score_dict:{wechat_score_dict}")
            print(word_wechat_score_dict)


if __name__ == "__main__":
    Weixinzhishu.get_score_test('weixin', 'weixinzhishu', 1 , "春晚")
    #
    # word_dict_demo = Weixinzhishu.update_wechat_score_test('weixin', 'weixinzhishu')
    # print(word_dict_demo)

    pass