Merge branch 'wangkun'

add gongzhonghao and weixinzhishu

README.MD

@@ -144,8 +144,14 @@ ps aux | grep run_gongzhonghao | grep -v grep | awk '{print $2}' | xargs kill -9
 nohup python3 -u /data5/piaoquan_crawler/weixinzhishu/weixinzhishu_main/weixinzhishu_inner_long.py >>/data5/piaoquan_crawler/weixinzhishu/logs/nohup_inner_long.log 2>&1 &
 nohup python3 -u /data5/piaoquan_crawler/weixinzhishu/weixinzhishu_main/weixinzhishu_out.py >>/data5/piaoquan_crawler/weixinzhishu/logs/nohup_out.log 2>&1 &
 nohup python3 -u /data5/piaoquan_crawler/weixinzhishu/weixinzhishu_main/weixinzhishu_inner_sort.py >>/data5/piaoquan_crawler/weixinzhishu/logs/nohup_inner_sort.log 2>&1 &
+线下调试
+抓取今日微信指数
+python3 /Users/wangkun/Desktop/crawler/piaoquan_crawler/weixinzhishu/weixinzhishu_main/run_weixinzhishu_today_score.py
+检测进程
+ps aux | grep WeChat.app
 ps aux | grep run_weixinzhishu
 ps aux | grep weixinzhishu | grep -v grep | awk '{print $2}' | xargs kill -9
+ps aux | grep 微信 | grep -v grep | awk '{print $2}' | xargs kill -9
 获取 wechat_key 设备: Mac Air 
 cd ~ && source ./base_profile && ps aux | grep weixinzhishu | grep -v grep | awk '{print $2}' | xargs kill -9 && cd /Users/piaoquan/Desktop/piaoquan_crawler && nohup python3 -u weixinzhishu/weixinzhishu_key/search_key_mac.py >> weixinzhishu/nohup.log 2>&1 &
 ```

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow.py

@@ -5,7 +5,6 @@ import datetime
 import difflib
 import json
 import os
-import random
 import shutil
 import sys
 import time
@@ -334,8 +333,8 @@ class GongzhonghaoFollow:
                             return
                         cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
 
-                    Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
-                    time.sleep(random.randint(60*5, 60*10))
+                    Common.logger(log_type, crawler).info('休眠 60 秒\n')
+                    time.sleep(60)
         except Exception as e:
             Common.logger(log_type, crawler).error(f"get_videoList异常:{e}\n")
 
@@ -504,8 +503,8 @@ class GongzhonghaoFollow:
                 Common.logger(log_type, crawler).info(f'获取 {user_name} 公众号视频\n')
                 cls.get_videoList(log_type, crawler, user_name, index, oss_endpoint, env)
                 cls.begin = 0
-                Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
-                time.sleep(random.randint(60*5, 60*10))
+                Common.logger(log_type, crawler).info('休眠 60 秒\n')
+                time.sleep(60)
             except Exception as e:
                 Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
 

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow_2.py

@@ -5,7 +5,6 @@ import datetime
 import difflib
 import json
 import os
-import random
 import shutil
 import sys
 import time
@@ -330,8 +329,8 @@ class GongzhonghaoFollow2:
                             return
                         cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
 
-                    Common.logger(log_type, crawler).info('随机休眠 60*3-60*8 秒\n')
-                    time.sleep(random.randint(60*5, 60*10))
+                    Common.logger(log_type, crawler).info('休眠 60 秒\n')
+                    time.sleep(60)
         except Exception as e:
             Common.logger(log_type, crawler).error(f"get_videoList异常:{e}\n")
 
@@ -500,8 +499,8 @@ class GongzhonghaoFollow2:
                 Common.logger(log_type, crawler).info(f'获取 {user_name} 公众号视频\n')
                 cls.get_videoList(log_type, crawler, user_name, index, oss_endpoint, env)
                 cls.begin = 0
-                Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
-                time.sleep(random.randint(60*5, 60*10))
+                Common.logger(log_type, crawler).info('休眠 60 秒\n')
+                time.sleep(60)
             except Exception as e:
                 Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
 

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow_3.py

@@ -5,7 +5,6 @@ import datetime
 import difflib
 import json
 import os
-import random
 import shutil
 import sys
 import time
@@ -330,8 +329,8 @@ class GongzhonghaoFollow3:
                             return
                         cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
 
-                    Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
-                    time.sleep(random.randint(60*5, 60*10))
+                    Common.logger(log_type, crawler).info('休眠 60 秒\n')
+                    time.sleep(60)
         except Exception as e:
             Common.logger(log_type, crawler).error("get_videoList异常:{}\n", e)
 
@@ -500,8 +499,8 @@ class GongzhonghaoFollow3:
                 Common.logger(log_type, crawler).info(f'获取 {user_name} 公众号视频\n')
                 cls.get_videoList(log_type, crawler, user_name, index, oss_endpoint, env)
                 cls.begin = 0
-                Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
-                time.sleep(random.randint(60*5, 60*10))
+                Common.logger(log_type, crawler).info('休眠 60 秒\n')
+                time.sleep(60)
             except Exception as e:
                 Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
 

gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow.py

@@ -4,6 +4,8 @@
 import argparse
 import os
 import sys
+import time
+
 sys.path.append(os.getcwd())
 from common.common import Common
 from gongzhonghao.gongzhonghao_follow.gongzhonghao_follow import GongzhonghaoFollow
@@ -24,7 +26,8 @@ class Main:
                                               env=env)
             Common.del_logs(log_type, crawler)
             GongzhonghaoFollow.begin = 0
-            Common.logger(log_type, crawler).info('公众号抓取一轮完毕\n')
+            Common.logger(log_type, crawler).info('公众号抓取一轮完毕，休眠 8 小时\n')
+            time.sleep(3600*8)
         except Exception as e:
             Common.logger(log_type, crawler).info(f"公众号抓取异常:{e}\n")
 

gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow_2.py

@@ -4,6 +4,8 @@
 import argparse
 import os
 import sys
+import time
+
 sys.path.append(os.getcwd())
 from common.common import Common
 from gongzhonghao.gongzhonghao_follow.gongzhonghao_follow_2 import GongzhonghaoFollow2
@@ -24,7 +26,8 @@ class Main:
                                                env=env)
             Common.del_logs(log_type, crawler)
             GongzhonghaoFollow2.begin = 0
-            Common.logger(log_type, crawler).info('公众号抓取一轮完毕\n')
+            Common.logger(log_type, crawler).info('公众号抓取一轮完毕，休眠 8 小时\n')
+            time.sleep(3600*8)
         except Exception as e:
             Common.logger(log_type, crawler).info(f"公众号抓取异常:{e}\n")
 

gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow_3.py

@@ -4,6 +4,8 @@
 import argparse
 import os
 import sys
+import time
+
 sys.path.append(os.getcwd())
 from common.common import Common
 from gongzhonghao.gongzhonghao_follow.gongzhonghao_follow_3 import GongzhonghaoFollow3
@@ -24,7 +26,8 @@ class Main:
                                                env=env)
             Common.del_logs(log_type, crawler)
             GongzhonghaoFollow3.begin = 0
-            Common.logger(log_type, crawler).info('公众号抓取一轮完毕\n')
+            Common.logger(log_type, crawler).info('公众号抓取一轮完毕，休眠 8 小时\n')
+            time.sleep(3600*8)
         except Exception as e:
             Common.logger(log_type, crawler).info(f"公众号抓取异常:{e}\n")
 

weixinzhishu/weixinzhishu_key/search_key_mac.py

@@ -2,10 +2,18 @@
 # @Author: wangkun
 # @Time: 2023/2/20
 """
-安装 atomac：
-1.翻墙
-2.pip3 install git+https://github.com/pyatom/pyatom/
+1. 安装 atomac：
+    1.1.翻墙
+    1.2.pip3 install git+https://github.com/pyatom/pyatom/
+2. 启动 Charles.exe:
+    2.1 选中 Proxy - Windows Proxy
+    2.2 选中 Tools - Auto Save - Enable Auto Save
+3. 启动 Python 脚本:
+    3.1 cd ./piaoquan_crawler/
+    3.2 python ./weixinzhishu/weixinzhishu_main/search_key_mac.py
+4. 每 10 秒获取最新search_key，写入飞书: https://w42nne6hzg.feishu.cn/sheets/shtcnqhMRUGunIfGnGXMOBYiy4K?sheet=sVL74k
 """
+
 import json
 import os
 import sys
@@ -30,10 +38,30 @@ class SearchKey:
             # 获取当前应用window
             window = automator.windows()[0]
             Common.logger(log_type, crawler).info(f"当前应用window:{window.AXTitle}")
+
+            # # 登录
+            # login_btn = window.findFirstR(AXRole="AXButton", AXDescription="进入微信")
+            # # print(login_btn.getAttributes())
+            # # print(f"AXEnabled:{login_btn.AXEnabled}")
+            # # print(f"AXDescription:{login_btn.AXDescription}")
+            # # print(f"AXParent:{login_btn.AXParent}")
+            # # print(f"AXSize:{login_btn.AXSize}")
+            # # print(f"AXFocused:{login_btn.AXFocused}")
+            # # print(f"AXRole:{login_btn.AXRole}")
+            # # print(f"AXTopLevelUIElement:{login_btn.AXTopLevelUIElement}")
+            # # print(f"AXHelp:{login_btn.AXHelp}")
+            # # print(f"AXPosition:{login_btn.AXPosition}")
+            # # print(f"AXWindow:{login_btn.AXWindow}")
+            # # print(f"AXRoleDescription:{login_btn.AXRoleDescription}")
+            # # print(f"AXIdentifier:{login_btn.AXIdentifier}")
+            # if login_btn:
+            #     Common.logger(log_type, crawler).info("发现登录按钮，点击登录")
+            #     login_btn.Press()
+            #     time.sleep(1)
+
             # 查找聊天按钮
             Common.logger(log_type, crawler).info("查找聊天按钮")
             chat_btn = window.findFirstR(AXHelp="微信", AXRole="AXRadioButton")
-            # print(chat_btn.getAttributes())
             # 点击聊天按钮，展开聊天列表
             Common.logger(log_type, crawler).info("点击聊天按钮")
             chat_btn.Press()
@@ -52,6 +80,7 @@ class SearchKey:
                 chat_help.clickMouseButtonLeft(chat_help_coor)
             time.sleep(1)
 
+            # 查找微信指数小程序消息
             Common.logger(log_type, crawler).info("查找微信指数小程序消息")
             weixinzhishu = window.findFirstR(AXValue="微信指数", AXRole="AXStaticText")
             Common.logger(log_type, crawler).info(f"微信指数小程序:{weixinzhishu}")
@@ -201,9 +230,9 @@ class SearchKey:
 
 
 if __name__ == "__main__":
-    # SearchKey.start_wechat("search", "weixinzhishu")
+    SearchKey.start_wechat("search", "weixinzhishu")
     # SearchKey.del_wechat_key("search", "weixinzhishu")
     # SearchKey.write_wechat_key("search", "weixinzhishu")
     # print(SearchKey.get_wechat_key("search", "weixinzhishu"))
-    SearchKey.main("search", "weixinzhishu")
+    # SearchKey.main("search", "weixinzhishu")
     pass

weixinzhishu/weixinzhishu_main/run_weixinzhishu_today_score.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/4/6
+import os
+import sys
+sys.path.append(os.getcwd())
+from common.common import Common
+from weixinzhishu.weixinzhishu_score.weixinzhishu_today_score import TodayScore
+
+
+def main(log_type, crawler, env):
+    Common.logger(log_type, crawler).info("开始抓取今日微信指数\n")
+    TodayScore.get_today_wechat_score(log_type, crawler, env)
+    Common.del_logs(log_type, crawler)
+    Common.logger(log_type, crawler).info("今日微信指数抓取完毕\n")
+
+
+if __name__ == "__main__":
+    main("today-score", "weixinzhishu", "prod")

weixinzhishu/weixinzhishu_score/weixinzhishu_score.py

@@ -26,11 +26,15 @@ class Weixinzhishu:
         :return: search_key, openid
         """
         try:
-            sheet = Feishu.get_values_batch(log_type, crawler, 'sVL74k')
-            for i in range(len(sheet)):
-                search_key = sheet[1][1]
-                openid = sheet[1][2]
-                return search_key, openid
+            while True:
+                sheet = Feishu.get_values_batch(log_type, crawler, 'sVL74k')
+                if sheet is None:
+                    time.sleep(1)
+                    continue
+                for i in range(len(sheet)):
+                    search_key = sheet[1][1]
+                    openid = sheet[1][2]
+                    return search_key, openid
         except Exception as e:
             Common.logger(log_type, crawler).error(f"wechat_key:{e}\n")
 
@@ -79,10 +83,6 @@ class Weixinzhishu:
         try:
             while True:
                 wechat_key = cls.get_wechat_key(log_type, crawler)
-                if wechat_key is None:
-                    Common.logger(log_type, crawler).info(f"wechat_key:{wechat_key}")
-                    time.sleep(10)
-                    continue
                 search_key = wechat_key[0]
                 openid = wechat_key[-1]
                 start_ymd = (date.today() + timedelta(days=-7)).strftime("%Y%m%d")
@@ -112,7 +112,8 @@ class Weixinzhishu:
                 if response.json()['code'] == -10000:
                     Common.logger(log_type, crawler).warning(f"response:{response.json()['msg']} 休眠 10 秒，重新获取")
                     time.sleep(10)
-                elif response.json()['code'] == -10002:
+                    continue
+                if response.json()['code'] == -10002:
                     Common.logger(log_type, crawler).info(f'{word}:该词暂未收录')
                     return word_wechat_score_dict
                 elif response.json()['code'] != 0:

weixinzhishu/weixinzhishu_score/weixinzhishu_today_score.py

@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/4/6
+import json
+import os
+import sys
+import time
+from datetime import date, timedelta
+import requests
+sys.path.append(os.getcwd())
+from common.common import Common
+from common.feishu import Feishu
+
+
+class TodayScore:
+    # 翻页参数
+    pageNum = 1
+
+    # 获取微信 key / openid
+    @classmethod
+    def get_wechat_key(cls, log_type, crawler):
+        """
+        获取微信 key / openid
+        https://w42nne6hzg.feishu.cn/sheets/shtcnqhMRUGunIfGnGXMOBYiy4K?sheet=sVL74k
+        :param log_type: 日志名
+        :param crawler: 哪款爬虫，填写:weixinzhishu
+        :return: search_key, openid
+        """
+        try:
+            while True:
+                sheet = Feishu.get_values_batch(log_type, crawler, 'sVL74k')
+                if sheet is None:
+                    time.sleep(1)
+                    continue
+                for i in range(len(sheet)):
+                    search_key = sheet[1][1]
+                    openid = sheet[1][2]
+                    return search_key, openid
+        except Exception as e:
+            Common.logger(log_type, crawler).error(f"wechat_key:{e}\n")
+
+    # 获取热词
+    @classmethod
+    def get_today_words(cls, log_type, crawler, env):
+        try:
+            if env == "dev:":
+                url = 'http://testhot-words-internal.piaoquantv.com/hot/word/getTodayWords'
+            else:
+                url = "http://hot-words-internal.piaoquantv.com/hot/word/getTodayWords"
+            params = {
+                'pageNum': cls.pageNum,  # 第几页，默认1，int
+                'pageSize': 100  # 请求条目数，默认为100，int
+            }
+            response = requests.post(url=url, json=params)
+            cls.pageNum += 1
+            if response.status_code != 200:
+                Common.logger(log_type, crawler).warning(f"get_today_words:{response.status_code}, {response.text}\n")
+            elif response.json()['message'] != "success":
+                Common.logger(log_type, crawler).warning(f"get_word_response:{response.status_code}, {response.json()}\n")
+            else:
+                word_list = response.json()['data']['words']
+                return word_list
+        except Exception as e:
+            Common.logger(log_type, crawler).error(f"get_today_words:{e}\n")
+
+    # 获取热词分数
+    @classmethod
+    def get_word_score(cls, log_type, crawler, word_id, word):
+        """
+        获取热词分数
+        :param log_type: 日志名
+        :param crawler: 哪款爬虫，填写:weixinzhishu
+        :param word_id: 热词 ID
+        :param word: 热词
+        :return: 热词 7 天指数，例如：
+        {'id': 1,
+        'word': '消息',
+        'wechatScores': [
+        {'score': 95521022, 'scoreDate': '2023-02-07'},
+        {'score': 97315283, 'scoreDate': '2023-02-08'},
+        {'score': 109845849, 'scoreDate': '2023-02-09'},
+        {'score': 107089560, 'scoreDate': '2023-02-10'},
+        {'score': 102658391, 'scoreDate': '2023-02-11'},
+        {'score': 93843701, 'scoreDate': '2023-02-12'},
+        {'score': 100211894, 'scoreDate': '2023-02-13'}]}
+        """
+        try:
+            while True:
+                wechat_key = cls.get_wechat_key(log_type, crawler)
+                start_ymd = (date.today() + timedelta(days=-7)).strftime("%Y%m%d")
+                end_ymd = (date.today() + timedelta(days=0)).strftime("%Y%m%d")
+                url = "https://search.weixin.qq.com/cgi-bin/wxaweb/wxindex"
+                payload = json.dumps({
+                    "openid": wechat_key[-1],
+                    # "openid": '123',
+                    "search_key": wechat_key[0],
+                    "cgi_name": "GetDefaultIndex",
+                    "start_ymd": start_ymd,
+                    "end_ymd": end_ymd,
+                    "query": word
+                })
+                headers = {
+                    'Host': 'search.weixin.qq.com',
+                    'content-type': 'application/json',
+                    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.32(0x1800202a) NetType/WIFI Language/zh_CN',
+                    'Referer': 'https://servicewechat.com/wxc026e7662ec26a3a/42/page-frame.html'
+                }
+                response = requests.request("POST", url=url, headers=headers, data=payload)
+                wechat_score_list = []
+                word_wechat_score_dict = {
+                    "id": word_id,
+                    "word": word,
+                    "wechatScores": wechat_score_list,
+                }
+                if response.json()['code'] == -10000:
+                    Common.logger(log_type, crawler).warning(f"response:{response.json()['msg']} 休眠 5 秒，重新获取")
+                    time.sleep(5)
+                    continue
+                if response.json()['code'] == -10002:
+                    Common.logger(log_type, crawler).info(f'{word}:该词暂未收录')
+                    return word_wechat_score_dict
+                elif response.json()['code'] != 0:
+                    Common.logger(log_type, crawler).info(f'response:{response.text}\n')
+                    return word_wechat_score_dict
+                else:
+                    time_index = response.json()['content']['resp_list'][0]['indexes'][0]['time_indexes']
+                    for i in range(len(time_index)):
+                        score_time = time_index[i]['time']
+                        score_time_str = f"{str(score_time)[:4]}-{str(score_time)[4:6]}-{str(score_time)[6:]}"
+                        score = time_index[i]['score']
+                        wechat_score_dict = {"score": score, "scoreDate": score_time_str}
+                        wechat_score_list.append(wechat_score_dict)
+                    return word_wechat_score_dict
+        except Exception as e:
+            Common.logger(log_type, crawler).error(f"get_word_score异常:{e}\n")
+
+    # 更新微信指数
+    @classmethod
+    def update_today_wechat_score(cls, log_type, crawler, wechat_score_data, env):
+        """
+        更新热词微信指数
+        :param log_type: 日志名
+        :param crawler: 哪款爬虫
+        :param wechat_score_data: 热词微信指数
+        :param env: 环境
+        :return: {"code":200, "message":"success"}
+        """
+        try:
+            if env == "dev":
+                url = 'http://testhot-words-internal.piaoquantv.com/hot/word/updateWechatScore'
+            else:
+                url = 'http://hot-words-internal.piaoquantv.com/hot/word/updateWechatScore'
+            params = {'data': wechat_score_data}
+            response = requests.post(url=url, json=params)
+            if response.status_code != 200:
+                Common.logger(log_type, crawler).warning(f"update_wechat_score_response:{response.status_code}, {response.text}\n")
+            elif response.json()["message"] != "success":
+                Common.logger(log_type, crawler).warning(f"update_wechat_score_response:{response.status_code}, {response.json()}\n")
+            else:
+                Common.logger(log_type, crawler).info(f"更新热词微信指数:{response.json()['message']}\n")
+        except Exception as e:
+            Common.logger(log_type, crawler).error(f"update_today_wechat_score:{e}\n")
+
+    # 获取微信指数
+    @classmethod
+    def get_today_wechat_score(cls, log_type, crawler, env):
+        """
+        获取微信指数
+        :param log_type: 日志名
+        :param crawler: 哪款爬虫
+        :param env: 环境
+        :return: 热词指数列表
+        """
+        score_num = 0
+        while True:
+            word_list = cls.get_today_words(log_type, crawler, env)
+            if len(word_list) == 0:
+                Common.logger(log_type, crawler).info(f"今日热词更新完毕")
+                cls.pageNum = 1
+                Common.logger(log_type, crawler).info(f"score_num: {score_num}")
+                return
+            else:
+                wechat_score_data = []
+                Common.logger(log_type, crawler).info(f"len(word_list):{len(word_list)}")
+                for i in range(len(word_list)):
+                    word_id = word_list[i]['id']
+                    word = word_list[i]['word']
+                    Common.logger(log_type, crawler).info(f"word_id:{word_id}")
+                    Common.logger(log_type, crawler).info(f"word:{word}")
+                    word_score_dict = cls.get_word_score(log_type, crawler, word_id, word)
+                    Common.logger(log_type, crawler).info(f"word_score_dict:{word_score_dict}\n")
+                    wechat_score_data.append(word_score_dict)
+                    if word_score_dict['wechatScores'] is not None:
+                        score_num += len(word_score_dict['wechatScores'])
+                Common.logger(log_type, crawler).info(f"wechat_score_data:{wechat_score_data}\n")
+                cls.update_today_wechat_score(log_type, crawler, wechat_score_data, env)
+
+
+if __name__ == "__main__":
+    print(TodayScore.get_today_words("today-score", "weixinzhishu", "dev"))
+    # print(TodayScore.get_word_score("today-score", "weixinzhishu", 1, "加班"))
+
+    pass