Merge branch 'master' of https://git.yishihui.com/Server/piaoquan_crawler

README.MD

@@ -144,8 +144,14 @@ ps aux | grep run_gongzhonghao | grep -v grep | awk '{print $2}' | xargs kill -9
 nohup python3 -u /data5/piaoquan_crawler/weixinzhishu/weixinzhishu_main/weixinzhishu_inner_long.py >>/data5/piaoquan_crawler/weixinzhishu/logs/nohup_inner_long.log 2>&1 &
 nohup python3 -u /data5/piaoquan_crawler/weixinzhishu/weixinzhishu_main/weixinzhishu_out.py >>/data5/piaoquan_crawler/weixinzhishu/logs/nohup_out.log 2>&1 &
 nohup python3 -u /data5/piaoquan_crawler/weixinzhishu/weixinzhishu_main/weixinzhishu_inner_sort.py >>/data5/piaoquan_crawler/weixinzhishu/logs/nohup_inner_sort.log 2>&1 &
+线下调试
+抓取今日微信指数
+python3 /Users/wangkun/Desktop/crawler/piaoquan_crawler/weixinzhishu/weixinzhishu_main/run_weixinzhishu_today_score.py
+检测进程
+ps aux | grep WeChat.app
 ps aux | grep run_weixinzhishu
 ps aux | grep weixinzhishu | grep -v grep | awk '{print $2}' | xargs kill -9
+ps aux | grep 微信 | grep -v grep | awk '{print $2}' | xargs kill -9
 获取 wechat_key 设备: Mac Air 
 cd ~ && source ./base_profile && ps aux | grep weixinzhishu | grep -v grep | awk '{print $2}' | xargs kill -9 && cd /Users/piaoquan/Desktop/piaoquan_crawler && nohup python3 -u weixinzhishu/weixinzhishu_key/search_key_mac.py >> weixinzhishu/nohup.log 2>&1 &
 ```

common/feishu.py

@@ -428,7 +428,7 @@ class Feishu:
             urllib3.disable_warnings()
             r = requests.get(url=url, headers=headers, params=data, verify=False, proxies=proxies)
             open_id = r.json()["data"]["mobile_users"][username][0]["open_id"]
-            Common.logger(log_type, crawler).info(f"{username}:{open_id}")
+            # Common.logger(log_type, crawler).info(f"{username}:{open_id}")
             # print(f"{username}:{open_id}")
             return open_id
         except Exception as e:
@@ -667,7 +667,7 @@ class Feishu:
             })
             urllib3.disable_warnings()
             r = requests.post(url, headers=headers, data=data, verify=False, proxies=proxies)
-            Common.logger(log_type, crawler).info(f'触发机器人消息:{r}, {r.json()["StatusMessage"]}')
+            Common.logger(log_type, crawler).info(f'触发机器人消息:{r.status_code}, {text}')
         except Exception as e:
             Common.logger(log_type, crawler).error(f"bot异常:{e}\n")
 

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow.py

@@ -5,12 +5,11 @@ import datetime
 import difflib
 import json
 import os
-import random
 import shutil
 import sys
 import time
 from hashlib import md5
-
+from requests.adapters import HTTPAdapter
 import requests
 import urllib3
 from selenium.webdriver import DesiredCapabilities
@@ -118,25 +117,32 @@ class GongzhonghaoFollow:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
-
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(f"token_1:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 过期啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"token_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n过期啦，请扫码更换token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if r.json()["base_resp"]["err_msg"] == "freq control":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(f"公众号_1:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if "list" not in r.json() or len(r.json()["list"]) == 0:
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(f"公众号_1:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
@@ -243,24 +249,35 @@ class GongzhonghaoFollow:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
-                    Common.logger(log_type, crawler).info(f"get_videoList:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"token_1:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 过期啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"token_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n过期啦，请扫码更换token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if r.json()["base_resp"]["err_msg"] == "freq control":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_1:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler,f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if 'app_msg_list' not in r.json():
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_1:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
@@ -334,8 +351,8 @@ class GongzhonghaoFollow:
                             return
                         cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
 
-                    Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
+                    Common.logger(log_type, crawler).info('休眠 60 秒\n')
-                    time.sleep(random.randint(60*5, 60*10))
+                    time.sleep(60)
         except Exception as e:
             Common.logger(log_type, crawler).error(f"get_videoList异常:{e}\n")
 
@@ -504,8 +521,8 @@ class GongzhonghaoFollow:
                 Common.logger(log_type, crawler).info(f'获取 {user_name} 公众号视频\n')
                 cls.get_videoList(log_type, crawler, user_name, index, oss_endpoint, env)
                 cls.begin = 0
-                Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
+                Common.logger(log_type, crawler).info('休眠 60 秒\n')
-                time.sleep(random.randint(60*5, 60*10))
+                time.sleep(60)
             except Exception as e:
                 Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
 

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow_2.py

@@ -5,7 +5,6 @@ import datetime
 import difflib
 import json
 import os
-import random
 import shutil
 import sys
 import time
@@ -13,6 +12,7 @@ from hashlib import md5
 
 import requests
 import urllib3
+from requests.adapters import HTTPAdapter
 from selenium.webdriver import DesiredCapabilities
 from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.common.by import By
@@ -116,24 +116,35 @@ class GongzhonghaoFollow2:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_2:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 过期啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"token_2:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n过期啦，请扫码更换token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if r.json()["base_resp"]["err_msg"] == "freq control":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_2:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_2:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if "list" not in r.json() or len(r.json()["list"]) == 0:
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_2:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_2:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
@@ -239,17 +250,26 @@ class GongzhonghaoFollow2:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
-                    Common.logger(log_type, crawler).info(f"get_videoList:{r.text}")
+                    Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_2:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 过期啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"token_2:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n过期啦，请扫码更换token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if r.json()["base_resp"]["err_msg"] == "freq control":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_2:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_2:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
@@ -257,6 +277,8 @@ class GongzhonghaoFollow2:
                 if 'app_msg_list' not in r.json():
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_2:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_2:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
@@ -330,8 +352,8 @@ class GongzhonghaoFollow2:
                             return
                         cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
 
-                    Common.logger(log_type, crawler).info('随机休眠 60*3-60*8 秒\n')
+                    Common.logger(log_type, crawler).info('休眠 60 秒\n')
-                    time.sleep(random.randint(60*5, 60*10))
+                    time.sleep(60)
         except Exception as e:
             Common.logger(log_type, crawler).error(f"get_videoList异常:{e}\n")
 
@@ -500,8 +522,8 @@ class GongzhonghaoFollow2:
                 Common.logger(log_type, crawler).info(f'获取 {user_name} 公众号视频\n')
                 cls.get_videoList(log_type, crawler, user_name, index, oss_endpoint, env)
                 cls.begin = 0
-                Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
+                Common.logger(log_type, crawler).info('休眠 60 秒\n')
-                time.sleep(random.randint(60*5, 60*10))
+                time.sleep(60)
             except Exception as e:
                 Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
 

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow_3.py

@@ -5,7 +5,6 @@ import datetime
 import difflib
 import json
 import os
-import random
 import shutil
 import sys
 import time
@@ -13,6 +12,7 @@ from hashlib import md5
 
 import requests
 import urllib3
+from requests.adapters import HTTPAdapter
 from selenium.webdriver import DesiredCapabilities
 from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.common.by import By
@@ -116,24 +116,35 @@ class GongzhonghaoFollow3:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_3:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 过期啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"token_3:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n过期啦，请扫码更换token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if r.json()["base_resp"]["err_msg"] == "freq control":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_3:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_3:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if "list" not in r.json() or len(r.json()["list"]) == 0:
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_3:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_3:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
@@ -239,24 +250,35 @@ class GongzhonghaoFollow3:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
-                    Common.logger(log_type, crawler).info(f"get_videoList:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_3:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 过期啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"token_3:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n过期啦，请扫码更换token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if r.json()["base_resp"]["err_msg"] == "freq control":
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_3:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_3:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if 'app_msg_list' not in r.json():
-                    Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
+                    Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(
+                        f"公众号_3:{token_dict['gzh_name']}, 更换日期:{token_dict['gzh_time']} 频控啦\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_3:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
@@ -330,8 +352,8 @@ class GongzhonghaoFollow3:
                             return
                         cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
 
-                    Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
+                    Common.logger(log_type, crawler).info('休眠 60 秒\n')
-                    time.sleep(random.randint(60*5, 60*10))
+                    time.sleep(60)
         except Exception as e:
             Common.logger(log_type, crawler).error("get_videoList异常:{}\n", e)
 
@@ -500,8 +522,8 @@ class GongzhonghaoFollow3:
                 Common.logger(log_type, crawler).info(f'获取 {user_name} 公众号视频\n')
                 cls.get_videoList(log_type, crawler, user_name, index, oss_endpoint, env)
                 cls.begin = 0
-                Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
+                Common.logger(log_type, crawler).info('休眠 60 秒\n')
-                time.sleep(random.randint(60*5, 60*10))
+                time.sleep(60)
             except Exception as e:
                 Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
 

gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow.py

@@ -4,6 +4,8 @@
 import argparse
 import os
 import sys
+import time
+
 sys.path.append(os.getcwd())
 from common.common import Common
 from gongzhonghao.gongzhonghao_follow.gongzhonghao_follow import GongzhonghaoFollow
@@ -24,7 +26,8 @@ class Main:
                                               env=env)
             Common.del_logs(log_type, crawler)
             GongzhonghaoFollow.begin = 0
-            Common.logger(log_type, crawler).info('公众号抓取一轮完毕\n')
+            Common.logger(log_type, crawler).info('公众号抓取一轮完毕，休眠 8 小时\n')
+            time.sleep(3600*8)
         except Exception as e:
             Common.logger(log_type, crawler).info(f"公众号抓取异常:{e}\n")
 

gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow_2.py

@@ -4,6 +4,8 @@
 import argparse
 import os
 import sys
+import time
+
 sys.path.append(os.getcwd())
 from common.common import Common
 from gongzhonghao.gongzhonghao_follow.gongzhonghao_follow_2 import GongzhonghaoFollow2
@@ -24,7 +26,8 @@ class Main:
                                                env=env)
             Common.del_logs(log_type, crawler)
             GongzhonghaoFollow2.begin = 0
-            Common.logger(log_type, crawler).info('公众号抓取一轮完毕\n')
+            Common.logger(log_type, crawler).info('公众号抓取一轮完毕，休眠 8 小时\n')
+            time.sleep(3600*8)
         except Exception as e:
             Common.logger(log_type, crawler).info(f"公众号抓取异常:{e}\n")
 

gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow_3.py

@@ -4,6 +4,8 @@
 import argparse
 import os
 import sys
+import time
+
 sys.path.append(os.getcwd())
 from common.common import Common
 from gongzhonghao.gongzhonghao_follow.gongzhonghao_follow_3 import GongzhonghaoFollow3
@@ -24,7 +26,8 @@ class Main:
                                                env=env)
             Common.del_logs(log_type, crawler)
             GongzhonghaoFollow3.begin = 0
-            Common.logger(log_type, crawler).info('公众号抓取一轮完毕\n')
+            Common.logger(log_type, crawler).info('公众号抓取一轮完毕，休眠 8 小时\n')
+            time.sleep(3600*8)
         except Exception as e:
             Common.logger(log_type, crawler).info(f"公众号抓取异常:{e}\n")
 

weixinzhishu/weixinzhishu_key/search_key_mac.py

@@ -2,10 +2,18 @@
 # @Author: wangkun
 # @Time: 2023/2/20
 """
-安装 atomac：
+1. 安装 atomac：
-1.翻墙
+    1.1.翻墙
-2.pip3 install git+https://github.com/pyatom/pyatom/
+    1.2.pip3 install git+https://github.com/pyatom/pyatom/
+2. 启动 Charles.exe:
+    2.1 选中 Proxy - Windows Proxy
+    2.2 选中 Tools - Auto Save - Enable Auto Save
+3. 启动 Python 脚本:
+    3.1 cd ./piaoquan_crawler/
+    3.2 python ./weixinzhishu/weixinzhishu_main/search_key_mac.py
+4. 每 10 秒获取最新search_key，写入飞书: https://w42nne6hzg.feishu.cn/sheets/shtcnqhMRUGunIfGnGXMOBYiy4K?sheet=sVL74k
 """
+
 import json
 import os
 import sys
@@ -30,10 +38,30 @@ class SearchKey:
             # 获取当前应用window
             window = automator.windows()[0]
             Common.logger(log_type, crawler).info(f"当前应用window:{window.AXTitle}")
+
+            # # 登录
+            # login_btn = window.findFirstR(AXRole="AXButton", AXDescription="进入微信")
+            # # print(login_btn.getAttributes())
+            # # print(f"AXEnabled:{login_btn.AXEnabled}")
+            # # print(f"AXDescription:{login_btn.AXDescription}")
+            # # print(f"AXParent:{login_btn.AXParent}")
+            # # print(f"AXSize:{login_btn.AXSize}")
+            # # print(f"AXFocused:{login_btn.AXFocused}")
+            # # print(f"AXRole:{login_btn.AXRole}")
+            # # print(f"AXTopLevelUIElement:{login_btn.AXTopLevelUIElement}")
+            # # print(f"AXHelp:{login_btn.AXHelp}")
+            # # print(f"AXPosition:{login_btn.AXPosition}")
+            # # print(f"AXWindow:{login_btn.AXWindow}")
+            # # print(f"AXRoleDescription:{login_btn.AXRoleDescription}")
+            # # print(f"AXIdentifier:{login_btn.AXIdentifier}")
+            # if login_btn:
+            #     Common.logger(log_type, crawler).info("发现登录按钮，点击登录")
+            #     login_btn.Press()
+            #     time.sleep(1)
+
             # 查找聊天按钮
             Common.logger(log_type, crawler).info("查找聊天按钮")
             chat_btn = window.findFirstR(AXHelp="微信", AXRole="AXRadioButton")
-            # print(chat_btn.getAttributes())
             # 点击聊天按钮，展开聊天列表
             Common.logger(log_type, crawler).info("点击聊天按钮")
             chat_btn.Press()
@@ -52,6 +80,7 @@ class SearchKey:
                 chat_help.clickMouseButtonLeft(chat_help_coor)
             time.sleep(1)
 
+            # 查找微信指数小程序消息
             Common.logger(log_type, crawler).info("查找微信指数小程序消息")
             weixinzhishu = window.findFirstR(AXValue="微信指数", AXRole="AXStaticText")
             Common.logger(log_type, crawler).info(f"微信指数小程序:{weixinzhishu}")
@@ -201,9 +230,9 @@ class SearchKey:
 
 
 if __name__ == "__main__":
-    # SearchKey.start_wechat("search", "weixinzhishu")
+    SearchKey.start_wechat("search", "weixinzhishu")
     # SearchKey.del_wechat_key("search", "weixinzhishu")
     # SearchKey.write_wechat_key("search", "weixinzhishu")
     # print(SearchKey.get_wechat_key("search", "weixinzhishu"))
-    SearchKey.main("search", "weixinzhishu")
+    # SearchKey.main("search", "weixinzhishu")
     pass

weixinzhishu/weixinzhishu_main/run_weixinzhishu_today_score.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/4/6
+import os
+import sys
+sys.path.append(os.getcwd())
+from common.common import Common
+from weixinzhishu.weixinzhishu_score.weixinzhishu_today_score import TodayScore
+
+
+def main(log_type, crawler, env):
+    Common.logger(log_type, crawler).info("开始抓取今日微信指数\n")
+    TodayScore.get_today_wechat_score(log_type, crawler, env)
+    Common.del_logs(log_type, crawler)
+    Common.logger(log_type, crawler).info("今日微信指数抓取完毕\n")
+
+
+if __name__ == "__main__":
+    main("today-score", "weixinzhishu", "prod")

weixinzhishu/weixinzhishu_score/weixinzhishu_score.py

@@ -26,11 +26,15 @@ class Weixinzhishu:
         :return: search_key, openid
         """
         try:
-            sheet = Feishu.get_values_batch(log_type, crawler, 'sVL74k')
+            while True:
-            for i in range(len(sheet)):
+                sheet = Feishu.get_values_batch(log_type, crawler, 'sVL74k')
-                search_key = sheet[1][1]
+                if sheet is None:
-                openid = sheet[1][2]
+                    time.sleep(1)
-                return search_key, openid
+                    continue
+                for i in range(len(sheet)):
+                    search_key = sheet[1][1]
+                    openid = sheet[1][2]
+                    return search_key, openid
         except Exception as e:
             Common.logger(log_type, crawler).error(f"wechat_key:{e}\n")
 
@@ -79,10 +83,6 @@ class Weixinzhishu:
         try:
             while True:
                 wechat_key = cls.get_wechat_key(log_type, crawler)
-                if wechat_key is None:
-                    Common.logger(log_type, crawler).info(f"wechat_key:{wechat_key}")
-                    time.sleep(10)
-                    continue
                 search_key = wechat_key[0]
                 openid = wechat_key[-1]
                 start_ymd = (date.today() + timedelta(days=-7)).strftime("%Y%m%d")
@@ -112,7 +112,8 @@ class Weixinzhishu:
                 if response.json()['code'] == -10000:
                     Common.logger(log_type, crawler).warning(f"response:{response.json()['msg']} 休眠 10 秒，重新获取")
                     time.sleep(10)
-                elif response.json()['code'] == -10002:
+                    continue
+                if response.json()['code'] == -10002:
                     Common.logger(log_type, crawler).info(f'{word}:该词暂未收录')
                     return word_wechat_score_dict
                 elif response.json()['code'] != 0:

weixinzhishu/weixinzhishu_score/weixinzhishu_today_score.py

@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/4/6
+import json
+import os
+import sys
+import time
+from datetime import date, timedelta
+import requests
+sys.path.append(os.getcwd())
+from common.common import Common
+from common.feishu import Feishu
+
+
+class TodayScore:
+    # 翻页参数
+    pageNum = 1
+
+    # 获取微信 key / openid
+    @classmethod
+    def get_wechat_key(cls, log_type, crawler):
+        """
+        获取微信 key / openid
+        https://w42nne6hzg.feishu.cn/sheets/shtcnqhMRUGunIfGnGXMOBYiy4K?sheet=sVL74k
+        :param log_type: 日志名
+        :param crawler: 哪款爬虫，填写:weixinzhishu
+        :return: search_key, openid
+        """
+        try:
+            while True:
+                sheet = Feishu.get_values_batch(log_type, crawler, 'sVL74k')
+                if sheet is None:
+                    time.sleep(1)
+                    continue
+                for i in range(len(sheet)):
+                    search_key = sheet[1][1]
+                    openid = sheet[1][2]
+                    return search_key, openid
+        except Exception as e:
+            Common.logger(log_type, crawler).error(f"wechat_key:{e}\n")
+
+    # 获取热词
+    @classmethod
+    def get_today_words(cls, log_type, crawler, env):
+        try:
+            if env == "dev:":
+                url = 'http://testhot-words-internal.piaoquantv.com/hot/word/getTodayWords'
+            else:
+                url = "http://hot-words-internal.piaoquantv.com/hot/word/getTodayWords"
+            params = {
+                'pageNum': cls.pageNum,  # 第几页，默认1，int
+                'pageSize': 100  # 请求条目数，默认为100，int
+            }
+            response = requests.post(url=url, json=params)
+            cls.pageNum += 1
+            if response.status_code != 200:
+                Common.logger(log_type, crawler).warning(f"get_today_words:{response.status_code}, {response.text}\n")
+            elif response.json()['message'] != "success":
+                Common.logger(log_type, crawler).warning(f"get_word_response:{response.status_code}, {response.json()}\n")
+            else:
+                word_list = response.json()['data']['words']
+                return word_list
+        except Exception as e:
+            Common.logger(log_type, crawler).error(f"get_today_words:{e}\n")
+
+    # 获取热词分数
+    @classmethod
+    def get_word_score(cls, log_type, crawler, word_id, word):
+        """
+        获取热词分数
+        :param log_type: 日志名
+        :param crawler: 哪款爬虫，填写:weixinzhishu
+        :param word_id: 热词 ID
+        :param word: 热词
+        :return: 热词 7 天指数，例如：
+        {'id': 1,
+        'word': '消息',
+        'wechatScores': [
+        {'score': 95521022, 'scoreDate': '2023-02-07'},
+        {'score': 97315283, 'scoreDate': '2023-02-08'},
+        {'score': 109845849, 'scoreDate': '2023-02-09'},
+        {'score': 107089560, 'scoreDate': '2023-02-10'},
+        {'score': 102658391, 'scoreDate': '2023-02-11'},
+        {'score': 93843701, 'scoreDate': '2023-02-12'},
+        {'score': 100211894, 'scoreDate': '2023-02-13'}]}
+        """
+        try:
+            while True:
+                wechat_key = cls.get_wechat_key(log_type, crawler)
+                start_ymd = (date.today() + timedelta(days=-7)).strftime("%Y%m%d")
+                end_ymd = (date.today() + timedelta(days=0)).strftime("%Y%m%d")
+                url = "https://search.weixin.qq.com/cgi-bin/wxaweb/wxindex"
+                payload = json.dumps({
+                    "openid": wechat_key[-1],
+                    # "openid": '123',
+                    "search_key": wechat_key[0],
+                    "cgi_name": "GetDefaultIndex",
+                    "start_ymd": start_ymd,
+                    "end_ymd": end_ymd,
+                    "query": word
+                })
+                headers = {
+                    'Host': 'search.weixin.qq.com',
+                    'content-type': 'application/json',
+                    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.32(0x1800202a) NetType/WIFI Language/zh_CN',
+                    'Referer': 'https://servicewechat.com/wxc026e7662ec26a3a/42/page-frame.html'
+                }
+                response = requests.request("POST", url=url, headers=headers, data=payload)
+                wechat_score_list = []
+                word_wechat_score_dict = {
+                    "id": word_id,
+                    "word": word,
+                    "wechatScores": wechat_score_list,
+                }
+                if response.json()['code'] == -10000:
+                    Common.logger(log_type, crawler).warning(f"response:{response.json()['msg']} 休眠 5 秒，重新获取")
+                    time.sleep(5)
+                    continue
+                if response.json()['code'] == -10002:
+                    Common.logger(log_type, crawler).info(f'{word}:该词暂未收录')
+                    return word_wechat_score_dict
+                elif response.json()['code'] != 0:
+                    Common.logger(log_type, crawler).info(f'response:{response.text}\n')
+                    return word_wechat_score_dict
+                else:
+                    time_index = response.json()['content']['resp_list'][0]['indexes'][0]['time_indexes']
+                    for i in range(len(time_index)):
+                        score_time = time_index[i]['time']
+                        score_time_str = f"{str(score_time)[:4]}-{str(score_time)[4:6]}-{str(score_time)[6:]}"
+                        score = time_index[i]['score']
+                        wechat_score_dict = {"score": score, "scoreDate": score_time_str}
+                        wechat_score_list.append(wechat_score_dict)
+                    return word_wechat_score_dict
+        except Exception as e:
+            Common.logger(log_type, crawler).error(f"get_word_score异常:{e}\n")
+
+    # 更新微信指数
+    @classmethod
+    def update_today_wechat_score(cls, log_type, crawler, wechat_score_data, env):
+        """
+        更新热词微信指数
+        :param log_type: 日志名
+        :param crawler: 哪款爬虫
+        :param wechat_score_data: 热词微信指数
+        :param env: 环境
+        :return: {"code":200, "message":"success"}
+        """
+        try:
+            if env == "dev":
+                url = 'http://testhot-words-internal.piaoquantv.com/hot/word/updateWechatScore'
+            else:
+                url = 'http://hot-words-internal.piaoquantv.com/hot/word/updateWechatScore'
+            params = {'data': wechat_score_data}
+            response = requests.post(url=url, json=params)
+            if response.status_code != 200:
+                Common.logger(log_type, crawler).warning(f"update_wechat_score_response:{response.status_code}, {response.text}\n")
+            elif response.json()["message"] != "success":
+                Common.logger(log_type, crawler).warning(f"update_wechat_score_response:{response.status_code}, {response.json()}\n")
+            else:
+                Common.logger(log_type, crawler).info(f"更新热词微信指数:{response.json()['message']}\n")
+        except Exception as e:
+            Common.logger(log_type, crawler).error(f"update_today_wechat_score:{e}\n")
+
+    # 获取微信指数
+    @classmethod
+    def get_today_wechat_score(cls, log_type, crawler, env):
+        """
+        获取微信指数
+        :param log_type: 日志名
+        :param crawler: 哪款爬虫
+        :param env: 环境
+        :return: 热词指数列表
+        """
+        score_num = 0
+        while True:
+            word_list = cls.get_today_words(log_type, crawler, env)
+            if len(word_list) == 0:
+                Common.logger(log_type, crawler).info(f"今日热词更新完毕")
+                cls.pageNum = 1
+                Common.logger(log_type, crawler).info(f"score_num: {score_num}")
+                return
+            else:
+                wechat_score_data = []
+                Common.logger(log_type, crawler).info(f"len(word_list):{len(word_list)}")
+                for i in range(len(word_list)):
+                    word_id = word_list[i]['id']
+                    word = word_list[i]['word']
+                    Common.logger(log_type, crawler).info(f"word_id:{word_id}")
+                    Common.logger(log_type, crawler).info(f"word:{word}")
+                    word_score_dict = cls.get_word_score(log_type, crawler, word_id, word)
+                    Common.logger(log_type, crawler).info(f"word_score_dict:{word_score_dict}\n")
+                    wechat_score_data.append(word_score_dict)
+                    if word_score_dict['wechatScores'] is not None:
+                        score_num += len(word_score_dict['wechatScores'])
+                Common.logger(log_type, crawler).info(f"wechat_score_data:{wechat_score_data}\n")
+                cls.update_today_wechat_score(log_type, crawler, wechat_score_data, env)
+
+
+if __name__ == "__main__":
+    print(TodayScore.get_today_words("today-score", "weixinzhishu", "dev"))
+    # print(TodayScore.get_word_score("today-score", "weixinzhishu", 1, "加班"))
+
+    pass