update

common/common.py

@@ -133,17 +133,21 @@ class Common:
             # 视频名
             video_name = "video.mp4"
 
-            # 下载视频
-            urllib3.disable_warnings()
-            # response = requests.get(video_url, stream=True, proxies=cls.tunnel_proxies(), verify=False)
-            response = requests.get(video_url, stream=True, proxies=proxies, verify=False)
-            try:
-                with open(video_path + video_name, "wb") as f:
-                    for chunk in response.iter_content(chunk_size=10240):
-                        f.write(chunk)
-                cls.logger(log_type, crawler).info("==========视频下载完成==========")
-            except Exception as e:
-                cls.logger(log_type, crawler).error(f"视频下载失败：{e}\n")
+            for i in range(3):
+                try:
+                    # 下载视频，最多重试三次
+                    urllib3.disable_warnings()
+                    # response = requests.get(video_url, stream=True, proxies=cls.tunnel_proxies(), verify=False)
+                    response = requests.get(video_url, stream=True, proxies=proxies, verify=False)
+
+                    with open(video_path + video_name, "wb") as f:
+                        for chunk in response.iter_content(chunk_size=10240):
+                            f.write(chunk)
+                    cls.logger(log_type, crawler).info("==========视频下载完成==========")
+                    break
+                except Exception as e:
+                    cls.logger(log_type, crawler).error(f"视频下载失败：{e}\n")
+                    time.sleep(1)
 
         # 下载音频
         elif text == "audio":

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow.py

@@ -136,7 +136,7 @@ class GongzhonghaoFollow:
                     continue
                 if "list" not in r.json() or len(r.json()["list"]) == 0:
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
-                    Common.logger(log_type, crawler).warning(f"get_gzh_url:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
@@ -253,14 +253,14 @@ class GongzhonghaoFollow:
                     continue
                 if r.json()["base_resp"]["err_msg"] == "freq control":
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
-                    Common.logger(log_type, crawler).warning(f"get_gzh_url:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler,f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if 'app_msg_list' not in r.json():
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
-                    Common.logger(log_type, crawler).warning(f"get_gzh_url:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
@@ -334,8 +334,8 @@ class GongzhonghaoFollow:
                             return
                         cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
 
-                    Common.logger(log_type, crawler).info('随机休眠 60-60*3 秒\n')
-                    time.sleep(random.randint(60, 60*3))
+                    Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
+                    time.sleep(random.randint(60*5, 60*10))
         except Exception as e:
             Common.logger(log_type, crawler).error(f"get_videoList异常:{e}\n")
 
@@ -496,18 +496,18 @@ class GongzhonghaoFollow:
 
     @classmethod
     def get_all_videos(cls, log_type, crawler, oss_endpoint, env):
-        try:
-            user_list = cls.get_users()
-            for user_dict in user_list:
+        user_list = cls.get_users()
+        for user_dict in user_list:
+            try:
                 user_name = user_dict['user_name']
                 index = user_dict['index']
                 Common.logger(log_type, crawler).info(f'获取 {user_name} 公众号视频\n')
                 cls.get_videoList(log_type, crawler, user_name, index, oss_endpoint, env)
                 cls.begin = 0
-                Common.logger(log_type, crawler).info('随机休眠 60-60*3 秒\n')
-                time.sleep(random.randint(60, 60*3))
-        except Exception as e:
-            Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
+                Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
+                time.sleep(random.randint(60*5, 60*10))
+            except Exception as e:
+                Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
 
 
 if __name__ == "__main__":

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow_2.py

@@ -242,21 +242,21 @@ class GongzhonghaoFollow2:
                 r = requests.get(url=url, headers=headers, params=params, verify=False)
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
-                    Common.logger(log_type, crawler).info(f"response:{r.text}")
+                    Common.logger(log_type, crawler).info(f"get_videoList:{r.text}")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"token_2:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n过期啦，请扫码更换token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if r.json()["base_resp"]["err_msg"] == "freq control":
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
-                    Common.logger(log_type, crawler).warning(f"get_gzh_url:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_2:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
                     continue
                 if 'app_msg_list' not in r.json():
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
-                    Common.logger(log_type, crawler).warning(f"get_gzh_url:{r.text}\n")
+                    Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
                     if 20 >= datetime.datetime.now().hour >= 10:
                         Feishu.bot(log_type, crawler, f"公众号_2:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦，请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
                     time.sleep(60 * 10)
@@ -330,8 +330,8 @@ class GongzhonghaoFollow2:
                             return
                         cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
 
-                    Common.logger(log_type, crawler).info('随机休眠 60-60*3 秒\n')
-                    time.sleep(random.randint(60, 60*3))
+                    Common.logger(log_type, crawler).info('随机休眠 60*3-60*8 秒\n')
+                    time.sleep(random.randint(60*5, 60*10))
         except Exception as e:
             Common.logger(log_type, crawler).error(f"get_videoList异常:{e}\n")
 
@@ -492,18 +492,18 @@ class GongzhonghaoFollow2:
 
     @classmethod
     def get_all_videos(cls, log_type, crawler, oss_endpoint, env):
-        try:
-            user_list = cls.get_users()
-            for user_dict in user_list:
+        user_list = cls.get_users()
+        for user_dict in user_list:
+            try:
                 user_name = user_dict['user_name']
                 index = user_dict['index']
                 Common.logger(log_type, crawler).info(f'获取 {user_name} 公众号视频\n')
                 cls.get_videoList(log_type, crawler, user_name, index, oss_endpoint, env)
                 cls.begin = 0
-                Common.logger(log_type, crawler).info('随机休眠 60-60*3 秒\n')
-                time.sleep(random.randint(60, 60*3))
-        except Exception as e:
-            Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
+                Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
+                time.sleep(random.randint(60*5, 60*10))
+            except Exception as e:
+                Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
 
 
 if __name__ == "__main__":

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow_3.py

@@ -330,8 +330,8 @@ class GongzhonghaoFollow3:
                             return
                         cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
 
-                    Common.logger(log_type, crawler).info('随机休眠 60-60*3 秒\n')
-                    time.sleep(random.randint(60, 60*3))
+                    Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
+                    time.sleep(random.randint(60*5, 60*10))
         except Exception as e:
             Common.logger(log_type, crawler).error("get_videoList异常:{}\n", e)
 
@@ -492,18 +492,18 @@ class GongzhonghaoFollow3:
 
     @classmethod
     def get_all_videos(cls, log_type, crawler, oss_endpoint, env):
-        try:
-            user_list = cls.get_users()
-            for user_dict in user_list:
+        user_list = cls.get_users()
+        for user_dict in user_list:
+            try:
                 user_name = user_dict['user_name']
                 index = user_dict['index']
                 Common.logger(log_type, crawler).info(f'获取 {user_name} 公众号视频\n')
                 cls.get_videoList(log_type, crawler, user_name, index, oss_endpoint, env)
                 cls.begin = 0
-                Common.logger(log_type, crawler).info('随机休眠 60-60*3 秒\n')
-                time.sleep(random.randint(60, 60*3))
-        except Exception as e:
-            Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
+                Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
+                time.sleep(random.randint(60*5, 60*10))
+            except Exception as e:
+                Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
 
 
 if __name__ == "__main__":