update

main/process.sh

@@ -68,20 +68,20 @@ else
   echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕定向爬虫策略 进程状态正常" >> ${log_path}
 fi
 
-# 小年糕小时榜爬虫策略
-echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 小年糕小时榜爬虫策略 进程状态" >> ${log_path}
-ps -ef | grep "run_xiaoniangao_hour.py" | grep -v "grep"
-if [ "$?" -eq 1 ];then
-  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
-  if [ ${env} = "dev" ];then
-    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour.py --log_type="hour" --crawler="xiaoniangao" --env="dev" xiaoniangao/logs/nohup-hour.log
-  else
-    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour.py --log_type="hour" --crawler="xiaoniangao" --env="prod" xiaoniangao/logs/nohup-hour.log
-  fi
-  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
-else
-  echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕小时榜爬虫策略 进程状态正常" >> ${log_path}
-fi
+## 小年糕小时榜爬虫策略
+#echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 小年糕小时榜爬虫策略 进程状态" >> ${log_path}
+#ps -ef | grep "run_xiaoniangao_hour.py" | grep -v "grep"
+#if [ "$?" -eq 1 ];then
+#  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
+#  if [ ${env} = "dev" ];then
+#    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour.py --log_type="hour" --crawler="xiaoniangao" --env="dev" xiaoniangao/logs/nohup-hour.log
+#  else
+#    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour.py --log_type="hour" --crawler="xiaoniangao" --env="prod" xiaoniangao/logs/nohup-hour.log
+#  fi
+#  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
+#else
+#  echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕小时榜爬虫策略 进程状态正常" >> ${log_path}
+#fi
 
 ## 小年糕播放量榜爬虫策略
 #echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 播放量榜爬虫策略 进程状态" >> ${log_path}

xiaoniangao/xiaoniangao_author/xiaoniangao_author_scheduling.py

@@ -278,19 +278,17 @@ class XiaoniangaoAuthorScheduling:
     @classmethod
     def get_author_videos(cls, log_type, crawler, user_list, rule_dict, env):
         for user_dict in user_list:
-            # try:
-            Common.logger(log_type, crawler).info(f"获取 {user_dict['nick_name']} 主页视频")
-            cls.get_videoList(log_type=log_type,
-                              crawler=crawler,
-                              rule_dict=rule_dict,
-                              user_dict=user_dict,
-                              env=env)
-            # except Exception as e:
-            #     Common.logger(log_type, crawler).error(f"抓取{user_dict['nick_name']}主页时异常:{e}\n")
+            try:
+                Common.logger(log_type, crawler).info(f"获取 {user_dict['nick_name']} 主页视频")
+                cls.get_videoList(log_type=log_type,
+                                  crawler=crawler,
+                                  rule_dict=rule_dict,
+                                  user_dict=user_dict,
+                                  env=env)
+            except Exception as e:
+                Common.logger(log_type, crawler).error(f"抓取{user_dict['nick_name']}主页时异常:{e}\n")
 
 
 if __name__ == "__main__":
-    # print(XiaoniangaoAuthorScheduling.repeat_video("follow", "xiaoniangao", "4919087666", "prod", "aliyun"))
-    # print(XiaoniangaoAuthorScheduling.repeat_video("follow", "xiaoniangao", "4919087666", "dev"))
-    # XiaoniangaoAuthorScheduling.get_users()
+
     pass

xiaoniangao/xiaoniangao_hour/xiaoniangao_hour_scheduling.py

@@ -9,7 +9,6 @@ import shutil
 import sys
 import time
 from hashlib import md5
-
 import requests
 import urllib3
 sys.path.append(os.getcwd())
@@ -18,7 +17,6 @@ from common.feishu import Feishu
 from common.publish import Publish
 from common.scheduling_db import MysqlHelper
 from common.public import get_config_from_mysql, download_rule
-
 proxies = {"http": None, "https": None}
 
 
@@ -576,8 +574,5 @@ class XiaoniangaoHourScheduling:
 
 if __name__ == "__main__":
     print(get_config_from_mysql(log_type='hour', source='xiaoniangao', env='dev', text='filter'))
-    # print(XiaoniangaoHour.get_uid_token())
-    # XiaoniangaoHour.get_videoList("test", "xiaoniangao", "dev")
-    # XiaoniangaoHour.update_videoList("test", "xiaoniangao", "小时榜爬虫策略", "out", "dev")
 
     pass