update

README.MD

@@ -104,9 +104,9 @@ ps aux | grep run_kuaishou | grep -v grep | awk '{print $2}' | xargs kill -9
 #### 小年糕
 ```commandline
 阿里云 102 服务器
-定向爬虫策略: ps aux | grep run_xiaoniangao_follow | grep -v grep | awk '{print $2}' | xargs kill -9 && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_follow.py --log_type="follow" --crawler="xiaoniangao" --env="prod"  xiaoniangao/nohup-follow.log
-小时榜爬虫策略: ps aux | grep run_xiaoniangao_hour | grep -v grep | awk '{print $2}' | xargs kill -9 && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour.py --log_type="hour" --crawler="xiaoniangao" --env="prod" xiaoniangao/nohup-hour.log
-播放量榜爬虫策略: ps aux | grep run_xiaoniangao_play | grep -v grep | awk '{print $2}' | xargs kill -9 && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_play.py --log_type="play" --crawler="xiaoniangao" --env="prod" xiaoniangao/nohup-play.log
+定向爬虫策略: /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_follow.py --log_type="follow" --crawler="xiaoniangao" --env="prod"  xiaoniangao/nohup-follow.log
+小时榜爬虫策略: /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour.py --log_type="hour" --crawler="xiaoniangao" --env="prod" xiaoniangao/nohup-hour.log
+播放量榜爬虫策略: /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_play.py --log_type="play" --crawler="xiaoniangao" --env="prod" xiaoniangao/nohup-play.log
 
 线下调试
 定向爬虫策略: sh main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_follow.py --log_type="follow" --crawler="xiaoniangao" --env="dev" xiaoniangao/nohup-follow.log
@@ -136,4 +136,11 @@ ps aux | grep run_xiaoniangao_play | grep -v grep | awk '{print $2}' | xargs kil
 杀进程命令
 ps aux | grep run_gongzhonghao
 ps aux | grep run_gongzhonghao | grep -v grep | awk '{print $2}' | xargs kill -9 
+```
+
+
+#### 爬虫进程监测
+```commandline
+阿里云 102 服务器：/usr/bin/sh /data5/piaoquan_crawler/main/process.sh "prod"
+线下调试：cd /Users/wangkun/Desktop/crawler/piaoquan_crawler/ && sh main/process.sh "dev"
 ```

main/main_logs/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/3/29

main/process.sh

@@ -0,0 +1,93 @@
+#! /bin/bash
+env=$1  # 环境
+if [ ${env} = "dev" ];then
+  piaoquan_crawler_dir=/Users/wangkun/Desktop/crawler/piaoquan_crawler/
+  profile_path=/etc/profile
+  python=python3
+  log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
+else
+  piaoquan_crawler_dir=/data5/piaoquan_crawler/
+  profile_path=/etc/profile
+  python=python
+  log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
+fi
+
+echo "$(date "+%Y-%m-%d %H:%M:%S") 开始监测爬虫进程状态\n" >> ${log_path}
+
+echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量..." >> ${log_path}
+cd ~ && source /etc/profile
+echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量完成！\n" >> ${log_path}
+
+# 公众号爬虫策略
+echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 公众号爬虫策略 进程状态" >> ${log_path}
+ps -ef | grep "run_gongzhonghao_follow.py" | grep -v "grep"
+if [ "$?" -eq 1 ];then
+  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
+  if [ ${env} = "dev" ];then
+    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow.py --log_type="follow" --crawler="gongzhonghao" --env="dev" gongzhonghao/nohup-follow.log
+  else
+    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow.py --log_type="follow" --crawler="gongzhonghao" --env="prod"  gongzhonghao/nohup-follow.log
+  fi
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!\n" >> ${log_path}
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 公众号爬虫策略 进程状态正常\n" >> ${log_path}
+fi
+
+# 小年糕定向爬虫策略
+echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 小年糕定向爬虫策略 进程状态" >> ${log_path}
+ps -ef | grep "run_xiaoniangao_follow.py" | grep -v "grep"
+if [ "$?" -eq 1 ];then
+  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
+  if [ ${env} = "dev" ];then
+    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_follow.py --log_type="follow" --crawler="xiaoniangao" --env="dev" xiaoniangao/nohup-follow.log
+  else
+    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_follow.py --log_type="follow" --crawler="xiaoniangao" --env="prod"  xiaoniangao/nohup-follow.log
+  fi
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!\n" >> ${log_path}
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕定向爬虫策略 进程状态正常\n" >> ${log_path}
+fi
+
+# 小年糕小时榜爬虫策略
+echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 小年糕小时榜爬虫策略 进程状态" >> ${log_path}
+ps -ef | grep "run_xiaoniangao_hour.py" | grep -v "grep"
+if [ "$?" -eq 1 ];then
+  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
+  if [ ${env} = "dev" ];then
+    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour.py --log_type="hour" --crawler="xiaoniangao" --env="dev" xiaoniangao/nohup-hour.log
+  else
+    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour.py --log_type="hour" --crawler="xiaoniangao" --env="prod" xiaoniangao/nohup-hour.log
+  fi
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!\n" >> ${log_path}
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕小时榜爬虫策略 进程状态正常\n" >> ${log_path}
+fi
+
+# 播放量榜爬虫策略
+echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 播放量榜爬虫策略 进程状态" >> ${log_path}
+ps -ef | grep "run_xiaoniangao_play.py" | grep -v "grep"
+if [ "$?" -eq 1 ];then
+  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
+  if [ ${env} = "dev" ];then
+    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_play.py --log_type="play" --crawler="xiaoniangao" --env="dev" xiaoniangao/nohup-play.log
+  else
+    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_play.py --log_type="play" --crawler="xiaoniangao" --env="prod" xiaoniangao/nohup-play.log
+  fi
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!\n" >> ${log_path}
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 播放量榜爬虫策略 进程状态正常\n" >> ${log_path}
+fi
+
+# 删除日志
+echo "$(date "+%Y-%m-%d %H:%M:%S") 开始清理 5 天前的日志文件" >> ${log_path}
+find ${piaoquan_crawler_dir}main/main_logs/ -mtime +5 -name "*.log" -exec rm -rf {} \;
+echo "$(date "+%Y-%m-%d %H:%M:%S") 日志文件清理完毕\n" >> ${log_path}
+exit 0
+
+# sh main/process.sh "dev"
+# ps aux | grep run_xiaoniangao
+# ps aux | grep run_xiaoniangao | grep -v grep | awk '{print $2}' | xargs kill -9
+
+# ps aux | grep run_gongzhonghao
+# ps aux | grep run_gongzhonghao | grep -v grep | awk '{print $2}' | xargs kill -9
+

main/scheduling_main.sh

@@ -8,7 +8,7 @@ env=$4          # 爬虫运行环境，正式环境: prod / 测试环境: dev
 nohup_dir=$5    # nohup日志存储路径，如: ./youtube/nohup.log
 
 echo "开始"
-echo ${env}
+#echo ${env}
 if [ ${env} = "--env=hk" ];then
   piaoquan_crawler_dir=/root/piaoquan_crawler/
   profile_path=/etc/profile
@@ -28,10 +28,10 @@ echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量..."
 cd ~ && source ${profile_path}
 echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量完成！"
 
-echo "$(date "+%Y-%m-%d %H:%M:%S") 正在杀进程..."
-grep_str=run_${crawler##*=}
-ps aux | grep ${grep_str} | grep Python | grep -v grep | awk '{print $2}' | xargs kill -9
-echo "$(date "+%Y-%m-%d %H:%M:%S") 进程已杀死！"
+#echo "$(date "+%Y-%m-%d %H:%M:%S") 正在杀进程..."
+#grep_str=run_${crawler##*=}
+#ps aux | grep ${grep_str} | grep Python | grep -v grep | awk '{print $2}' | xargs kill -9
+#echo "$(date "+%Y-%m-%d %H:%M:%S") 进程已杀死！"
 
 if [ ${env} = "--env=hk" ];then
   echo "升级yt-dlp"
@@ -42,21 +42,21 @@ else
   echo "$(date "+%Y-%m-%d %H:%M:%S") 代码更新完成！"
 fi
 
-if [ ${env} = "--env=hk" ];then
-  echo "无需重启Appium及adb服务"
-elif [ ${env} = "--env=prod" ];then
-  echo "无需重启Appium及adb服务"
-else
-  echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启Appium..."
-  ps aux | grep Appium.app | grep -v grep | awk '{print $2}' | xargs kill -9
-  nohup ${node_path} /Applications/Appium.app/Contents/Resources/app/node_modules/appium/build/lib/main.js >>./nohup.log 2>&1 &
-  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启Appium完毕!"
-
-  echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启adb..."
-  adb kill-server
-  adb start-server
-  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启adb完毕!"
-fi
+#if [ ${env} = "--env=hk" ];then
+#  echo "无需重启Appium及adb服务"
+#elif [ ${env} = "--env=prod" ];then
+#  echo "无需重启Appium及adb服务"
+#else
+#  echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启Appium..."
+#  ps aux | grep Appium.app | grep -v grep | awk '{print $2}' | xargs kill -9
+#  nohup ${node_path} /Applications/Appium.app/Contents/Resources/app/node_modules/appium/build/lib/main.js >>./nohup.log 2>&1 &
+#  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启Appium完毕!"
+#
+#  echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启adb..."
+#  adb kill-server
+#  adb start-server
+#  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启adb完毕!"
+#fi
 
 echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启服务..."
 cd ${piaoquan_crawler_dir}