小年糕线下 代码优化

main/process_offline1.sh

@@ -0,0 +1,197 @@
+#! /bin/bash
+# **********线下爬虫********** #
+env=$1          # 爬虫运行环境，正式环境: prod / 测试环境: dev
+#echo ${env}
+if [ ${env} = "dev" ];then
+  piaoquan_crawler_dir=/Users/wangkun/Desktop/crawler/piaoquan_crawler/
+  profile_path=/etc/profile
+  node_path=/opt/homebrew/bin/node
+  log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
+else
+  piaoquan_crawler_dir=/Users/piaoquan/Desktop/piaoquan_crawler/
+  profile_path=./base_profile
+  node_path=/usr/local/bin/node
+  log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
+fi
+
+time=$(date +%H:%M:%S)
+echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量..." >> ${log_path}
+cd ~ && source ${profile_path}
+echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量完成！" >> ${log_path}
+
+echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启Appium..." >> ${log_path}
+ps -ef | grep "/Applications/Appium.app/Contents/Resources/app/node_modules/appium/build/lib/main.js" | grep -v "grep"
+if [ "$?" -eq 1 ];then
+  echo "$(date "+%Y-%m-%d %H:%M:%S") Appium异常停止,正在重启!" >> ${log_path}
+  nohup ${node_path} /Applications/Appium.app/Contents/Resources/app/node_modules/appium/build/lib/main.js >>./nohup.log 2>&1 &
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启Appium完毕!" >> ${log_path}
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") Appium 进程状态正常" >> ${log_path}
+fi
+
+# 吉祥幸福
+if [[ "$time" > "00:00:59"  &&  "$time" < "01:59:59" ]];then
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 吉祥幸福 爬虫脚本任务" >> ${log_path}
+  ps aux | grep run_ppqsift | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps -ef | grep "run_jixiangxingfu_recommend.py" | grep -v "grep"
+  if [ "$?" -eq 1 ];then
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 吉祥幸福爬虫, 异常停止, 正在重启!" >> ${log_path}
+    adb forward --remove-all
+    cd ${piaoquan_crawler_dir}
+    nohup python3 -u jixiangxingfu/jixiangxingfu_main/run_jixiangxingfu_recommend.py --log_type="recommend" --crawler="jixiangxingfu" --env=${env} >>jixiangxingfu/logs/nohup-recommend.log 2>&1 &
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
+  else
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 吉祥幸福 进程状态正常" >> ${log_path}
+  fi
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 吉祥幸福 爬虫脚本任务结束" >> ${log_path}
+fi
+
+
+# 众妙音信-new
+if [[ "$time" > "02:00:00"  &&  "$time" < "03:59:59" ]];then
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 众妙音信-new 爬虫脚本任务" >> ${log_path}
+  ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps -ef | grep "run_zmyx_recommend.py" | grep -v "grep"
+  if [ "$?" -eq 1 ];then
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 众妙音信-new小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
+    adb forward --remove-all
+    cd ${piaoquan_crawler_dir}
+    nohup python3 -u zhongmiaoyinxin/zhongmiaoyinxin_main/run_zmyx_recommend.py --log_type="recommend" --crawler="zhongmiaoyinxin" --env=${env} >>zhongmiaoyinxin/logs/nohup-recommend.log 2>&1 &
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
+  else
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 众妙音信-new小程序爬虫, 进程状态正常" >> ${log_path}
+  fi
+
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 众妙音信 爬虫脚本任务结束" >> ${log_path}
+fi
+
+# 漂漂圈
+if [[ "$time" > "04:00:00"  &&  "$time" < "06:59:59" ]];then
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 漂漂圈 爬虫脚本任务" >> ${log_path}
+  ps aux | grep run_ppqsift | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps -ef | grep "run_ppq_recommend.py" | grep -v "grep"
+  if [ "$?" -eq 1 ];then
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
+    adb forward --remove-all
+    cd ${piaoquan_crawler_dir}
+    nohup python3 -u piaopiaoquan/piaopiaoquan_main/run_ppq_recommend.py --log_type="recommend" --crawler="piaopiaoquan" --env=${env} >>piaopiaoquan/logs/nohup-recommend.log 2>&1 &
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
+  else
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈 程序爬虫, 进程状态正常" >> ${log_path}
+  fi
+
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈 爬虫脚本任务结束" >> ${log_path}
+fi
+
+# 漂漂圈-精选
+if [[ "$time" > "07:00:00"  &&  "$time" < "09:59:59" ]];then
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 漂漂圈精选 爬虫脚本任务" >> ${log_path}
+  ps aux | grep run_ppq | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps -ef | grep "run_ppqsift_recommend.py" | grep -v "grep"
+  if [ "$?" -eq 1 ];then
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈精选 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
+    adb forward --remove-all
+    cd ${piaoquan_crawler_dir}
+    nohup python3 -u piaopiaoquan/piaopiaoquan_main/run_ppqsift_recommend.py --log_type="recommend" --crawler="piaopiaoquan" --env=${env} >>piaopiaoquan/logs/nohup-recommend.log 2>&1 &
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
+  else
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈精选 程序爬虫, 进程状态正常" >> ${log_path}
+  fi
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈精选 爬虫脚本任务结束" >> ${log_path}
+fi
+
+# 小年糕+
+if [[ "$time" > "10:00:00"  &&  "$time" < "14:59:59" || "$time" > "18:00:00"  &&  "$time" < "20:59:59" ]];then
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 小年糕+ 爬虫脚本任务" >> ${log_path}
+  ps aux | grep run_ppqsift | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_ppq | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps -ef | grep "run_xngplus_recommend.py" | grep -v "grep"
+  if [ "$?" -eq 1 ];then
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕+ 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
+    adb forward --remove-all
+    cd ${piaoquan_crawler_dir}
+    nohup python3.10 -u xiaoniangaoplus/xiaoniangaoplus_main/run_xngplus_recommend.py --log_type="recommend" --crawler="xiaoniangaoplus" --env=${env} >>xiaoniangaoplus/logs/nohup-recommend.log 2>&1 &
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
+  else
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕+ 程序爬虫, 进程状态正常" >> ${log_path}
+  fi
+
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕+ 爬虫脚本任务结束" >> ${log_path}
+fi
+
+# 祝福圈子
+if [[ "$time" > "22:00:00"  &&  "$time" < "23:59:59" || "$time" > "15:00:00"  &&  "$time" < "17:59:59" ]];then
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 祝福圈子 爬虫脚本任务" >> ${log_path}
+  ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps -ef | grep "run_zfqz_recommend.py" | grep -v "grep"
+  if [ "$?" -eq 1 ];then
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 祝福圈子 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
+    adb forward --remove-all
+    cd ${piaoquan_crawler_dir}
+    nohup python3 -u zhufuquanzi/zhufuquanzi_main/run_zfqz_recommend.py --log_type="recommend" --crawler="zhufuquanzi" --env=${env} >>zhufuquanzi/logs/nohup-recommend.log 2>&1 &
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
+  else
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 祝福圈子 程序爬虫, 进程状态正常" >> ${log_path}
+  fi
+
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 祝福圈子 爬虫脚本任务结束" >> ${log_path}
+fi
+
+
+
+# 小年糕-rule
+if [[ "$time" > "21:00:00"  &&  "$time" < "21:59:59" ]];then
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 小年糕-rule 爬虫脚本任务" >> ${log_path}
+  ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps -ef | grep "run_xngrule_recommend.py" | grep -v "grep"
+  if [ "$?" -eq 1 ];then
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕-rule 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
+    adb forward --remove-all
+    cd ${piaoquan_crawler_dir}
+    nohup python3 -u xiaoniangaoplus/xiaoniangaoplus_main/run_xngrule_recommend.py --log_type="recommend" --crawler="xiaoniangaoplus" --env=${env} >>xiaoniangaoplus/logs/nohup-recommend.log 2>&1 &
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
+  else
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕-rule 程序爬虫, 进程状态正常" >> ${log_path}
+  fi
+
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕-rule 爬虫脚本任务结束" >> ${log_path}
+fi
+
+
+# 删除日志
+echo "$(date "+%Y-%m-%d %H:%M:%S") 开始清理 10 天前的日志文件" >> ${log_path}
+find ${piaoquan_crawler_dir}main/main_logs/ -mtime +10 -name "*.log" -exec rm -rf {} \;
+echo "$(date "+%Y-%m-%d %H:%M:%S") 日志文件清理完毕" >> ${log_path}
+exit 0