process_offline1.sh 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197
  1. #! /bin/bash
  2. # **********线下爬虫********** #
  3. env=$1 # 爬虫运行环境,正式环境: prod / 测试环境: dev
  4. #echo ${env}
  5. if [ ${env} = "dev" ];then
  6. piaoquan_crawler_dir=/Users/wangkun/Desktop/crawler/piaoquan_crawler/
  7. profile_path=/etc/profile
  8. node_path=/opt/homebrew/bin/node
  9. log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
  10. else
  11. piaoquan_crawler_dir=/Users/piaoquan/Desktop/piaoquan_crawler/
  12. profile_path=./base_profile
  13. node_path=/usr/local/bin/node
  14. log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
  15. fi
  16. time=$(date +%H:%M:%S)
  17. echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量..." >> ${log_path}
  18. cd ~ && source ${profile_path}
  19. echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量完成!" >> ${log_path}
  20. echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启Appium..." >> ${log_path}
  21. ps -ef | grep "/Applications/Appium.app/Contents/Resources/app/node_modules/appium/build/lib/main.js" | grep -v "grep"
  22. if [ "$?" -eq 1 ];then
  23. echo "$(date "+%Y-%m-%d %H:%M:%S") Appium异常停止,正在重启!" >> ${log_path}
  24. nohup ${node_path} /Applications/Appium.app/Contents/Resources/app/node_modules/appium/build/lib/main.js >>./nohup.log 2>&1 &
  25. echo "$(date "+%Y-%m-%d %H:%M:%S") 重启Appium完毕!" >> ${log_path}
  26. else
  27. echo "$(date "+%Y-%m-%d %H:%M:%S") Appium 进程状态正常" >> ${log_path}
  28. fi
  29. # 吉祥幸福
  30. if [[ "$time" > "00:00:59" && "$time" < "01:59:59" ]];then
  31. echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 吉祥幸福 爬虫脚本任务" >> ${log_path}
  32. ps aux | grep run_ppqsift | grep -v grep | awk '{print $2}' | xargs kill -9
  33. ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
  34. ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
  35. ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
  36. ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
  37. ps -ef | grep "run_jixiangxingfu_recommend.py" | grep -v "grep"
  38. if [ "$?" -eq 1 ];then
  39. echo "$(date "+%Y-%m-%d %H:%M:%S") 吉祥幸福爬虫, 异常停止, 正在重启!" >> ${log_path}
  40. adb forward --remove-all
  41. cd ${piaoquan_crawler_dir}
  42. nohup python3 -u jixiangxingfu/jixiangxingfu_main/run_jixiangxingfu_recommend.py --log_type="recommend" --crawler="jixiangxingfu" --env=${env} >>jixiangxingfu/logs/nohup-recommend.log 2>&1 &
  43. echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
  44. else
  45. echo "$(date "+%Y-%m-%d %H:%M:%S") 吉祥幸福 进程状态正常" >> ${log_path}
  46. fi
  47. else
  48. echo "$(date "+%Y-%m-%d %H:%M:%S") 吉祥幸福 爬虫脚本任务结束" >> ${log_path}
  49. fi
  50. # 众妙音信-new
  51. if [[ "$time" > "02:00:00" && "$time" < "03:59:59" ]];then
  52. echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 众妙音信-new 爬虫脚本任务" >> ${log_path}
  53. ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
  54. ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
  55. ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
  56. ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
  57. ps -ef | grep "run_zmyx_recommend.py" | grep -v "grep"
  58. if [ "$?" -eq 1 ];then
  59. echo "$(date "+%Y-%m-%d %H:%M:%S") 众妙音信-new小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
  60. adb forward --remove-all
  61. cd ${piaoquan_crawler_dir}
  62. nohup python3 -u zhongmiaoyinxin/zhongmiaoyinxin_main/run_zmyx_recommend.py --log_type="recommend" --crawler="zhongmiaoyinxin" --env=${env} >>zhongmiaoyinxin/logs/nohup-recommend.log 2>&1 &
  63. echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
  64. else
  65. echo "$(date "+%Y-%m-%d %H:%M:%S") 众妙音信-new小程序爬虫, 进程状态正常" >> ${log_path}
  66. fi
  67. else
  68. echo "$(date "+%Y-%m-%d %H:%M:%S") 众妙音信 爬虫脚本任务结束" >> ${log_path}
  69. fi
  70. # 漂漂圈
  71. if [[ "$time" > "04:00:00" && "$time" < "06:59:59" ]];then
  72. echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 漂漂圈 爬虫脚本任务" >> ${log_path}
  73. ps aux | grep run_ppqsift | grep -v grep | awk '{print $2}' | xargs kill -9
  74. ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
  75. ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
  76. ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
  77. ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
  78. ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
  79. ps -ef | grep "run_ppq_recommend.py" | grep -v "grep"
  80. if [ "$?" -eq 1 ];then
  81. echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
  82. adb forward --remove-all
  83. cd ${piaoquan_crawler_dir}
  84. nohup python3 -u piaopiaoquan/piaopiaoquan_main/run_ppq_recommend.py --log_type="recommend" --crawler="piaopiaoquan" --env=${env} >>piaopiaoquan/logs/nohup-recommend.log 2>&1 &
  85. echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
  86. else
  87. echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈 程序爬虫, 进程状态正常" >> ${log_path}
  88. fi
  89. else
  90. echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈 爬虫脚本任务结束" >> ${log_path}
  91. fi
  92. # 漂漂圈-精选
  93. if [[ "$time" > "07:00:00" && "$time" < "09:59:59" ]];then
  94. echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 漂漂圈精选 爬虫脚本任务" >> ${log_path}
  95. ps aux | grep run_ppq | grep -v grep | awk '{print $2}' | xargs kill -9
  96. ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
  97. ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
  98. ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
  99. ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
  100. ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
  101. ps -ef | grep "run_ppqsift_recommend.py" | grep -v "grep"
  102. if [ "$?" -eq 1 ];then
  103. echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈精选 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
  104. adb forward --remove-all
  105. cd ${piaoquan_crawler_dir}
  106. nohup python3 -u piaopiaoquan/piaopiaoquan_main/run_ppqsift_recommend.py --log_type="recommend" --crawler="piaopiaoquan" --env=${env} >>piaopiaoquan/logs/nohup-recommend.log 2>&1 &
  107. echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
  108. else
  109. echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈精选 程序爬虫, 进程状态正常" >> ${log_path}
  110. fi
  111. else
  112. echo "$(date "+%Y-%m-%d %H:%M:%S") 漂漂圈精选 爬虫脚本任务结束" >> ${log_path}
  113. fi
  114. # 小年糕+
  115. if [[ "$time" > "10:00:00" && "$time" < "14:59:59" || "$time" > "18:00:00" && "$time" < "20:59:59" ]];then
  116. echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 小年糕+ 爬虫脚本任务" >> ${log_path}
  117. ps aux | grep run_ppqsift | grep -v grep | awk '{print $2}' | xargs kill -9
  118. ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
  119. ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
  120. ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
  121. ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
  122. ps aux | grep run_ppq | grep -v grep | awk '{print $2}' | xargs kill -9
  123. ps -ef | grep "run_xngplus_recommend.py" | grep -v "grep"
  124. if [ "$?" -eq 1 ];then
  125. echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕+ 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
  126. adb forward --remove-all
  127. cd ${piaoquan_crawler_dir}
  128. nohup python3.10 -u xiaoniangaoplus/xiaoniangaoplus_main/run_xngplus_recommend.py --log_type="recommend" --crawler="xiaoniangaoplus" --env=${env} >>xiaoniangaoplus/logs/nohup-recommend.log 2>&1 &
  129. echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
  130. else
  131. echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕+ 程序爬虫, 进程状态正常" >> ${log_path}
  132. fi
  133. else
  134. echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕+ 爬虫脚本任务结束" >> ${log_path}
  135. fi
  136. # 祝福圈子
  137. if [[ "$time" > "22:00:00" && "$time" < "23:59:59" || "$time" > "15:00:00" && "$time" < "17:59:59" ]];then
  138. echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 祝福圈子 爬虫脚本任务" >> ${log_path}
  139. ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
  140. ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
  141. ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
  142. ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
  143. ps -ef | grep "run_zfqz_recommend.py" | grep -v "grep"
  144. if [ "$?" -eq 1 ];then
  145. echo "$(date "+%Y-%m-%d %H:%M:%S") 祝福圈子 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
  146. adb forward --remove-all
  147. cd ${piaoquan_crawler_dir}
  148. nohup python3 -u zhufuquanzi/zhufuquanzi_main/run_zfqz_recommend.py --log_type="recommend" --crawler="zhufuquanzi" --env=${env} >>zhufuquanzi/logs/nohup-recommend.log 2>&1 &
  149. echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
  150. else
  151. echo "$(date "+%Y-%m-%d %H:%M:%S") 祝福圈子 程序爬虫, 进程状态正常" >> ${log_path}
  152. fi
  153. else
  154. echo "$(date "+%Y-%m-%d %H:%M:%S") 祝福圈子 爬虫脚本任务结束" >> ${log_path}
  155. fi
  156. # 小年糕-rule
  157. if [[ "$time" > "21:00:00" && "$time" < "21:59:59" ]];then
  158. echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 小年糕-rule 爬虫脚本任务" >> ${log_path}
  159. ps aux | grep run_zmyx | grep -v grep | awk '{print $2}' | xargs kill -9
  160. ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
  161. ps -ef | grep "run_xngrule_recommend.py" | grep -v "grep"
  162. if [ "$?" -eq 1 ];then
  163. echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕-rule 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
  164. adb forward --remove-all
  165. cd ${piaoquan_crawler_dir}
  166. nohup python3 -u xiaoniangaoplus/xiaoniangaoplus_main/run_xngrule_recommend.py --log_type="recommend" --crawler="xiaoniangaoplus" --env=${env} >>xiaoniangaoplus/logs/nohup-recommend.log 2>&1 &
  167. echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
  168. else
  169. echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕-rule 程序爬虫, 进程状态正常" >> ${log_path}
  170. fi
  171. else
  172. echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕-rule 爬虫脚本任务结束" >> ${log_path}
  173. fi
  174. # 删除日志
  175. echo "$(date "+%Y-%m-%d %H:%M:%S") 开始清理 10 天前的日志文件" >> ${log_path}
  176. find ${piaoquan_crawler_dir}main/main_logs/ -mtime +10 -name "*.log" -exec rm -rf {} \;
  177. echo "$(date "+%Y-%m-%d %H:%M:%S") 日志文件清理完毕" >> ${log_path}
  178. exit 0