piaoquan_crawler/main/process_offline.sh

#! /bin/bash
# **********线下爬虫********** #

env=$1  # 环境
if [ ${env} = "dev" ];then
  piaoquan_crawler_dir=/Users/wangkun/Desktop/crawler/piaoquan_crawler/
  profile_path=/etc/profile
  python=python3
  log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
else
  piaoquan_crawler_dir=/Users/piaoquan/Desktop/crawler/piaoquan_crawler/
#  profile_path=/etc/profile
  profile_path=./base_profile
  python=python3
  log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
fi

time=$(date +%H:%M:%S)
echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量..." >> ${log_path}
cd ~ && source ${profile_path}
echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量完成！" >> ${log_path}

## 吉祥幸福
#if [[ "$time" > "00:00:0" ]] && [[ "$time" < "08:59:59" ]]; then
#  echo "开始启动 吉祥幸福 爬虫脚本任务" >> ${log_path}
#  ps aux | grep run_zhongmiaoyinxin | grep -v grep | awk '{print $2}' | xargs kill -9
#  ps aux | grep run_zhiqingtiantiankan | grep -v grep | awk '{print $2}' | xargs kill -9
#  ps aux | grep run_ganggangdouchuan | grep -v grep | awk '{print $2}' | xargs kill -9
#  ps -ef | grep "run_jixiangxingfu_recommend.py" | grep -v "grep"
#  if [ "$?" -eq 1 ];then
#    echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
#    if [ ${env} = "dev" ];then
#      cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./jixiangxingfu/jixiangxingfu_main/run_jixiangxingfu_recommend.py --log_type="recommend" --crawler="jixiangxingfu" --env="dev" jixiangxingfu/logs/nohup-recommend.log
#    else
#      cd ${piaoquan_crawler_dir} && /bin/sh main/scheduling_main.sh ./jixiangxingfu/jixiangxingfu_main/run_jixiangxingfu_recommend.py --log_type="recommend" --crawler="jixiangxingfu" --env="prod" jixiangxingfu/logs/nohup-recommend.log
#    fi
#    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
#  else
#    echo "$(date "+%Y-%m-%d %H:%M:%S") 吉祥幸福 进程状态正常" >> ${log_path}
#  fi
#else
#  echo "吉祥幸福 爬虫脚本任务结束" >> ${log_path}
#fi

## 知青天天看
#if [[ "$time" > "09:00:0" ]] && [[ "$time" < "12:59:59" ]]; then
#  echo "开始启动 知青天天看 爬虫脚本任务" >> ${log_path}
#
#else
#  echo "知青天天看 爬虫脚本任务结束" >> ${log_path}
#fi

# 刚刚都传
if [[ "$time" > "13:00:0" ]] && [[ "$time" < "22:59:59" ]]; then
  echo "开始启动 刚刚都传 爬虫脚本任务" >> ${log_path}
  ps aux | grep run_zhongmiaoyinxin | grep -v grep | awk '{print $2}' | xargs kill -9
  ps aux | grep run_zhiqingtiantiankan | grep -v grep | awk '{print $2}' | xargs kill -9
  ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
  ps -ef | grep "run_ganggangdouchuan_recommend.py" | grep -v "grep"
  if [ "$?" -eq 1 ];then
    echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
    if [ ${env} = "dev" ];then
      cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./ganggangdouchuan/ganggangdouchuan_main/run_ganggangdouchuan_recommend.py --log_type="recommend" --crawler="ganggangdouchuan" --env="dev" ganggangdouchuan/logs/nohup-recommend.log
    else
      cd ${piaoquan_crawler_dir} && /bin/sh main/scheduling_main.sh ./ganggangdouchuan/ganggangdouchuan_main/run_ganggangdouchuan_recommend.py --log_type="recommend" --crawler="ganggangdouchuan" --env="prod" ganggangdouchuan/logs/nohup-recommend.log
    fi
    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
  else
    echo "$(date "+%Y-%m-%d %H:%M:%S") 刚刚都传 进程状态正常" >> ${log_path}
  fi
else
  echo "刚刚都传 爬虫脚本任务结束" >> ${log_path}
fi

## 众妙音信
#if [[ "$time" > "17:00:0" ]] && [[ "$time" < "23:59:59" ]]; then
#  echo "开始启动 众妙音信 爬虫脚本任务" >> ${log_path}
#
#else
#  echo "众妙音信 爬虫脚本任务结束" >> ${log_path}
#fi


# 删除日志
echo "$(date "+%Y-%m-%d %H:%M:%S") 开始清理 5 天前的日志文件" >> ${log_path}
find ${piaoquan_crawler_dir}main/main_logs/ -mtime +5 -name "*.log" -exec rm -rf {} \;
echo "$(date "+%Y-%m-%d %H:%M:%S") 日志文件清理完毕" >> ${log_path}
exit 0