piaoquan_crawler/main/process.sh

#! /bin/bash
env=$1  # 环境
if [ ${env} = "dev" ];then
  piaoquan_crawler_dir=/Users/wangkun/Desktop/crawler/piaoquan_crawler/
  profile_path=/etc/profile
  python=python3
  log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
elif [ ${env} = "hk" ];then
  piaoquan_crawler_dir=/root/piaoquan_crawler/
  profile_path=/etc/profile
  python=python3
  log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
else
  piaoquan_crawler_dir=/data5/piaoquan_crawler/
  profile_path=/etc/profile
  python=python
  log_path=${piaoquan_crawler_dir}main/main_logs/process-$(date +%Y-%m-%d).log
fi

time=$(date +%H:%M:%S)
echo "$(date "+%Y-%m-%d %H:%M:%S") 开始监测爬虫进程状态" >> ${log_path}

echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量..." >> ${log_path}
cd ~ && source /etc/profile
echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量完成！" >> ${log_path}

## 公众号爬虫策略
#echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 公众号爬虫策略 1-100个账号 进程状态" >> ${log_path}
#ps -ef | grep "run_gongzhonghao_follow.py" | grep -v "grep"
#if [ "$?" -eq 1 ];then
#  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
#  if [ ${env} = "dev" ];then
#    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow.py --log_type="author" --crawler="gongzhonghao" --env="dev" gongzhonghao/logs/nohup-follow.log
#  else
#    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow.py --log_type="author" --crawler="gongzhonghao" --env="prod"  gongzhonghao/logs/nohup-follow.log
#  fi
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
#else
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 公众号爬虫策略1-100个账号 进程状态正常" >> ${log_path}
#fi
#
#echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 公众号爬虫策略101-145个账号 进程状态" >> ${log_path}
#ps -ef | grep "run_gongzhonghao_follow_2.py" | grep -v "grep"
#if [ "$?" -eq 1 ];then
#  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
#  if [ ${env} = "dev" ];then
#    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow_2.py --log_type="follow-2" --crawler="gongzhonghao" --env="dev" gongzhonghao/logs/nohup-follow-2.log
#  else
#    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./gongzhonghao/gongzhonghao_main/run_gongzhonghao_follow_2.py --log_type="follow-2" --crawler="gongzhonghao" --env="prod"  gongzhonghao/logs/nohup-follow-2.log
#  fi
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
#else
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 公众号爬虫策略101-145个账号 进程状态正常" >> ${log_path}
#fi

# 小年糕定向爬虫策略
echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 小年糕定向爬虫策略 进程状态" >> ${log_path}
ps -ef | grep "run_xiaoniangao_follow.py" | grep -v "grep"
if [ "$?" -eq 1 ];then
  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
  if [ ${env} = "dev" ];then
    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_follow.py --log_type="author" --crawler="xiaoniangao" --env="dev" xiaoniangao/logs/nohup-follow.log
  else
    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_follow.py --log_type="author" --crawler="xiaoniangao" --env="prod"  xiaoniangao/logs/nohup-follow.log
  fi
  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕定向爬虫策略 进程状态正常" >> ${log_path}
fi

# 小年糕小时榜爬虫策略
echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 小年糕小时榜爬虫策略 进程状态" >> ${log_path}
ps -ef | grep "run_xiaoniangao_hour.py" | grep -v "grep"
if [ "$?" -eq 1 ];then
  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
  if [ ${env} = "dev" ];then
    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour.py --log_type="hour" --crawler="xiaoniangao" --env="dev" xiaoniangao/logs/nohup-hour.log
  else
    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour.py --log_type="hour" --crawler="xiaoniangao" --env="prod" xiaoniangao/logs/nohup-hour.log
  fi
  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕小时榜爬虫策略 进程状态正常" >> ${log_path}
fi

# 小年糕播放量榜爬虫策略
echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 播放量榜爬虫策略 进程状态" >> ${log_path}
ps -ef | grep "run_xiaoniangao_play.py" | grep -v "grep"
if [ "$?" -eq 1 ];then
  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
  if [ ${env} = "dev" ];then
    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_play.py --log_type="play" --crawler="xiaoniangao" --env="dev" xiaoniangao/logs/nohup-play.log
  else
    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./xiaoniangao/xiaoniangao_main/run_xiaoniangao_play.py --log_type="play" --crawler="xiaoniangao" --env="prod" xiaoniangao/logs/nohup-play.log
  fi
  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") 播放量榜爬虫策略 进程状态正常" >> ${log_path}
fi


## 快手定向爬虫策略
#echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 快手定向爬虫策略 进程状态" >> ${log_path}
#ps -ef | grep "run_kuaishou_follow.py" | grep -v "grep"
#if [ "$?" -eq 1 ];then
#  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
#  if [ ${env} = "dev" ];then
#    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./kuaishou/kuaishou_main/run_kuaishou_follow.py --log_type="author" --crawler="kuaishou" --env="dev" kuaishou/logs/nohup-follow.log
#  else
#    cd ${piaoquan_crawler_dir} && /usr/bin/sh ./main/main.sh ./kuaishou/kuaishou_main/run_kuaishou_follow.py --log_type="author" --crawler="kuaishou" --strategy="定向爬虫策略" --oss_endpoint="inner" --env="prod" --machine="aliyun" kuaishou/logs/nohup-follow.log
#  fi
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
#else
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 快手定向爬虫策略 进程状态正常" >> ${log_path}
#fi

## 快手推荐爬虫策略
#echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 快手推荐爬虫策略 进程状态" >> ${log_path}
#ps -ef | grep "run_kuaishou_recommend.py" | grep -v "grep"
#if [ "$?" -eq 1 ];then
#  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
#  if [ ${env} = "dev" ];then
#    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./kuaishou/kuaishou_main/run_kuaishou_follow.py --log_type="author" --crawler="kuaishou" --env="dev" kuaishou/logs/nohup-recommend.log
#  else
#    cd ${piaoquan_crawler_dir} && /usr/bin/sh ./main/main.sh ./kuaishou/kuaishou_main/run_kuaishou_recommend.py --log_type="recommend" --crawler="kuaishou" --strategy="推荐爬虫策略" --oss_endpoint="inner" --env="prod" --machine="aliyun" kuaishou/logs/nohup-recommend.log
#  fi
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
#else
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 快手推荐爬虫策略 进程状态正常" >> ${log_path}
#fi

# 抖音推荐爬虫策略
echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 抖音推荐爬虫策略 进程状态" >> ${log_path}
ps -ef | grep "run_douyin_recommend.py" | grep -v "grep"
if [ "$?" -eq 1 ];then
  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
  if [ ${env} = "dev" ];then
    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./kuaishou/kuaishou_main/run_kuaishou_follow.py --log_type="author" --crawler="kuaishou" --env="dev" douyin/logs/nohup-recommend.log
  else
    cd ${piaoquan_crawler_dir} && /usr/bin/sh ./main/main.sh ./douyin/douyin_main/run_douyin_recommend.py --log_type="recommend" --crawler="douyin" --strategy="抖音推荐爬虫策略" --oss_endpoint="inner" --env="prod" --machine="aliyun" douyin/logs/nohup-recommend.log
  fi
  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") 抖音推荐爬虫策略 进程状态正常" >> ${log_path}
fi

# 抖音定向爬虫策略
echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 抖音定向爬虫策略 进程状态" >> ${log_path}
ps -ef | grep "run_douyin_follow.py" | grep -v "grep"
if [ "$?" -eq 1 ];then
  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
  if [ ${env} = "dev" ];then
    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./kuaishou/douyin_main/run_douyin_follow.py --log_type="author" --crawler="douyin" --env="dev" douyin/logs/nohup-follow.log
  else
    cd ${piaoquan_crawler_dir} && /usr/bin/sh ./main/main.sh ./douyin/douyin_main/run_douyin_follow.py --log_type="author" --crawler="douyin" --strategy="抖音定向爬虫策略" --oss_endpoint="inner" --env="prod" --machine="aliyun" douyin/logs/nohup-author.log
  fi
  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") 抖音推荐爬虫策略 进程状态正常" >> ${log_path}
fi

# 西瓜定向爬虫策略
echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 西瓜定向爬虫策略 进程状态" >> ${log_path}
ps -ef | grep "run_xigua_follow.py" | grep -v "grep"
if [ "$?" -eq 1 ];then
  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
  if [ ${env} = "dev" ];then
    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xigua/xigua_main/run_xigua_follow.py --log_type="author" --crawler="xigua" --env="dev" xigua/logs/nohup-follow.log
  else
    cd ${piaoquan_crawler_dir} && /usr/bin/sh ./main/main.sh ./xigua/xigua_main/run_xigua_follow.py --log_type="author" --crawler="xigua" --strategy="定向爬虫策略" --oss_endpoint="inner" --env="prod" --machine="aliyun" xigua/logs/nohup-follow.log
  fi
  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") 西瓜定向爬虫策略 进程状态正常" >> ${log_path}
fi

# 西瓜推荐榜爬虫策略
echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 西瓜推荐榜爬虫策略 进程状态" >> ${log_path}
ps -ef | grep "run_xigua_recommend.py" | grep -v "grep"
if [ "$?" -eq 1 ];then
  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
  if [ ${env} = "dev" ];then
    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xigua/xigua_main/run_xigua_recommend.py --log_type="recommend" --crawler="xigua" --env="dev" xigua/logs/nohup-recommend.log
  else
    cd ${piaoquan_crawler_dir} && /usr/bin/sh main/scheduling_main.sh ./xigua/xigua_main/run_xigua_recommend.py --log_type="recommend" --crawler="xigua" --env="prod" xigua/logs/nohup-recommend.log
  fi
  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") 西瓜推荐榜爬虫策略 进程状态正常" >> ${log_path}
fi

# 西瓜搜索爬虫策略
if [[ "$time" > "00:00:00" ]] && [[ "$time" < "00:10:00" ]]; then
  echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 西瓜搜索爬虫策略 进程状态" >> ${log_path}
  ps -ef | grep "run_xigua_search_new" | grep -v "grep"
  if [ "$?" -eq 1 ];then
    echo "$(date "+%Y-%m-%d %H:%M:%S") 西瓜搜索爬虫策略, 异常停止, 正在重启!" >> ${log_path}
    if [ ${env} = "dev" ];then
      cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./xigua/xigua_main/run_xigua_search_new.py --log_type="search" --crawler="xigua" --env="dev" xigua/logs/nohup-search.log
    else
      cd ${piaoquan_crawler_dir} && /usr/bin/sh main/scheduling_main.sh ./xigua/xigua_main/run_xigua_search_new.py --log_type="search" --crawler="xigua" --env="prod" xigua/logs/nohup-search.log
    fi
    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
  else
    echo "$(date "+%Y-%m-%d %H:%M:%S") 西瓜搜索爬虫策略 进程状态正常" >> ${log_path}
  fi
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") 不在任务启动时间范围: 西瓜搜索爬虫" >> ${log_path}
fi

# youtube定向爬虫策略
echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 youtube定向爬虫策略 进程状态" >> ${log_path}
ps -ef | grep "run_youtube_follow.py" | grep -v "grep"
if [ "$?" -eq 1 ];then
  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
  if [ ${env} = "dev" ];then
    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./youtube/youtube_main/run_youtube_follow.py --log_type="author" --crawler="youtube" --env="dev" youtube/logs/nohup-follow.log
  else
    cd ${piaoquan_crawler_dir} && /usr/bin/sh ./main/main.sh ./youtube/youtube_main/run_youtube_follow.py --log_type="author" --crawler="youtube" --strategy="定向爬虫策略" --oss_endpoint="hk" --env="hk" --machine="aliyun_hk" youtube/logs/nohup-follow.log
  fi
  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") youtube定向爬虫策略 进程状态正常" >> ${log_path}
fi

## 本山祝福小程序爬虫
#echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 本山祝福小程序爬虫 进程状态" >> ${log_path}
#ps -ef | grep "run_benshanzhufu" | grep -v "grep"
#if [ "$?" -eq 1 ];then
#  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
#  if [ ${env} = "dev" ];then
#    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./benshanzhufu/benshanzhufu_main/run_benshanzhufu_recommend.py --log_type="recommend" --crawler="benshanzhufu" --env="dev" benshanzhufu/logs/nohup-recommend.log
#  else
#    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./benshanzhufu/benshanzhufu_main/run_benshanzhufu_recommend.py --log_type="recommend" --crawler="benshanzhufu" --env="prod"  benshanzhufu/logs/nohup-recommend.log
#  fi
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
#else
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 本山祝福小程序爬虫 进程状态正常" >> ${log_path}
#fi

## 岁岁年年迎福气小程序爬虫
#echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 岁岁年年迎福气小程序爬虫 进程状态" >> ${log_path}
#ps -ef | grep "run_suisuiniannianyingfuqi" | grep -v "grep"
#if [ "$?" -eq 1 ];then
#  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
#  if [ ${env} = "dev" ];then
#    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./suisuiniannianyingfuqi/suisuiniannianyingfuqi_main/run_suisuiniannianyingfuqi_recommend.py --log_type="recommend" --crawler="suisuiniannianyingfuqi" --env="dev" suisuiniannianyingfuqi/logs/nohup-recommend.log
#  else
#    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./suisuiniannianyingfuqi/suisuiniannianyingfuqi_main/run_suisuiniannianyingfuqi_recommend.py --log_type="recommend" --crawler="suisuiniannianyingfuqi" --env="prod"  suisuiniannianyingfuqi/logs/nohup-recommend.log
#  fi
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
#else
#  echo "$(date "+%Y-%m-%d %H:%M:%S") 岁岁年年迎福气小程序爬虫 进程状态正常" >> ${log_path}
#fi

# 微信指数监控
echo "$(date "+%Y-%m-%d %H:%M:%S") 正在监测 微信指数 bot 爬虫 进程状态" >> ${log_path}
ps -ef | grep "run_weixinzhishu_bot" | grep -v "grep"
if [ "$?" -eq 1 ];then
  echo "$(date "+%Y-%m-%d_%H:%M:%S") 异常停止,正在重启!" >> ${log_path}
  if [ ${env} = "dev" ];then
    cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./weixinzhishu/weixinzhishu_main/run_weixinzhishu_bot.py --log_type="bot" --crawler="weixinzhishu" --env="dev" weixinzhishu/logs/nohup-bot.log
  else
    cd ${piaoquan_crawler_dir} && /usr/bin/sh /data5/piaoquan_crawler/main/scheduling_main.sh ./weixinzhishu/weixinzhishu_main/run_weixinzhishu_bot.py --log_type="bot" --crawler="weixinzhishu" --env="prod"  weixinzhishu/logs/nohup-bot.log
  fi
  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") 微信指数 bot 爬虫 进程状态正常" >> ${log_path}
fi


# 删除日志
echo "$(date "+%Y-%m-%d %H:%M:%S") 开始清理 5 天前的日志文件" >> ${log_path}
find ${piaoquan_crawler_dir}main/main_logs/ -mtime +5 -name "*.log" -exec rm -rf {} \;
echo "$(date "+%Y-%m-%d %H:%M:%S") 日志文件清理完毕" >> ${log_path}
exit 0