piaoquan_crawler/scheduling/scheduling_v3/scheduling_v3.sh

#!/bin/bash
###############################################
#
# 通过 run_scheduling_task_v3.py 调起当前 shell
# 根据 crawler 及 task 的传递值，启动各个爬虫任务
#
###############################################

crawler_dir=$1  # 爬虫执行路径，如: ./youtube/youtube_main/run_youtube_follow.py
log_type=$2     # 日志命名格式，如: follow，则在 youtube/logs/目录下，生成 2023-02-08-follow.log
crawler=$3      # 哪款爬虫，如: youtube / kanyikan / weixinzhishu
task=$4         # 爬虫任务
oss_endpoint=$5 # OSS网关，内网: inner / 外网: out / 香港: hk
env=$6          # 爬虫运行环境，正式环境: prod / 测试环境: dev
nohup_dir=$7    # nohup日志存储路径，如: ./youtube/nohup.log

echo "$(date "+%Y-%m-%d %H:%M:%S") 开始调度爬虫任务"
echo "crawler_dir:"${crawler_dir}
echo "log_type:"${log_type}
echo "crawler:"${crawler}
echo "task:"${task}
echo "oss_endpoint:"${oss_endpoint}
echo "env:"${env}
echo "nohup_dir:"${nohup_dir}

if [ ${env} = "--env=hk" ];then
  piaoquan_crawler_dir=/root/piaoquan_crawler/
  profile_path=/etc/profile
  python=python3
elif [ ${env} = "--env=prod" ];then
  piaoquan_crawler_dir=/data5/piaoquan_crawler/
  profile_path=/etc/profile
  python=python
elif [ ${env} = "--env=dev" ];then
  piaoquan_crawler_dir=/Users/wangkun/Desktop/crawler/piaoquan_crawler/
  profile_path=/etc/profile
  node_path=/opt/homebrew/bin/node
  python=python3
else
  echo ${env}
fi

echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量..."
cd ~ && source ${profile_path}
echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量完成！"

if [ ${env} = "--env=hk" ];then
  echo "升级yt-dlp"
  pip3 install yt-dlp -U
else
  echo "$(date "+%Y-%m-%d %H:%M:%S") 正在更新代码..."
#  cd ${piaoquan_crawler_dir} && git pull origin master --force && rm -f ${piaoquan_crawler_dir}main/nohup.log && rm -f ${piaoquan_crawler_dir}${nohup_dir}
  echo "$(date "+%Y-%m-%d %H:%M:%S") 代码更新完成！"
fi

echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启服务..."
cd ${piaoquan_crawler_dir}
nohup ${python} -u ${crawler_dir} ${log_type} ${crawler} ${task} ${oss_endpoint} ${env} >> ${nohup_dir} 2>&1 &
echo "$(date "+%Y-%m-%d %H:%M:%S") 服务重启完毕!"

exit 0