#!/bin/bash ############################################### # # 通过 run_scheduling_task_v3.py 调起当前 shell # 根据 crawler 及 task 的传递值,启动各个爬虫任务 # ############################################### crawler_dir=$1 # 爬虫执行路径,如: ./youtube/youtube_main/run_youtube_follow.py log_type=$2 # 日志命名格式,如: follow,则在 youtube/logs/目录下,生成 2023-02-08-follow.log crawler=$3 # 哪款爬虫,如: youtube / kanyikan / weixinzhishu task=$4 # 爬虫任务 oss_endpoint=$5 # OSS网关,内网: inner / 外网: out / 香港: hk env=$6 # 爬虫运行环境,正式环境: prod / 测试环境: dev nohup_dir=$7 # nohup日志存储路径,如: ./youtube/nohup.log echo "$(date "+%Y-%m-%d %H:%M:%S") 开始调度爬虫任务" echo "crawler_dir:"${crawler_dir} echo "log_type:"${log_type} echo "crawler:"${crawler} echo "task:"${task} echo "oss_endpoint:"${oss_endpoint} echo "env:"${env} echo "nohup_dir:"${nohup_dir} if [ ${env} = "--env=hk" ];then piaoquan_crawler_dir=/root/piaoquan_crawler/ profile_path=/etc/profile python=python3 elif [ ${env} = "--env=prod" ];then piaoquan_crawler_dir=/data5/piaoquan_crawler/ profile_path=/etc/profile python=python elif [ ${env} = "--env=dev" ];then piaoquan_crawler_dir=/Users/wangkun/Desktop/crawler/piaoquan_crawler/ profile_path=/etc/profile node_path=/opt/homebrew/bin/node python=python3 else echo ${env} fi echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量..." cd ~ && source ${profile_path} echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量完成!" if [ ${env} = "--env=hk" ];then echo "升级yt-dlp" pip3 install yt-dlp -U else echo "$(date "+%Y-%m-%d %H:%M:%S") 正在更新代码..." # cd ${piaoquan_crawler_dir} && git pull origin master --force && rm -f ${piaoquan_crawler_dir}main/nohup.log && rm -f ${piaoquan_crawler_dir}${nohup_dir} echo "$(date "+%Y-%m-%d %H:%M:%S") 代码更新完成!" fi echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启服务..." cd ${piaoquan_crawler_dir} nohup ${python} -u ${crawler_dir} ${log_type} ${crawler} ${task} ${oss_endpoint} ${env} >> ${nohup_dir} 2>&1 & echo "$(date "+%Y-%m-%d %H:%M:%S") 服务重启完毕!" exit 0