update scheduling

README.MD

@@ -17,14 +17,19 @@ ${nohup_dir}:       nohup日志存储路径，如: shceduling/nohup-task.log
 阿里云 102 服务器
 sh ./main/scheduling_main.sh scheduling/scheduling_main/run_write_task.py --log_type="scheduling-write" --crawler="scheduling" --env="prod" nohup-write.log 
 sh ./main/scheduling_main.sh scheduling/scheduling_main/run_scheduling_task.py --log_type="scheduling-task" --crawler="scheduling" --env="prod" nohup-task.log 
-
+# 读取任务写入 Redis，1分钟/次
+*/1 * * * * cd /data5/piaoquan_crawler && /usr/bin/sh ./main/scheduling_main.sh scheduling/scheduling_v3/run_write_task_v3.py --log_type="scheduling-write" --crawler="scheduling" --env="prod" scheduling/logs/scheduling-write.log
+# 调度任务，5秒/次
+* * * * * for i in {1..12}; do cd /data5/piaoquan_crawler && /usr/bin/sh ./main/scheduling_main.sh scheduling/scheduling_v3/run_scheduling_task_v3.py --log_type="scheduling-task" --crawler="scheduling" --env="prod" scheduling/logs/scheduling-task.log; sleep 5; done
 香港服务器
 sh ./main/scheduling_main.sh scheduling/scheduling_main/run_write_task.py --log_type="scheduling-write" --crawler="scheduling" --env="prod" shceduling/nohup-write.log 
 sh ./main/scheduling_main.sh scheduling/scheduling_main/run_scheduling_task.py --log_type="scheduling-task" --crawler="scheduling" --env="prod" shceduling/nohup-task.log 
 
 线下调试
-sh ./main/scheduling_main.sh scheduling/scheduling_main/run_write_task.py --log_type="scheduling-write" --crawler="scheduling" --env="dev"  ./scheduling/nohup-write.log 
-sh ./main/scheduling_main.sh scheduling/scheduling_main/run_scheduling_task.py --log_type="scheduling-task" --crawler="scheduling" --env="dev"  ./scheduling/nohup-task.log 
+# 读取任务写入 Redis
+sh ./main/scheduling_main.sh scheduling/scheduling_v3/run_write_task_v3.py --log_type="scheduling-write" --crawler="scheduling" --env="dev"  scheduling/logs/scheduling-write.log 
+# 调度任务
+sh ./main/scheduling_main.sh scheduling/scheduling_v3/run_scheduling_task_v3.py --log_type="scheduling-task" --crawler="scheduling" --env="dev"  scheduling/logs/scheduling-task.log 
 
 杀进程
 ps aux | grep scheduling

scheduling/logs/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/4/19

scheduling/scheduling_main/scheduling.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+crawler_dir=$1  # 爬虫执行路径，如: ./youtube/youtube_main/run_youtube_follow.py
+log_type=$2     # 日志命名格式，如: follow，则在 youtube/logs/目录下，生成 2023-02-08-follow.log
+crawler=$3      # 哪款爬虫，如: youtube / kanyikan / weixinzhishu
+task=$4         # 爬虫任务
+oss_endpoint=$5 # OSS网关，内网: inner / 外网: out / 香港: hk
+env=$6          # 爬虫运行环境，正式环境: prod / 测试环境: dev
+#machine=$7      # 爬虫运行机器，阿里云服务器: aliyun_hk / aliyun / macpro / macair / local
+nohup_dir=$7    # nohup日志存储路径，如: ./youtube/nohup.log
+
+echo "开始"
+#echo "env:"${env}
+
+if [ ${env} = "--env=hk" ];then
+  piaoquan_crawler_dir=/root/piaoquan_crawler/
+  profile_path=/etc/profile
+  python=python3
+elif [ ${env} = "--env=prod" ];then
+  piaoquan_crawler_dir=/data5/piaoquan_crawler/
+  profile_path=/etc/profile
+  python=python
+elif [ ${env} = "--env=local" ];then
+  piaoquan_crawler_dir=/Users/wangkun/Desktop/crawler/piaoquan_crawler/
+  profile_path=/etc/profile
+  node_path=/opt/homebrew/bin/node
+  python=python3
+else
+  echo ${env}
+fi
+
+echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量..."
+cd ~ && source ${profile_path}
+echo "$(date "+%Y-%m-%d %H:%M:%S") 更新环境变量完成！"
+
+#echo "$(date "+%Y-%m-%d %H:%M:%S") 正在杀进程..."
+#grep_str=run_${crawler##*=}
+#ps aux | grep ${grep_str} | grep Python | grep -v grep | awk '{print $2}' | xargs kill -9
+#echo "$(date "+%Y-%m-%d %H:%M:%S") 进程已杀死！"
+
+if [ ${env} = "--env=hk" ];then
+  echo "升级yt-dlp"
+  pip3 install yt-dlp -U
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 正在更新代码..."
+  cd ${piaoquan_crawler_dir} && git pull origin master --force && rm -f ${piaoquan_crawler_dir}main/nohup.log && rm -f ${piaoquan_crawler_dir}${nohup_dir}
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 代码更新完成！"
+fi
+
+if [ ${env} = "--env=hk" ];then
+  echo "无需重启Appium及adb服务"
+elif [ ${env} = "--env=prod" ];then
+  echo "无需重启Appium及adb服务"
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启Appium..."
+  ps aux | grep Appium.app | grep -v grep | awk '{print $2}' | xargs kill -9
+  nohup ${node_path} /Applications/Appium.app/Contents/Resources/app/node_modules/appium/build/lib/main.js >>./nohup.log 2>&1 &
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启Appium完毕!"
+
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启adb..."
+  adb kill-server
+  adb start-server
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 重启adb完毕!"
+fi
+
+echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启服务..."
+cd ${piaoquan_crawler_dir}
+nohup ${python} -u ${crawler_dir} ${log_type} ${crawler} ${task} ${oss_endpoint} ${env} >>${nohup_dir} 2>&1 &
+echo "$(date "+%Y-%m-%d %H:%M:%S") 服务重启完毕!"
+
+exit 0

scheduling/scheduling_v3/crawler_scheduling_v3.py

@@ -4,6 +4,8 @@
 import os
 import sys
 import time
+from hashlib import md5
+
 sys.path.append(os.getcwd())
 from common.common import Common
 from common.scheduling_db import MysqlHelper, RedisHelper
@@ -96,7 +98,37 @@ class SchedulingV3:
             oss_endpoint = "out"
 
         # 正式环境，调度任务
-        Common.logger(log_type, crawler).info(f"开始调度任务:{task}\n")
+        Common.logger(log_type, crawler).info(f"开始调度任务")
+        # task_str = [
+        #     ('task_id', str(task['id'])),
+        #     ('task_name', str(task['task_name'])),
+        #     ('source', str(task['source'])),
+        #     ('start_time', str(task['start_time'])),
+        #     ('interval', str(task['interval'])),
+        #     ('mode', str(task['mode'])),
+        #     ('duration_min', eval(task['rule'])['duration']['min']),
+        #     ('duration_max', eval(task['rule'])['duration']['max']),
+        #     ('play_cnt_min', eval(task['rule'])['playCnt']['min']),
+        #     ('play_cnt_max', eval(task['rule'])['playCnt']['max']),
+        #     ('publish_day_min', eval(task['rule'])['period']['min']),
+        #     ('publish_day_max', eval(task['rule'])['period']['max']),
+        #     ('fans_min', eval(task['rule'])['fans']['min']),
+        #     ('fans_max', eval(task['rule'])['fans']['max']),
+        #     ('videos_min', eval(task['rule'])['videos']['min']),
+        #     ('videos_max', eval(task['rule'])['videos']['max']),
+        #     ('video_like_min', eval(task['rule'])['like']['min']),
+        #     ('video_like_max', eval(task['rule'])['like']['max']),
+        #     ('video_width_min', eval(task['rule'])['videoWidth']['min']),
+        #     ('video_width_max', eval(task['rule'])['videoWidth']['max']),
+        #     ('video_height_min', eval(task['rule'])['videoHeight']['min']),
+        #     ('video_height_max', eval(task['rule'])['videoHeight']['max']),
+        #     ('spider_name', str(task['spider_name'])),
+        #     ('machine', str(task['machine'])),
+        #     ('status', str(task['status'])),
+        #     ('create_time', str(task['create_time'])),
+        #     ('update_time', str(task['update_time'])),
+        #     ('operator', str(task['operator']))
+        # ]
         task_str = [
             ('task_id', str(task['id'])),
             ('task_name', str(task['task_name'])),
@@ -104,22 +136,7 @@ class SchedulingV3:
             ('start_time', str(task['start_time'])),
             ('interval', str(task['interval'])),
             ('mode', str(task['mode'])),
-            ('duration_min', eval(task['rule'])['duration']['min']),
-            ('duration_max', eval(task['rule'])['duration']['max']),
-            ('play_cnt_min', eval(task['rule'])['playCnt']['min']),
-            ('play_cnt_max', eval(task['rule'])['playCnt']['max']),
-            ('publish_day_min', eval(task['rule'])['period']['min']),
-            ('publish_day_max', eval(task['rule'])['period']['max']),
-            ('fans_min', eval(task['rule'])['fans']['min']),
-            ('fans_max', eval(task['rule'])['fans']['max']),
-            ('videos_min', eval(task['rule'])['videos']['min']),
-            ('videos_max', eval(task['rule'])['videos']['max']),
-            ('video_like_min', eval(task['rule'])['like']['min']),
-            ('video_like_max', eval(task['rule'])['like']['max']),
-            ('video_width_min', eval(task['rule'])['videoWidth']['min']),
-            ('video_width_max', eval(task['rule'])['videoWidth']['max']),
-            ('video_height_min', eval(task['rule'])['videoHeight']['min']),
-            ('video_height_max', eval(task['rule'])['videoHeight']['max']),
+            ('rule', task['rule']),
             ('spider_name', str(task['spider_name'])),
             ('machine', str(task['machine'])),
             ('status', str(task['status'])),
@@ -127,17 +144,18 @@ class SchedulingV3:
             ('update_time', str(task['update_time'])),
             ('operator', str(task['operator']))
         ]
-        task_str = str(task_str).replace(' ', '')
-        cmd = f"""sh scheduling/scheduling_main/scheduling_v3.sh {source}/{source}_main/{spider_name}.py --log_type="{mode}" --crawler="{source}" --task="{task_str}" --oss_endpoint="{oss_endpoint}" --env="{env}" >>{source}/logs/{source}-scheduling.log """
-        Common.logger(log_type, crawler).info(f"cmd:{cmd}\n")
+        task_str = str(task_str).replace(' ', '').replace('"', "'").replace("\/", "").replace("/", "")
+        cmd = f"""sh scheduling/scheduling_v3/scheduling_v3.sh {source}/{source}_main/{spider_name}.py --log_type="{mode}" --crawler="{source}" --task="{task_str}" --oss_endpoint="{oss_endpoint}" --env="{env}" {source}/logs/{source}-{mode}-scheduling.log """
+        Common.logger(log_type, crawler).info(f"cmd:{cmd}")
         os.system(cmd)
+        Common.logger(log_type, crawler).info(f"调度任务结束")
 
 
 if __name__ == "__main__":
     # print(Scheduling.get_task("scheduling", "scheduling", "dev"))
     # Scheduling.update_task("scheduling", "scheduling", 8, 1681833600000, 1, "dev")
-    # Scheduling.write_redis("scheduling", "scheduling", "dev")
-    # print(Scheduling.get_redis("scheduling", "scheduling", "dev"))
-    SchedulingV3.scheduling_task("scheduling", "scheduling", "dev")
+    SchedulingV3.write_redis("scheduling", "scheduling", "dev")
+    # print(SchedulingV3.get_redis("scheduling", "scheduling", "dev"))
+    # SchedulingV3.scheduling_task("scheduling", "scheduling", "dev")
     pass
 

scheduling/scheduling_v3/demo.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/4/19

scheduling/scheduling_v3/run_scheduling_task_v3.py

@@ -4,7 +4,6 @@
 import argparse
 import os
 import sys
-
 sys.path.append(os.getcwd())
 from common.common import Common
 from scheduling.scheduling_v3.crawler_scheduling_v3 import SchedulingV3
@@ -13,9 +12,9 @@ from scheduling.scheduling_v3.crawler_scheduling_v3 import SchedulingV3
 class SchedulingTask:
     @classmethod
     def scheduling_task(cls, log_type, crawler, env):
-        Common.logger(log_type, crawler).info("开始调度爬虫任务")
+        # Common.logger(log_type, crawler).info("开始调度爬虫任务")
         SchedulingV3.scheduling_task(log_type, crawler, env)
-        Common.logger(log_type, crawler).info("爬虫任务调度完成")
+        # Common.logger(log_type, crawler).info("爬虫任务调度完成")
         Common.del_logs(log_type, crawler)
 
 

scheduling/scheduling_v3/scheduling_v3.sh

@@ -1,4 +1,10 @@
 #!/bin/bash
+###############################################
+#
+# 通过 run_scheduling_task_v3.py 调起当前 shell
+# 根据 crawler 及 task 的传递值，启动各个爬虫任务
+#
+###############################################
 
 crawler_dir=$1  # 爬虫执行路径，如: ./youtube/youtube_main/run_youtube_follow.py
 log_type=$2     # 日志命名格式，如: follow，则在 youtube/logs/目录下，生成 2023-02-08-follow.log
@@ -6,11 +12,16 @@ crawler=$3      # 哪款爬虫，如: youtube / kanyikan / weixinzhishu
 task=$4         # 爬虫任务
 oss_endpoint=$5 # OSS网关，内网: inner / 外网: out / 香港: hk
 env=$6          # 爬虫运行环境，正式环境: prod / 测试环境: dev
-#machine=$7      # 爬虫运行机器，阿里云服务器: aliyun_hk / aliyun / macpro / macair / local
 nohup_dir=$7    # nohup日志存储路径，如: ./youtube/nohup.log
 
 echo "开始"
-#echo "env:"${env}
+echo "crawler_dir:"${crawler_dir}
+echo "log_type:"${log_type}
+echo "crawler:"${crawler}
+echo "task:"${task}
+echo "oss_endpoint:"${oss_endpoint}
+echo "env:"${env}
+echo "nohup_dir:"${nohup_dir}
 
 if [ ${env} = "--env=hk" ];then
   piaoquan_crawler_dir=/root/piaoquan_crawler/
@@ -20,7 +31,7 @@ elif [ ${env} = "--env=prod" ];then
   piaoquan_crawler_dir=/data5/piaoquan_crawler/
   profile_path=/etc/profile
   python=python
-elif [ ${env} = "--env=local" ];then
+elif [ ${env} = "--env=dev" ];then
   piaoquan_crawler_dir=/Users/wangkun/Desktop/crawler/piaoquan_crawler/
   profile_path=/etc/profile
   node_path=/opt/homebrew/bin/node
@@ -38,13 +49,13 @@ if [ ${env} = "--env=hk" ];then
   pip3 install yt-dlp -U
 else
   echo "$(date "+%Y-%m-%d %H:%M:%S") 正在更新代码..."
-  cd ${piaoquan_crawler_dir} && git pull origin master --force && rm -f ${piaoquan_crawler_dir}main/nohup.log && rm -f ${piaoquan_crawler_dir}${nohup_dir}
+#  cd ${piaoquan_crawler_dir} && git pull origin master --force && rm -f ${piaoquan_crawler_dir}main/nohup.log && rm -f ${piaoquan_crawler_dir}${nohup_dir}
   echo "$(date "+%Y-%m-%d %H:%M:%S") 代码更新完成！"
 fi
 
 echo "$(date "+%Y-%m-%d %H:%M:%S") 正在重启服务..."
 cd ${piaoquan_crawler_dir}
-nohup ${python} -u ${crawler_dir} ${log_type} ${crawler} ${task} ${env} >>${nohup_dir} 2>&1 &
+nohup ${python} -u ${crawler_dir} ${log_type} ${crawler} ${task} ${oss_endpoint} ${env} >> ${nohup_dir} 2>&1 &
 echo "$(date "+%Y-%m-%d %H:%M:%S") 服务重启完毕!"
 
 exit 0

xiaoniangao/logs/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/4/19

xiaoniangao/xiaoniangao_main/run_xiaoniangao_follow_scheduling.py

@@ -1,35 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Author: wangkun
-# @Time: 2023/3/13
-import argparse
-import os
-import sys
-sys.path.append(os.getcwd())
-from common.common import Common
-from xiaoniangao.xiaoniangao_follow.xiaoniangao_follow import XiaoniangaoFollow
-
-
-def main(log_type, crawler, env):
-    if env == "dev":
-        oss_endpoint = "out"
-    else:
-        oss_endpoint = "inner"
-    Common.logger(log_type, crawler).info('开始抓取 小年糕 定向榜\n')
-    XiaoniangaoFollow.get_follow_videos(log_type=log_type,
-                                        crawler=crawler,
-                                        strategy="定向爬虫策略",
-                                        oss_endpoint=oss_endpoint,
-                                        env=env)
-    Common.del_logs(log_type, crawler)
-    Common.logger(log_type, crawler).info('抓取完一轮\n')
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()  ## 新建参数解释器对象
-    parser.add_argument('--log_type', type=str)  ## 添加参数,注明参数类型
-    parser.add_argument('--crawler')  ## 添加参数
-    parser.add_argument('--env')  ## 添加参数
-    args = parser.parse_args()  ### 参数赋值，也可以通过终端赋值
-    main(log_type=args.log_type,
-         crawler=args.crawler,
-         env=args.env)

xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour_scheduling.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/3/15
+import argparse
+import datetime
+import os
+import sys
+sys.path.append(os.getcwd())
+from common.common import Common
+from xiaoniangao.xiaoniangao_hour.xiaoniangao_hour_scheduling import XiaoniangaoHour
+
+
+def main(log_type, crawler, task, oss_endpoint, env):
+    Common.logger(log_type, crawler).info(f"{type(task)}:{task}")
+    task = eval(task)
+    Common.logger(log_type, crawler).info(f"{type(task)}\n")
+    Common.logger(log_type, crawler).info(f"{task}\n")
+    Common.logger(log_type, crawler).info(f"{oss_endpoint}")
+    Common.logger(log_type, crawler).info(f"{env}")
+    # 获取符合规则的视频，写入小时级数据_feeds
+    # XiaoniangaoHour.get_videoList(log_type, crawler, env)
+    # now = datetime.datetime.now()
+    # if now.hour == 10 and 0 <= now.minute <= 10:
+    #     Common.logger(log_type, crawler).info("开始更新/下载上升榜")
+    #     XiaoniangaoHour.update_videoList(log_type=log_type,
+    #                                      crawler=crawler,
+    #                                      strategy="小时榜爬虫策略",
+    #                                      oss_endpoint=oss_endpoint,
+    #                                      env=env)
+    #
+    # elif now.hour == 15 and now.minute <= 10:
+    #     Common.logger(log_type, crawler).info("开始更新/下载上升榜")
+    #     XiaoniangaoHour.update_videoList(log_type=log_type,
+    #                                      crawler=crawler,
+    #                                      strategy="小时榜爬虫策略",
+    #                                      oss_endpoint=oss_endpoint,
+    #                                      env=env)
+    #
+    # elif now.hour == 20 and now.minute <= 10:
+    #     Common.logger(log_type, crawler).info("开始更新/下载上升榜")
+    #     XiaoniangaoHour.update_videoList(log_type=log_type,
+    #                                      crawler=crawler,
+    #                                      strategy="小时榜爬虫策略",
+    #                                      oss_endpoint=oss_endpoint,
+    #                                      env=env)
+    # Common.del_logs(log_type, crawler)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()  ## 新建参数解释器对象
+    parser.add_argument('--log_type', type=str)  ## 添加参数,注明参数类型
+    parser.add_argument('--crawler')  ## 添加参数
+    parser.add_argument('--task')  ## 添加参数
+    parser.add_argument('--oss_endpoint')  ## 添加参数
+    parser.add_argument('--env')  ## 添加参数
+    args = parser.parse_args()  ### 参数赋值，也可以通过终端赋值
+    main(log_type=args.log_type,
+         crawler=args.crawler,
+         task=args.task,
+         oss_endpoint=args.oss_endpoint,
+         env=args.env)