add 欢欢喜喜祝福到

huanhuanxixizhufudao/huanhuanxixizhufudao_main/run_hhxxzfd_recommend.py

@@ -1,22 +1,23 @@
 # -*- coding: utf-8 -*-
-# @Author: wang
+# @Author:
 # @Time: 2023/9/7
 import argparse
 import random
 import multiprocessing
-import time
 
 from mq_http_sdk.mq_client import *
 from mq_http_sdk.mq_consumer import *
 from mq_http_sdk.mq_exception import MQExceptionBase
 
+
 sys.path.append(os.getcwd())
 from common.common import Common
-from common.public import get_consumer, ack_message, task_fun_mq
+from common.public import get_consumer, ack_message, task_fun_mq, get_rule_from_mysql
 from common.scheduling_db import MysqlHelper
 from huanhuanxixizhufudao.huanhuanxixizhufudao_recommend.huanhuanxixizhufudao_recommend import HhxxzfdRecommend
 
 
+
 def run(args1, args2, args3, args4, args5):
     HhxxzfdRecommend.start_wechat(log_type=args1,
                                crawler=args2,
@@ -27,7 +28,7 @@ def run(args1, args2, args3, args4, args5):
 
 class HhxxzfdMain:
     @classmethod
-    def zhufuquanzi_main(cls, log_type, crawler, topic_name, group_id, env):
+    def main(cls, log_type, crawler, topic_name, group_id, env):
         consumer = get_consumer(topic_name, group_id)
         # 长轮询表示如果Topic没有消息，则客户端请求会在服务端挂起3秒，3秒内如果有消息可以消费则立即返回响应。
         # 长轮询时间3秒（最多可设置为30秒）。
@@ -88,7 +89,13 @@ class HhxxzfdMain:
                     Common.logging(log_type, crawler, env, f"用户列表:{user_list}\n")
                     Common.logger(log_type, crawler).info(f'开始抓取:{task_dict["taskName"]}\n')
                     Common.logging(log_type, crawler, env, f'开始抓取:{task_dict["taskName"]}\n')
-
+                    new_r = get_rule_from_mysql(task_id=task_id, log_type=log_type, crawler=crawler, env=env)
+                    r_d = {}
+                    for item in new_r:
+                        for k, val in item.items():
+                            r_d[k] = val
+                    Common.logger(log_type, crawler).info(f"抓取规则:{r_d}")
+                    Common.logging(log_type, crawler, env, f"抓取规则:{r_d}")
                     process = multiprocessing.Process(
                         target=run,
                         args=(log_type, crawler, rule_dict, our_uid, env)
@@ -96,13 +103,20 @@ class HhxxzfdMain:
                     process.start()
                     print("进程开始")
 
-                    while True:
+                    for i in range(10):
                         if not process.is_alive():
 
                             print("进程异常，准备重启")
                             process.terminate()
                             os.system("adb forward --remove-all")
                             time.sleep(60)
+                            new_r = get_rule_from_mysql(task_id=task_id, log_type=log_type, crawler=crawler, env=env)
+                            r_d = {}
+                            for item in new_r:
+                                for k, val in item.items():
+                                    r_d[k] = val
+                            Common.logger(log_type, crawler).info(f"抓取规则:{r_d}")
+                            Common.logging(log_type, crawler, env, f"抓取规则:{r_d}")
                             process = multiprocessing.Process(
                                 target=run,
                                 args=(log_type, crawler, rule_dict, our_uid, env)
@@ -140,8 +154,8 @@ if __name__ == "__main__":
     parser.add_argument('--group_id')  ## 添加参数
     parser.add_argument('--env')  ## 添加参数
     args = parser.parse_args()  ### 参数赋值，也可以通过终端赋值
-    HhxxzfdMain.zhufuquanzi_main(log_type=args.log_type,
+    HhxxzfdMain.main(log_type=args.log_type,
                               crawler=args.crawler,
                               topic_name=args.topic_name,
                               group_id=args.group_id,
-                              env=args.env)
+                              env=args.env)

huanhuanxixizhufudao/huanhuanxixizhufudao_recommend/huanhuanxixizhufudao_recommend.py

@@ -128,7 +128,8 @@ class HhxxzfdRecommend:
 
     @classmethod
     def repeat_video(cls, log_type, crawler, video_id, env):
-        sql = f""" select * from crawler_video where platform in ("众妙音信", "刚刚都传", "吉祥幸福", "知青天天看", "zhufuquanzi", "祝福圈子", "haitunzhufu", "海豚祝福") and out_video_id="{video_id}"; """
+        sql = f""" select * from crawler_video where platform in ("{crawler}","{cls.platform}") and out_video_id="{video_id}"; """
+
         repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
         return len(repeat_video)