吉祥祝福为你传递好运 运行脚本优化

jixiangzhufuweinichuandihaoyun/jixiangzhufuweinichuandihaoyun_main/run_jxzfwncdhy_recommend.py

@@ -1,19 +1,17 @@
 import argparse
-import random
-
 from mq_http_sdk.mq_client import *
 from mq_http_sdk.mq_consumer import *
 from mq_http_sdk.mq_exception import MQExceptionBase
 
 
-
 sys.path.append(os.getcwd())
-from common.common import Common
 from common.public import task_fun_mq, get_consumer, ack_message
 from common.scheduling_db import MysqlHelper
+from common import AliyunLogger
 from jixiangzhufuweinichuandihaoyun.jixiangzhufuweinichuandihaoyun_recommend.jixiangzhufuweinichuandihaoyun_recommend import Jxzfwncdhyspcheduling
 from jixiangzhufuweinichuandihaoyun.jixiangzhufuweinichuandihaoyun_recommend.zhufudaigeinimenxiqiyangyang_recommend import Zfdgnmxqyyspcheduling
 from jixiangzhufuweinichuandihaoyun.jixiangzhufuweinichuandihaoyun_recommend.zhufusonggeiniwendingjixiang_recommend import Zfsgnwdjxspcheduling
+
 def main(log_type, crawler, topic_name, group_id, env):
     consumer = get_consumer(topic_name, group_id)
     # 长轮询表示如果Topic没有消息，则客户端请求会在服务端挂起3秒，3秒内如果有消息可以消费则立即返回响应。
@@ -21,17 +19,12 @@ def main(log_type, crawler, topic_name, group_id, env):
     wait_seconds = 30
     # 一次最多消费3条（最多可设置为16条）。
     batch = 1
-    Common.logger(log_type, crawler).info(
-        f'{10 * "="}Consume And Ack Message From Topic{10 * "="}\n'
-        f"WaitSeconds:{wait_seconds}\n"
-        f"TopicName:{topic_name}\n"
-        f"MQConsumer:{group_id}"
-    )
-    Common.logging(
-        log_type,
-        crawler,
-        env,
-        f'{10 * "="}Consume And Ack Message From Topic{10 * "="}\n'
+    AliyunLogger.logging(
+        code="1000",
+        platform=crawler,
+        mode=log_type,
+        env=env,
+        message=f'{10 * "="}Consume And Ack Message From Topic{10 * "="}\n'
         f"WaitSeconds:{wait_seconds}\n"
         f"TopicName:{topic_name}\n"
         f"MQConsumer:{group_id}",
@@ -41,24 +34,12 @@ def main(log_type, crawler, topic_name, group_id, env):
             # 长轮询消费消息。
             recv_msgs = consumer.consume_message(batch, wait_seconds)
             for msg in recv_msgs:
-                xng_author_start_time = int(time.time())
-                Common.logger(log_type, crawler).info(
-                    f"Receive\n"
-                    f"MessageId:{msg.message_id}\n"
-                    f"MessageBodyMD5:{msg.message_body_md5}\n"
-                    f"MessageTag:{msg.message_tag}\n"
-                    f"ConsumedTimes:{msg.consumed_times}\n"
-                    f"PublishTime:{msg.publish_time}\n"
-                    f"Body:{msg.message_body}\n"
-                    f"NextConsumeTime:{msg.next_consume_time}\n"
-                    f"ReceiptHandle:{msg.receipt_handle}\n"
-                    f"Properties:{msg.properties}"
-                )
-                Common.logging(
-                    log_type,
-                    crawler,
-                    env,
-                    f"Receive\n"
+                AliyunLogger.logging(
+                    code="1000",
+                    platform=crawler,
+                    mode=log_type,
+                    env=env,
+                    message=f"Receive\n"
                     f"MessageId:{msg.message_id}\n"
                     f"MessageBodyMD5:{msg.message_body_md5}\n"
                     f"MessageTag:{msg.message_tag}\n"
@@ -76,10 +57,25 @@ def main(log_type, crawler, topic_name, group_id, env):
                     recv_msgs=recv_msgs,
                     consumer=consumer,
                 )
-
-                # 处理爬虫业务
+                # 解析 task_dict
                 task_dict = task_fun_mq(msg.message_body)["task_dict"]
+                AliyunLogger.logging(
+                    code="1000",
+                    platform=crawler,
+                    mode=log_type,
+                    env=env,
+                    message="f调度任务:{task_dict}",
+                )
+                # 解析 rule_dict
                 rule_dict = task_fun_mq(msg.message_body)["rule_dict"]
+                AliyunLogger.logging(
+                    code="1000",
+                    platform=crawler,
+                    mode=log_type,
+                    env=env,
+                    message=f"抓取规则:{rule_dict}\n",
+                )
+                # 解析 user_list
                 task_id = task_dict["id"]
                 select_user_sql = (
                     f"""select * from crawler_user_v3 where task_id={task_id}"""
@@ -87,66 +83,73 @@ def main(log_type, crawler, topic_name, group_id, env):
                 user_list = MysqlHelper.get_values(
                     log_type, crawler, select_user_sql, env, action=""
                 )
-                Common.logger(log_type, crawler).info(f"调度任务:\n{task_dict}")
-                Common.logging(log_type, crawler, env, f"调度任务:{task_dict}")
-                Common.logger(log_type, crawler).info(f"抓取规则:\n{rule_dict}")
-                Common.logging(log_type, crawler, env, f"抓取规则:{rule_dict}")
-                Common.logger(log_type, crawler).info(f"用户列表:\n{user_list}")
-                Common.logging(log_type, crawler, env, f"用户列表:\n{user_list}")
-                Common.logger(log_type, crawler).info(f'开始抓取:{task_dict["taskName"]}\n')
-                Common.logging(
-                    log_type, crawler, env, f'开始抓取:{task_dict["taskName"]}\n'
+                AliyunLogger.logging(
+                    code="1003",
+                    platform=crawler,
+                    mode=log_type,
+                    env=env,
+                    message="开始抓取"
+                )
+                AliyunLogger.logging(
+                    code="1000",
+                    platform=crawler,
+                    mode=log_type,
+                    env=env,
+                    message="开始抓取吉祥祝福为你传递好运——推荐",
                 )
-                our_uid_list = []
-                for user in user_list:
-                    our_uid_list.append(user["uid"])
-                our_uid = random.choice(our_uid_list)
                 HH = Jxzfwncdhyspcheduling(
-                    log_type=log_type,
-                    crawler=crawler,
+                    platform=crawler,
+                    mode=log_type,
                     rule_dict=rule_dict,
-                    env=env,
-                    our_uid=our_uid
+                    user_list=user_list,
+                    env=env
                 )
                 HH.get_videoList()
-
                 YY = Zfdgnmxqyyspcheduling(
-                    log_type=log_type,
-                    crawler=crawler,
+                    platform=crawler,
+                    mode=log_type,
                     rule_dict=rule_dict,
-                    env=env,
-                    our_uid=our_uid
+                    user_list=user_list,
+                    env=env
                 )
-
                 YY.get_videoList()
                 XX = Zfsgnwdjxspcheduling(
-                    log_type=log_type,
-                    crawler=crawler,
+                    platform=crawler,
+                    mode=log_type,
                     rule_dict=rule_dict,
-                    env=env,
-                    our_uid=our_uid
+                    user_list=user_list,
+                    env=env
                 )
-
                 XX.get_videoList()
-                Common.logger(log_type, crawler).info("抓取一轮结束\n")
-                Common.logging(log_type, crawler, env, "抓取一轮结束\n")
+
+                AliyunLogger.logging(
+                    code="1000",
+                    platform=crawler,
+                    mode=log_type,
+                    env=env,
+                    message="完成抓取——吉祥祝福为你传递好运",
+                )
+                AliyunLogger.logging(
+                    code="1004", platform=crawler, mode=log_type, env=env,message="结束一轮抓取"
+                )
 
         except MQExceptionBase as err:
             # Topic中没有消息可消费。
             if err.type == "MessageNotExist":
-                Common.logger(log_type, crawler).info(
-                    f"No new message! RequestId:{err.req_id}\n"
-                )
-                Common.logging(
-                    log_type, crawler, env, f"No new message! RequestId:{err.req_id}\n"
+                AliyunLogger.logging(
+                    code="2000",
+                    platform=crawler,
+                    mode=log_type,
+                    env=env,
+                    message=f"No new message! RequestId:{err.req_id}\n",
                 )
                 continue
-
-            Common.logger(log_type, crawler).info(
-                f"Consume Message Fail! Exception:{err}\n"
-            )
-            Common.logging(
-                log_type, crawler, env, f"Consume Message Fail! Exception:{err}\n"
+            AliyunLogger.logging(
+                code="2000",
+                platform=crawler,
+                mode=log_type,
+                env=env,
+                message=f"Consume Message Fail! Exception:{err}\n",
             )
             time.sleep(2)
             continue