增加小年糕原来阿里云日志

xiaoniangao/xiaoniangao_author/xiaoniangao_author_v2.py

@@ -56,6 +56,12 @@ class XiaoNianGaoAuthor:
                     env=self.env,
                     message="本轮已经抓取足够数量的视频，已经自动退出",
                 )
+                Common.logging(
+                    log_type=self.mode,
+                    crawler=self.platform,
+                    env=self.env,
+                    message="本轮已经抓取足够数量的视频，已经自动退出",
+                )
                 return
 
     def get_video_list(self, user_dict):
@@ -92,6 +98,12 @@ class XiaoNianGaoAuthor:
                 Common.logger(self.mode, self.platform).info(
                     f"get_videoList:{response.text}\n"
                 )
+                Common.logging(
+                    log_type=self.mode,
+                    crawler=self.platform,
+                    env=self.env,
+                    message=f"get_videoList:{response.text}\n"
+                )
                 AliyunLogger.logging(
                     code="2000",
                     platform=self.platform,
@@ -104,6 +116,12 @@ class XiaoNianGaoAuthor:
                 Common.logger(self.mode, self.platform).info(
                     f"get_videoList:{response.json()}\n"
                 )
+                Common.logging(
+                    log_type=self.mode,
+                    crawler=self.platform,
+                    env=self.env,
+                    message=f"get_videoList:{response.json()}\n"
+                )
                 AliyunLogger.logging(
                     code="2000",
                     platform=self.platform,
@@ -114,6 +132,12 @@ class XiaoNianGaoAuthor:
                 return
             elif len(response.json()["data"]["list"]) == 0:
                 Common.logger(self.mode, self.platform).info(f"没有更多数据啦~\n")
+                Common.logging(
+                    log_type=self.mode,
+                    crawler=self.platform,
+                    env=self.env,
+                    message=f"没有更多数据啦~\n"
+                )
                 AliyunLogger.logging(
                     code="2000",
                     platform=self.platform,
@@ -134,6 +158,12 @@ class XiaoNianGaoAuthor:
                             env=self.env,
                             message="扫描到一条视频",
                         )
+                        Common.logging(
+                            log_type=self.mode,
+                            crawler=self.platform,
+                            env=self.env,
+                            message=f"扫描到一条视频"
+                        )
                         self.process_video_obj(video_obj, user_dict)
                     except Exception as e:
                         AliyunLogger.logging(
@@ -144,6 +174,12 @@ class XiaoNianGaoAuthor:
                             data=video_obj,
                             message="抓取单条视频异常, 报错原因是: {}".format(e),
                         )
+                        Common.logging(
+                            log_type=self.mode,
+                            crawler=self.platform,
+                            env=self.env,
+                            message="抓取单条视频异常, 报错原因是: {}".format(e),
+                        )
 
     def process_video_obj(self, video_obj, user_dict):
         trace_id = self.platform + str(uuid.uuid1())
@@ -228,7 +264,12 @@ class XiaoNianGaoAuthor:
                 trace_id=trace_id,
                 message="成功发送 MQ 至 ETL",
             )
-
+            Common.logging(
+                log_type=self.mode,
+                crawler=self.platform,
+                env=self.env,
+                message="成功发送 MQ 至 ETL",
+            )
 
 # if __name__ == "__main__":
 #     XNGA = XiaoNianGaoAuthor(

xiaoniangao/xiaoniangao_main/run_xng_author_v2.py

@@ -2,12 +2,12 @@
 # @Author: luojunhui
 # @Time: 2023/10/23
 import argparse
-import random
 from mq_http_sdk.mq_client import *
 from mq_http_sdk.mq_consumer import *
 from mq_http_sdk.mq_exception import MQExceptionBase
 
 sys.path.append(os.getcwd())
+from common.common import Common
 from common.public import task_fun_mq, get_consumer, ack_message
 from common.scheduling_db import MysqlHelper
 from common.aliyun_log import AliyunLogger
@@ -31,6 +31,10 @@ def main(my_platform, mode, topic_name, group_id, env):
         f"TopicName:{topic_name}\n"
         f"MQConsumer:{group_id}",
     )
+    Common.logging(log_type=mode, crawler=my_platform, env=env, message=f'{10 * "="}Consume And Ack Message From Topic{10 * "="}\n'
+                                           f'WaitSeconds:{wait_seconds}\n'
+                                           f'TopicName:{topic_name}\n'
+                                           f'MQConsumer:{group_id}')
     while True:
         try:
             # 长轮询消费消息。
@@ -52,6 +56,16 @@ def main(my_platform, mode, topic_name, group_id, env):
                     f"ReceiptHandle:{msg.receipt_handle}\n"
                     f"Properties:{msg.properties}",
                 )
+                Common.logging(mode, my_platform, env, f"Receive\n"
+                                                       f"MessageId:{msg.message_id}\n"
+                                                       f"MessageBodyMD5:{msg.message_body_md5}\n"
+                                                       f"MessageTag:{msg.message_tag}\n"
+                                                       f"ConsumedTimes:{msg.consumed_times}\n"
+                                                       f"PublishTime:{msg.publish_time}\n"
+                                                       f"Body:{msg.message_body}\n"
+                                                       f"NextConsumeTime:{msg.next_consume_time}\n"
+                                                       f"ReceiptHandle:{msg.receipt_handle}\n"
+                                                       f"Properties:{msg.properties}")
                 # ack_mq_message
                 ack_message(
                     log_type=mode,