Merge remote-tracking branch 'origin/master'

common/pipeline.py

@@ -5,12 +5,13 @@ from common.scheduling_db import MysqlHelper
 
 
 class PiaoQuanPipeline:
-    def __init__(self, platform, mode, rule_dict, env, item):
+    def __init__(self, platform, mode, rule_dict, env, item, trace_id):
         self.platform = platform
         self.mode = mode
         self.item = item
         self.rule_dict = rule_dict
         self.env = env
+        self.trace_id = trace_id
 
     # 视频的发布时间限制, 属于是规则过滤
     def publish_time_flag(self):
@@ -26,6 +27,7 @@ class PiaoQuanPipeline:
         ):
             AliyunLogger.logging(
                 code="2004",
+                trace_id=self.trace_id,
                 platform=self.platform,
                 mode=self.mode,
                 env=self.env,
@@ -46,6 +48,7 @@ class PiaoQuanPipeline:
         if any(word in cleaned_title for word in sensitive_words):
             AliyunLogger.logging(
                 code="2003",
+                trace_id=self.trace_id,
                 platform=self.platform,
                 mode=self.mode,
                 env=self.env,
@@ -81,6 +84,7 @@ class PiaoQuanPipeline:
                     flag = 0 <= int(self.item[key]) <= max_value
                     AliyunLogger.logging(
                         code="2004",
+                        trace_id=self.trace_id,
                         platform=self.platform,
                         mode=self.mode,
                         env=self.env,
@@ -98,6 +102,7 @@ class PiaoQuanPipeline:
                     )
                     AliyunLogger.logging(
                         code="2004",
+                        trace_id=self.trace_id,
                         platform=self.platform,
                         mode=self.mode,
                         env=self.env,
@@ -127,6 +132,7 @@ class PiaoQuanPipeline:
         if repeat_video:
             AliyunLogger.logging(
                 code="2002",
+                trace_id=self.trace_id,
                 platform=self.platform,
                 mode=self.mode,
                 env=self.env,

youlegaoxiaoxiaoshipin/youlegaoxiaoxiaoshipin_recommend/youlegaoxiaoxiaoshipin_scheduling.py

@@ -5,6 +5,7 @@ import json
 import os
 import sys
 import time
+import uuid
 import random
 import requests
 
@@ -131,6 +132,7 @@ class YLGXXSPScheduling:
                 )
 
     def process_video_obj(self, video_obj):
+        trace_id = self.platform + str(uuid.uuid1())
         video_id = video_obj.get("data", {}).get("id", 0)
         video_title = clean_title(video_obj.get("data", {}).get("title", "no title"))
         video_time = video_obj["data"]["duration"]
@@ -164,6 +166,7 @@ class YLGXXSPScheduling:
             rule_dict=self.rule_dict,
             env=self.env,
             item=video_dict,
+            trace_id=trace_id
         )
         flag = rule_pipeline.process_item()
         if flag:
@@ -186,6 +189,7 @@ class YLGXXSPScheduling:
                 mode=self.mode,
                 env=self.env,
                 data=video_dict,
+                trace_id=trace_id,
                 message="成功发送 MQ 至 ETL",
             )
 

zhufushenghuo/zhufushenghuo_recommend/zfsh_recommend.py

@@ -12,13 +12,14 @@ from common.public import download_rule, get_config_from_mysql
 
 proxies = {"http": None, "https": None}
 headers = {
-            'Host': 'mini.vvuiiu.cn',
-            'accept': '*/*',
-            'content-type': 'application/json',
-            'accept-language': 'zh-cn',
-            'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
-            'referer': 'https://servicewechat.com/wxa903dc775e85eb5b/11/page-frame.html'
-        }
+    'Host': 'mini.vvuiiu.cn',
+    'accept': '*/*',
+    'content-type': 'application/json',
+    'accept-language': 'zh-cn',
+    'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
+    'referer': 'https://servicewechat.com/wxa903dc775e85eb5b/11/page-frame.html'
+}
+
 
 class ZfshRecommend:
     platform = ("祝福生活")
@@ -58,8 +59,6 @@ class ZfshRecommend:
             Common.logger(log_type, crawler).error(f" 获取视频链接异常:{e}\n")
             Common.logging(log_type, crawler, env, f"获取视频链接异常:{e}\n")
 
-
-
     @classmethod
     def repeat_video(cls, log_type, crawler, video_id, env):
         sql = f""" select * from crawler_video where platform in ("{crawler}","{cls.platform}") and create_time>='2023-06-26' and out_video_id="{video_id}"; """
@@ -195,7 +194,6 @@ class ZfshRecommend:
                 Common.logging(log_type, crawler, env, f"抓取第{page}页时异常:{e}\n")
 
 
-
 if __name__ == "__main__":
     rule_dict1 = {"period": {"min": 365, "max": 365},
                   "duration": {"min": 30, "max": 1800},