小年糕推荐流增加日志上报

application/config/config.py

@@ -2,6 +2,7 @@
 crawler_api_domain = 'http://8.217.192.46:8889'
 zhufuquanzi_view_api = crawler_api_domain + '/crawler/zhu_fu_quan_zi/detail_exposure'
 zhufuquanzi_history_api = crawler_api_domain + '/crawler/zhu_fu_quan_zi/detail_history'
+zhufuquanzi_log_upload_api = crawler_api_domain + '/crawler/zhu_fu_quan_zi/log_upload'
 xiaoniangao_view_api = crawler_api_domain + '/crawler/xiao_nian_gao_plus/detail_exposure'
 xiaoniangao_history_api = crawler_api_domain + '/crawler/xiao_nian_gao_plus/detail_history'
-zhufuquanzi_log_upload_api = crawler_api_domain + '/crawler/zhu_fu_quan_zi/log_upload'
+xiaoniangao_log_upload_api = crawler_api_domain + '/crawler/xiao_nian_gao_plus/log_upload'

spider/crawler_online/xiaoniangaotuijianliu.py

@@ -5,6 +5,7 @@ import time
 import uuid
 import json
 from datetime import datetime
+from typing import Dict, Any
 
 import cv2
 import requests
@@ -14,7 +15,7 @@ from application.common.feishu.feishu_utils import FeishuUtils
 from application.common.gpt import GPT4oMini
 from application.common.mysql.sql import Sql
 from application.common.redis.xng_redis import xng_in_video_data
-from application.config.config import xiaoniangao_view_api,xiaoniangao_history_api
+from application.config.config import xiaoniangao_view_api,xiaoniangao_history_api,xiaoniangao_log_upload_api
 sys.path.append(os.getcwd())
 
 from application.items import VideoItem
@@ -92,7 +93,38 @@ def video_history(video_view_lists):
         print(f"{xiaoniangao_history_api}请求异常: {e}")
     except json.JSONDecodeError:
         print(f"{xiaoniangao_history_api}响应不是有效的 JSON 格式")
-
+def log_upload(video_objs):
+    headers = {
+        "Content-Type": "application/json"
+    }
+    payload = {
+        "e": video_objs
+    }
+    try:
+        # 发送 POST 请求
+        response = requests.post(
+            xiaoniangao_log_upload_api,
+            headers=headers,
+            json=payload  # 自动将字典转换为 JSON
+        )
+        # 检查 HTTP 状态码
+        if response.status_code == 200:
+            # 解析 JSON 响应
+            result = response.json()
+            # 提取关键字段
+            code = result.get("code")
+            msg = result.get("msg")
+            # 业务逻辑处理（示例）
+            if code == 0:
+                print(f"{xiaoniangao_log_upload_api}请求成功")
+            else:
+                print(f"{xiaoniangao_log_upload_api}请求失败，错误码: {code}, 消息: {msg}")
+        else:
+            print(f"{xiaoniangao_log_upload_api} 请求失败，状态码: {response.status_code}")
+    except requests.exceptions.RequestException as e:
+        print(f"{xiaoniangao_log_upload_api}请求异常: {e}")
+    except json.JSONDecodeError:
+        print(f"{xiaoniangao_log_upload_api}响应不是有效的 JSON 格式")
 
 class XNGTJLRecommend(object):
 
@@ -146,6 +178,7 @@ class XNGTJLRecommend(object):
                 ),
                 return
             video_view_lists = []
+            video_log_uploads = []
             for index, video_obj in enumerate(response['data']['data'], 1):
                 try:
                     self.aliyun_log.logging(
@@ -153,6 +186,7 @@ class XNGTJLRecommend(object):
                     )
                     vid = video_obj['id']
                     video_view_lists.append(str(vid))
+                    video_log_uploads.append(self.build_video_log(video_obj, index))
                     self.process_video_obj(video_obj, title_rule)
                 except Exception as e:
                     self.aliyun_log.logging(
@@ -166,6 +200,7 @@ class XNGTJLRecommend(object):
 
                 time.sleep(random.randint(5, 10))
             video_history(video_view_lists)
+            log_upload(video_log_uploads)
 
     def process_video_obj(self, video_obj, title_rule):
         """
@@ -278,6 +313,32 @@ class XNGTJLRecommend(object):
             return False
         return True
 
+    def build_video_log(self, video_obj: Dict[str, Any], index: int) -> Dict[str, Any]:
+        """构建视频日志对象"""
+        return {
+            "ac": "show",
+            "md_ver": "2.0",
+            "data": {
+                "page": "discoverIndexPage",
+                "topic": "recommend",
+                "tpl_id": str(video_obj['tpl_id']),
+                "profile_ct": str(video_obj['p_ct']),
+                "flv": "0",
+                "sign": video_obj['sign'],
+                "serial_id": video_obj['serial_id'],
+                "type": "post",
+                "name": "post",
+                "src_page": "discoverIndexPage/recommend",
+                "aid": str(video_obj['album_id']),
+                "cid": str(video_obj['id']),
+                "feed_idx": index - 1,
+                "cmid": str(video_obj['user']['mid']),
+                "user_ct": "1638346045009"
+            },
+            "t": int(time.time() * 1000),
+            "ab": {}
+        }
+
     def run(self):
         self.get_recommend_list()