automatic_crawler/spider/crawler_online/zhufuquanzituijianliu.py

import os
import random
import sys
import time
import uuid
import json
from datetime import datetime

import cv2
import requests

from application.common.feishu import FsData
from application.common.feishu.feishu_utils import FeishuUtils
from application.common.gpt import GPT4oMini
from application.common.mysql.sql import Sql
from typing import Dict, Any
from application.common.redis.xng_redis import xng_in_video_data
from application.config.config import zhufuquanzi_view_api,zhufuquanzi_history_api,zhufuquanzi_log_upload_api

sys.path.append(os.getcwd())

from application.items import VideoItem
from application.pipeline import PiaoQuanPipeline
from application.common.messageQueue import MQ
from application.common.log import AliyunLogger
from application.common.mysql import MysqlHelper


def video_view(content_id, account_id):
    headers = {
        "Content-Type": "application/json"
    }
    payload = {
        "content_id": str(content_id),
        "account_id": str(account_id)
    }
    try:
        # 发送 POST 请求
        response = requests.post(
            zhufuquanzi_view_api,
            headers=headers,
            json=payload  # 自动将字典转换为 JSON
        )
        # 检查 HTTP 状态码
        if response.status_code == 200:
            # 解析 JSON 响应
            result = response.json()
            # 提取关键字段
            code = result.get("code")
            msg = result.get("msg")
            # 业务逻辑处理（示例）
            if code == 0:
                print("请求成功")
            else:
                print(f"{zhufuquanzi_view_api}请求失败，错误码: {code}, 消息: {msg}")
        else:
            print(f"{zhufuquanzi_view_api}HTTP 请求失败，状态码: {response.status_code}")
    except requests.exceptions.RequestException as e:
        print(f"{zhufuquanzi_view_api}请求异常: {e}")
    except json.JSONDecodeError:
        print(f"{zhufuquanzi_view_api}响应不是有效的 JSON 格式")

def video_history(video_view_lists):
    headers = {
        "Content-Type": "application/json"
    }
    payload = {
        "content_ids": video_view_lists
    }
    try:
        # 发送 POST 请求
        response = requests.post(
            zhufuquanzi_history_api,
            headers=headers,
            json=payload  # 自动将字典转换为 JSON
        )
        # 检查 HTTP 状态码
        if response.status_code == 200:
            # 解析 JSON 响应
            result = response.json()
            # 提取关键字段
            code = result.get("code")
            msg = result.get("msg")
            # 业务逻辑处理（示例）
            if code == 0:
                print("请求成功")
            else:
                print(f"{zhufuquanzi_history_api}请求失败，错误码: {code}, 消息: {msg}")
        else:
            print(f"{zhufuquanzi_history_api}HTTP 请求失败，状态码: {response.status_code}")
    except requests.exceptions.RequestException as e:
        print(f"{zhufuquanzi_history_api}请求异常: {e}")
    except json.JSONDecodeError:
        print(f"{zhufuquanzi_history_api}响应不是有效的 JSON 格式")


def log_upload(video_objs):
    headers = {
        "Content-Type": "application/json"
    }
    payload = {
        "e": video_objs
    }
    try:
        # 发送 POST 请求
        response = requests.post(
            zhufuquanzi_log_upload_api,
            headers=headers,
            json=payload  # 自动将字典转换为 JSON
        )
        # 检查 HTTP 状态码
        if response.status_code == 200:
            # 解析 JSON 响应
            result = response.json()
            # 提取关键字段
            code = result.get("code")
            msg = result.get("msg")
            # 业务逻辑处理（示例）
            if code == 0:
                print(f"{zhufuquanzi_log_upload_api}请求成功")
            else:
                print(f"{zhufuquanzi_log_upload_api}请求失败，错误码: {code}, 消息: {msg}")
        else:
            print(f"{zhufuquanzi_log_upload_api}HTTP 请求失败，状态码: {response.status_code}")
    except requests.exceptions.RequestException as e:
        print(f"{zhufuquanzi_log_upload_api}请求异常: {e}")
    except json.JSONDecodeError:
        print(f"{zhufuquanzi_log_upload_api}响应不是有效的 JSON 格式")






class ZFQZTJLRecommend(object):
    """
    祝福圈子推荐流
    """

    def __init__(self, platform, mode, rule_dict, user_list, env="prod"):
        self.limit_flag = False
        self.platform = platform
        self.mode = mode
        self.rule_dict = rule_dict
        self.user_list = user_list
        self.env = env
        self.download_cnt = 0
        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
        self.expire_flag = False
        self.aliyun_log = AliyunLogger(mode=self.mode, platform=self.platform)
        self.mysql = MysqlHelper(mode=self.mode, platform=self)

    def get_video_duration(self, video_link: str) -> int:
        cap = cv2.VideoCapture(video_link)
        if cap.isOpened():
            rate = cap.get(5)
            frame_num = cap.get(7)
            duration = int(frame_num / rate)
            return duration
        return 0

    def get_recommend_list(self):
        print("祝福圈子推荐流开始")
        """
        获取推荐页视频
        """

        headers = {
            'Content-Type': 'application/json'
        }
        data_rule = FsData()
        title_rule = data_rule.get_title_rule()
        for i in range(4):
        # 目前90%视频会被过滤掉，改为频率和策略
        # for i in range(100):
            url = "http://8.217.192.46:8889/crawler/zhu_fu_quan_zi/recommend"
            payload = json.dumps({})
            response = requests.request("POST", url, headers=headers, data=payload)
            response = response.json()
            if  response['code'] != 0:
                self.aliyun_log.logging(
                    code="3000",
                    message="抓取单条视频失败,请求失败"
                    ),
                return
            video_view_lists = []
            video_log_uploads = []
            for index, video_obj in enumerate(response['data']['data'], 1):
                try:
                    self.aliyun_log.logging(
                        code="1001", message="扫描到一条视频", data=video_obj
                    )
                    vid = video_obj['id']
                    video_view_lists.append(str(vid))
                    video_log_uploads.append(self.build_video_log(video_obj,index))
                    self.process_video_obj(video_obj,title_rule)
                except Exception as e:
                    self.aliyun_log.logging(
                        code="3000",
                        message="抓取单条视频失败, 该视频位于第{}页第{}条报错原因是{}".format(
                            1, index, e
                        ),
                    )
                if self.limit_flag:
                    return
                time.sleep(random.randint(5, 10))
                # time.sleep(random.randint(5, 30))
            video_history(video_view_lists)
            log_upload(video_log_uploads)
    def process_video_obj(self, video_obj, title_rule):
        """
        处理视频
        :param video_obj:
        """
        time.sleep(random.randint(3, 8))
        trace_id = self.platform + str(uuid.uuid1())
        our_user = random.choice(self.user_list)
        item = VideoItem()
        vid = video_obj['id']
        mid = int(video_obj['user']['mid'])
        print(f"vid={vid},mid={mid}")
        try:
            user_name = video_obj['user']['nick']
            avatar_url = video_obj['user']['hurl']
            sql = Sql()
            max_id = sql.select_id(mid)
            if max_id:
                sql.update_name_url(mid, avatar_url, user_name)
            else:
                time.sleep(1)
                link = sql.select_id_status(mid)
                if link:
                    sql.insert_name_url(mid, avatar_url, user_name)
                    print(f"开始写入{mid}")
                    xng_in_video_data(json.dumps({"mid": mid}))
        except Exception as e:
            print(f"写入异常{e}")
            pass
        url =  video_obj["v_url"]
        duration = self.get_video_duration(url)
        item.add_video_info("video_id", video_obj["id"])
        item.add_video_info("video_title", video_obj["title"])
        item.add_video_info("play_cnt", int(video_obj["play_pv"]))
        item.add_video_info("publish_time_stamp", int(int(video_obj["t"])/1000))
        item.add_video_info("out_user_id", video_obj["id"])
        item.add_video_info("cover_url", video_obj["url"])
        item.add_video_info("like_cnt", 0)
        item.add_video_info("share_cnt", int(video_obj["share"]))
        item.add_video_info("comment_cnt", int(video_obj["comment_count"]))
        item.add_video_info("video_url", video_obj["v_url"])
        item.add_video_info("out_video_id", video_obj["id"])
        item.add_video_info("duration", int(duration))
        item.add_video_info("platform", self.platform)
        item.add_video_info("strategy", self.mode)
        item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
        item.add_video_info("user_id", our_user["uid"])
        item.add_video_info("user_name", our_user["nick_name"])

        mq_obj = item.produce_item()
        pipeline = PiaoQuanPipeline(
            platform=self.platform,
            mode=self.mode,
            rule_dict=self.rule_dict,
            env=self.env,
            item=mq_obj,
            trace_id=trace_id,
        )
        if pipeline.process_item():
            title_list = title_rule.split(",")
            title = video_obj["title"]
            contains_keyword = any(keyword in title for keyword in title_list)
            if contains_keyword:
                new_title = GPT4oMini.get_ai_mini_title(title)
                if new_title:
                    item.add_video_info("video_title", new_title)
                    current_time = datetime.now()
                    formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S")
                    values = [
                        [
                            video_obj["v_url"],
                            video_obj["url"],
                            title,
                            new_title,
                            formatted_time,
                        ]
                    ]
                    FeishuUtils.insert_columns("U5dXsSlPOhiNNCtEfgqcm1iYnpf", "GVottu", "ROWS", 1, 2)
                    time.sleep(0.5)
                    FeishuUtils.update_values("U5dXsSlPOhiNNCtEfgqcm1iYnpf", "GVottu", "A2:Z2", values)
            self.download_cnt += 1
            self.mq.send_msg(mq_obj)
            self.aliyun_log.logging(code="1002", message="成功发送至 ETL", data=mq_obj)
            video_view(vid, mid)
            self.aliyun_log.logging(code="1010", message="触发曝光", data=mq_obj)
            if self.download_cnt >= int(
                    self.rule_dict.get("videos_cnt", {}).get("min", 200)
            ):
                self.limit_flag = True

    def build_video_log(self, video_obj: Dict[str, Any], index: int) -> Dict[str, Any]:
        """构建视频日志对象"""
        return {
            "ac": "show",
            "md_ver": "2.0",
            "data": {
                "page": "discoverIndexPage",
                "topic": "recommend",
                "tpl_id": str(video_obj['tpl_id']),
                "profile_ct": str(video_obj['p_ct']),
                "flv": "0",
                "sign": video_obj['sign'],
                "serial_id": video_obj['serial_id'],
                "type": "post",
                "name": "post",
                "src_page": "discoverIndexPage/recommend",
                "aid": str(video_obj['album_id']),
                "cid": str(video_obj['id']),
                "feed_idx": index - 1,
                "cmid": str(video_obj['user']['mid']),
                "user_ct": "1638346045009"
            },
            "t": int(time.time() * 1000),
            "ab": {}
        }

    def run(self):
        self.get_recommend_list()


if __name__ == '__main__':
    J = ZFQZTJLRecommend(
        platform="zhufuquanzituijianliu",
        mode="recommend",
        rule_dict={},
        user_list=[{"uid": 75590470, "link": "zfqz推荐流_接口1", "nick_name": "做你的尾巴"}, {"uid": 75590471, "link": "zfqz推荐流_接口2", "nick_name": "能够相遇"}, {"uid": 75590472, "link": "zfqz推荐流_接口3", "nick_name": "一别两宽各生欢喜"}, {"uid": 75590473, "link": "zfqz推荐流_接口4", "nick_name": "惹火"}, {"uid": 75590475, "link": "zfqz推荐流_接口5", "nick_name": "顾九"}, {"uid": 75590476, "link": "zfqz推荐流_接口6", "nick_name": "宠一身脾气惯一身毛病"}],

    )
    J.get_recommend_list()
    # J.logic()