piaoquan_crawler/benshanzhufu/benshanzhufu_recommend/benshanzhufu_recommend.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2023/4/13
# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/4/25
import json
import os
import random
import shutil
import sys
import time
from hashlib import md5
from urllib import parse
import requests
import urllib3
sys.path.append(os.getcwd())
from common.common import Common
from common.scheduling_db import MysqlHelper
from common.feishu import Feishu
from common.publish import Publish
proxies = {"http": None, "https": None}


class BenshanzhufuRecommend:
    # 翻页参数
    visitor_key = ""
    page = 1
    platform = "本山祝福"

    # 过滤词库
    @classmethod
    def benshanzhufu_config(cls, log_type, crawler, text, env):
        select_sql = f"""select * from crawler_config where source="benshanzhufu" """
        contents = MysqlHelper.get_values(log_type, crawler, select_sql, env, action='')
        title_list = []
        filter_list = []
        for content in contents:
            config = content['config']
            config_dict = eval(config)
            for k, v in config_dict.items():
                if k == "title":
                    title_list_config = v.split(",")
                    for title in title_list_config:
                        title_list.append(title)
                if k == "filter":
                    filter_list_config = v.split(",")
                    for filter_word in filter_list_config:
                        filter_list.append(filter_word)
        if text == "title":
            return title_list
        elif text == "filter":
            return filter_list

    @classmethod
    def repeat_video(cls, log_type, crawler, video_id, env):
        sql = f""" select * from crawler_video where platform="本山祝福" and out_video_id="{video_id}"; """
        repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
        return len(repeat_video)

    # 推荐列表获取视频
    @classmethod
    def get_videoList(cls, log_type, crawler, oss_endpoint, env):
        while True:
            now = int(time.time() * 1000)
            url = "https://bszf.wentingyou.cn/index.php/v111/index/index?parameter="
            header = {
                "content-time": str(now),
                # "visitorKey": "165086930003741",
                "chatKey": "wx0fb8149da961d3b0",
                "cache-time": str(now),
                "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) "
                              "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 "
                              "MicroMessenger/8.0.20(0x1800142d) NetType/WIFI Language/zh_CN",
                "Referer": "https://servicewechat.com/wx0fb8149da961d3b0/2/page-frame.html"
            }
            parameter = {
                "page": random.randint(1, 76),
                "ini_id": cls.visitor_key
            }
            params = parse.quote(json.dumps(parameter))
            url = url + str(params)
            # try:
            urllib3.disable_warnings()
            r = requests.get(headers=header, url=url, proxies=proxies, verify=False)
            if r.status_code != 200:
                Common.logger(log_type, crawler).warning(f"get_videoList:{r.status_code}, {r.text}\n")
                cls.visitor_key = ""
                cls.page = 1
                return
            elif r.json()['message'] != "list success":
                Common.logger(log_type, crawler).warning(f"get_videoList:{r.status_code}, {r.json()}\n")
                cls.visitor_key = ""
                cls.page = 1
                return
            elif "data" not in r.json():
                Common.logger(log_type, crawler).warning(f"get_videoList:{r.status_code}, {r.json()}\n")
                cls.visitor_key = ""
                cls.page = 1
                return
            elif len(r.json()['data']["list"]) == 0:
                Common.logger(log_type, crawler).info(f"没有更多数据了~ {r.json()}\n")
                cls.visitor_key = ""
                cls.page = 1
                return
            else:
                # 翻页
                cls.visitor_key = r.json()["data"]["visitor_key"]
                cls.page += 1
                feeds = r.json()["data"]["list"]
                for i in range(len(feeds)):
                    video_title = feeds[i].get("title", "").strip().replace("\n", "")\
                            .replace("/", "").replace("本山祝福", "").replace(" ", "")\
                            .replace(" ", "").replace("&NBSP", "").replace("\r", "")\
                            .replace("#", "").replace(".", "。").replace("\\", "")\
                            .replace(":", "").replace("*", "").replace("？", "")\
                            .replace("?", "").replace('"', "").replace("<", "")\
                            .replace(">", "").replace("|", "").replace("'", "").replace('"', "")
                    video_id = str(feeds[i].get("nid", ""))
                    play_cnt = 0
                    comment_cnt = feeds[i].get("commentCount", 0)
                    share_cnt = 0
                    like_cnt = 0
                    publish_time_stamp = feeds[i].get("update_time", 0)
                    publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp))
                    user_name = "本山祝福"
                    user_id = "benshanzhufu"
                    cover_url = feeds[i].get("video_cover", "")
                    video_url = feeds[i].get("video_url", "")
                    if ".mp4" not in video_url:
                        video_url = ""

                    video_dict = {
                        'video_title': video_title,
                        'video_id': video_id,
                        'play_cnt': play_cnt,
                        'comment_cnt': comment_cnt,
                        'like_cnt': like_cnt,
                        'share_cnt': share_cnt,
                        'publish_time_stamp': publish_time_stamp,
                        'publish_time_str': publish_time_str,
                        'user_name': user_name,
                        'user_id': user_id,
                        'avatar_url': cover_url,
                        'cover_url': cover_url,
                        'video_url': video_url,
                        'session': f"benshanzhufu-{int(time.time())}"
                    }
                    for k, v in video_dict.items():
                        Common.logger(log_type, crawler).info(f"{k}:{v}")

                    # 过滤无效视频
                    if video_id == "" or cover_url == "" or video_url == "":
                        Common.logger(log_type, crawler).info("无效视频\n")
                    elif any(str(word) if str(word) in video_title else False for word in cls.benshanzhufu_config(log_type, crawler, "filter", env)) is True:
                        Common.logger(log_type, crawler).info('已中过滤词\n')
                    elif cls.repeat_video(log_type, crawler, video_id, env) != 0:
                        Common.logger(log_type, crawler).info('视频已下载\n')
                    else:
                        cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
            # except Exception as e:
            #     Common.logger(log_type, crawler).error(f"get_videoList异常:{e}\n")

    # 下载 / 上传
    @classmethod
    def download_publish(cls, log_type, crawler, video_dict, oss_endpoint, env):
        # try:
        # 下载视频
        Common.download_method(log_type=log_type, crawler=crawler, text='video', title=video_dict['video_title'], url=video_dict['video_url'])
        ffmpeg_dict = Common.ffmpeg(log_type, crawler, f"./{crawler}/videos/{video_dict['video_title']}/video.mp4")
        if ffmpeg_dict is None:
            md_title = md5(video_dict['video_title'].encode('utf8')).hexdigest()
            shutil.rmtree(f"./{crawler}/videos/{md_title}/")
            Common.logger(log_type, crawler).info("视频size=0，删除成功\n")
            return
        video_dict["duration"] = ffmpeg_dict["duration"]
        video_dict["video_width"] = ffmpeg_dict["width"]
        video_dict["video_height"] = ffmpeg_dict["height"]

        # 下载封面
        Common.download_method(log_type=log_type, crawler=crawler, text='cover', title=video_dict['video_title'], url=video_dict['cover_url'])
        # 保存视频信息至txt
        Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_dict)

        # 上传视频
        Common.logger(log_type, crawler).info("开始上传视频...")
        our_video_id = Publish.upload_and_publish(log_type=log_type,
                                                  crawler=crawler,
                                                  strategy="推荐榜爬虫策略",
                                                  our_uid="recommend",
                                                  env=env,
                                                  oss_endpoint=oss_endpoint)
        if env == 'dev':
            our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
        else:
            our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
        Common.logger(log_type, crawler).info("视频上传完成")

        if our_video_id is None:
            # 删除视频文件夹
            shutil.rmtree(f"./{crawler}/videos/{video_dict['video_title']}")
            return

        # 视频写入飞书
        Feishu.insert_columns(log_type, crawler, "440018", "ROWS", 1, 2)
        upload_time = int(time.time())
        values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)),
                   "推荐榜爬虫策略",
                   video_dict['video_id'],
                   video_dict['video_title'],
                   our_video_link,
                   video_dict['play_cnt'],
                   video_dict['comment_cnt'],
                   video_dict['like_cnt'],
                   video_dict['share_cnt'],
                   video_dict['duration'],
                   f"{video_dict['video_width']}*{video_dict['video_height']}",
                   video_dict['publish_time_str'],
                   video_dict['user_name'],
                   video_dict['user_id'],
                   video_dict['avatar_url'],
                   video_dict['cover_url'],
                   video_dict['video_url']]]
        time.sleep(0.5)
        Feishu.update_values(log_type, crawler, "440018", "E2:Z2", values)
        Common.logger(log_type, crawler).info(f"视频已保存至云文档\n")

        rule_dict = {}
        # 视频信息保存数据库
        insert_sql = f""" insert into crawler_video(video_id,
                                                out_user_id,
                                                platform,
                                                strategy,
                                                out_video_id,
                                                video_title,
                                                cover_url,
                                                video_url,
                                                duration,
                                                publish_time,
                                                play_cnt,
                                                crawler_rule,
                                                width,
                                                height)
                                                values({our_video_id},
                                                "{video_dict['user_id']}",
                                                "{cls.platform}",
                                                "推荐榜爬虫策略",
                                                "{video_dict['video_id']}",
                                                "{video_dict['video_title']}",
                                                "{video_dict['cover_url']}",
                                                "{video_dict['video_url']}",
                                                {int(video_dict['duration'])},
                                                "{video_dict['publish_time_str']}",
                                                {int(video_dict['play_cnt'])},
                                                '{json.dumps(rule_dict)}',
                                                {int(video_dict['video_width'])},
                                                {int(video_dict['video_height'])}) """
        Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
        MysqlHelper.update_values(log_type, crawler, insert_sql, env, action='')
        Common.logger(log_type, crawler).info('视频信息插入数据库成功！\n')

        # except Exception as e:
        #     Common.logger(log_type, crawler).error(f"download_publish异常:{e}\n")
        #     # 删除视频文件夹
        #     shutil.rmtree(f"./{crawler}/videos/")
        #     return


if __name__ == "__main__":
    BenshanzhufuRecommend.get_videoList("recommend", "benshanzhufu", "out", "dev")

    pass