piaoquan_crawler/xiaoniangao/xiaoniangao_play/xiaoniangao_play.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2023/3/16
import json
import os
import random
import shutil
import sys
import time
import requests
import urllib3

from common.scheduling_db import MysqlHelper

sys.path.append(os.getcwd())
from common.common import Common
from common.feishu import Feishu
from common.publish import Publish
proxies = {"http": None, "https": None}


class XiaoniangaoPlay:
    platform = "小年糕"

    # 生成 uid、token
    @classmethod
    def get_uid_token(cls):
        words = "abcdefghijklmnopqrstuvwxyz0123456789"
        uid = f"""{"".join(random.sample(words, 8))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 12))}"""
        token = "".join(random.sample(words, 32))
        uid_token_dict = {
            "uid": uid,
            "token": token
        }
        return uid_token_dict

    # 过滤敏感词
    @classmethod
    def filter_words(cls, log_type, crawler):
        try:
            while True:
                # 敏感词库列表
                word_list = []
                # 从云文档读取所有敏感词，添加到词库列表
                filter_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "DRAnZh")
                if filter_sheet is None:
                    Common.logger(log_type, crawler).info(f"filter_sheet:{filter_sheet}")
                    continue
                for i in filter_sheet:
                    for j in i:
                        # 过滤空的单元格内容
                        if j is None:
                            pass
                        else:
                            word_list.append(j)
                return word_list
        except Exception as e:
            Common.logger(log_type, crawler).error(f"filter_words:{e}\n")

    # 基础门槛规则
    @classmethod
    def download_rule(cls, video_dict):
        """
        下载视频的基本规则
        :param video_dict: 视频信息，字典格式
        :return: 满足规则，返回 True；反之，返回 False
        """
        # 视频时长
        if int(float(video_dict['duration'])) >= 40:
            # 宽或高
            if int(video_dict['video_width']) >= 0 or int(video_dict['video_height']) >= 0:
                # 播放量
                if int(video_dict['play_cnt']) >= 80000:
                    # 点赞量
                    if int(video_dict['like_cnt']) >= 0:
                        # 分享量
                        if int(video_dict['share_cnt']) >= 0:
                            # 发布时间 <= 60 天
                            if int(time.time()) - int(video_dict['publish_time_stamp']) <= 3600 * 24 * 60:
                                return True
                            else:
                                return False
                        else:
                            return False
                    else:
                        return False
                else:
                    return False
            return False
        return False

    # 获取表情及符号
    @classmethod
    def get_expression(cls):
        while True:
            expression_list = []
            char_list = []
            char_sheet = Feishu.get_values_batch("hour", "xiaoniangao", "BhlbST")
            if char_sheet is None:
                continue
            for i in range(len(char_sheet)):
                if char_sheet[i][0] is not None:
                    expression_list.append(char_sheet[i][0])
                if char_sheet[i][1] is not None:
                    char_list.append(char_sheet[i][1])
            return expression_list, char_list

    # 获取列表
    @classmethod
    def get_videoList(cls, log_type, crawler, strategy, oss_endpoint, env):
        uid_token_dict = cls.get_uid_token()
        url = "https://kapi.xiaoniangao.cn/trends/get_recommend_trends"
        headers = {
            # "x-b3-traceid": cls.play_x_b3_traceid,
            "x-b3-traceid": '1dc0a6d0929a2b',
            # "X-Token-Id": cls.play_x_token_id,
            "X-Token-Id": 'ae99a4953804085ebb0ae36fa138031d-1146052582',
            # "uid": cls.play_uid,
            "uid": uid_token_dict['uid'],
            "content-type": "application/json",
            "Accept-Encoding": "gzip,compress,br,deflate",
            "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
                          ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
                          'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
            # "Referer": cls.play_referer
            "Referer": 'https://servicewechat.com/wxd7911e4c177690e4/620/page-frame.html'
        }
        data = {
            "log_params": {
                "page": "discover_rec",
                "common": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.2",
                    "net": "wifi",
                    "scene": 1089
                }
            },
            "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!750x500r/crop/750x500/interlace/1/format/jpg",
            "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!80x80r/crop/80x80/interlace/1/format/jpg",
            "share_width": 625,
            "share_height": 500,
            "ext": {
                "fmid": 0,
                "items": {}
            },
            "app": "xng",
            "rec_scene": "discover_rec",
            "log_common_params": {
                "e": [{
                    "data": {
                        "page": "discoverIndexPage",
                        "topic": "recommend"
                    },
                    "ab": {}
                }],
                "ext": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.3",
                    "net": "wifi",
                    "scene": "1089"
                },
                "pj": "1",
                "pf": "2",
                "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29"
            },
            "refresh": False,
            # "token": cls.play_token,
            "token": uid_token_dict['token'],
            # "uid": cls.play_uid,
            "uid": uid_token_dict['uid'],
            "proj": "ma",
            "wx_ver": "8.0.20",
            "code_ver": "3.62.0"
        }
        try:
            urllib3.disable_warnings()
            r = requests.post(url=url, headers=headers, json=data, proxies=proxies, verify=False)
            if "data" not in r.text or r.status_code != 200:
                Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}")
            elif "data" not in r.json():
                Common.logger(log_type, crawler).info(f"get_videoList:{r.json()}")
            elif "list" not in r.json()["data"]:
                Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()['data']}")
            else:
                # 视频列表数据
                feeds = r.json()["data"]["list"]
                for i in range(len(feeds)):
                    # 标题，表情随机加在片头、片尾，或替代句子中间的标点符号
                    if "title" in feeds[i]:
                        befor_video_title = feeds[i]["title"].strip().replace("\n", "") \
                            .replace("/", "").replace("\r", "").replace("#", "") \
                            .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
                            .replace(":", "").replace("*", "").replace("？", "") \
                            .replace("?", "").replace('"', "").replace("<", "") \
                            .replace(">", "").replace("|", "").replace(" ", "").replace("#表情", "").replace("#符号", "")

                        expression = cls.get_expression()
                        expression_list = expression[0]
                        char_list = expression[1]
                        # 随机取一个表情
                        expression = random.choice(expression_list)
                        # 生成标题list[表情+title, title+表情]
                        expression_title_list = [expression + befor_video_title, befor_video_title + expression]
                        # 从标题list中随机取一个标题
                        title_list1 = random.choice(expression_title_list)
                        # 生成标题:原标题+符号
                        title_list2 = befor_video_title + random.choice(char_list)
                        # 表情和标题组合，与标题和符号组合，汇总成待使用的标题列表
                        title_list4 = [title_list2, title_list1]
                        # 最终标题
                        video_title = random.choice(title_list4)
                    else:
                        video_title = 0

                    # 视频 ID
                    if "vid" in feeds[i]:
                        video_id = feeds[i]["vid"]
                    else:
                        video_id = 0

                    # 播放量
                    if "play_pv" in feeds[i]:
                        video_play_cnt = feeds[i]["play_pv"]
                    else:
                        video_play_cnt = 0

                    # 评论量
                    if "comment_count" in feeds[i]:
                        video_comment_cnt = feeds[i]["comment_count"]
                    else:
                        video_comment_cnt = 0

                    # 点赞量
                    if "favor" in feeds[i]:
                        video_like_cnt = feeds[i]["favor"]["total"]
                    else:
                        video_like_cnt = 0

                    # 分享量
                    if "share" in feeds[i]:
                        video_share_cnt = feeds[i]["share"]
                    else:
                        video_share_cnt = 0

                    # 时长
                    if "du" in feeds[i]:
                        video_duration = int(feeds[i]["du"] / 1000)
                    else:
                        video_duration = 0

                    # 宽和高
                    if "w" or "h" in feeds[i]:
                        video_width = feeds[i]["w"]
                        video_height = feeds[i]["h"]
                    else:
                        video_width = 0
                        video_height = 0

                    # 发布时间
                    if "t" in feeds[i]:
                        video_send_time = feeds[i]["t"]
                    else:
                        video_send_time = 0
                    publish_time_stamp = int(int(video_send_time)/1000)
                    publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp))

                    # 用户名 / 头像
                    if "user" in feeds[i]:
                        user_name = feeds[i]["user"]["nick"].strip().replace("\n", "") \
                            .replace("/", "").replace("快手", "").replace(" ", "") \
                            .replace(" ", "").replace("&NBSP", "").replace("\r", "")
                        head_url = feeds[i]["user"]["hurl"]
                    else:
                        user_name = 0
                        head_url = 0

                    # 用户 ID
                    profile_id = feeds[i]["id"]

                    # 用户 mid
                    profile_mid = feeds[i]["user"]["mid"]

                    # 视频封面
                    if "url" in feeds[i]:
                        cover_url = feeds[i]["url"]
                    else:
                        cover_url = 0

                    # 视频播放地址
                    if "v_url" in feeds[i]:
                        video_url = feeds[i]["v_url"]
                    else:
                        video_url = 0

                    video_dict = {
                        "video_title": video_title,
                        "video_id": video_id,
                        "duration": video_duration,
                        "play_cnt": video_play_cnt,
                        "like_cnt": video_like_cnt,
                        "comment_cnt": video_comment_cnt,
                        "share_cnt": video_share_cnt,
                        "user_name": user_name,
                        "publish_time_stamp": publish_time_stamp,
                        "publish_time_str": publish_time_str,
                        "video_width": video_width,
                        "video_height": video_height,
                        "avatar_url": head_url,
                        "profile_id": profile_id,
                        "profile_mid": profile_mid,
                        "cover_url": cover_url,
                        "video_url": video_url,
                        "session": f"xiaoniangao-play-{int(time.time())}"

                    }

                    cls.download_publish(log_type=log_type,
                                         crawler=crawler,
                                         video_dict=video_dict,
                                         strategy=strategy,
                                         oss_endpoint=oss_endpoint,
                                         env=env)

        except Exception as e:
            Common.logger(log_type, crawler).error("get_play_feeds异常:{}", e)

    @classmethod
    def repeat_video(cls, log_type, crawler, video_id, env):
        sql = f""" select * from crawler_video where platform="小年糕" and out_video_id="{video_id}"; """
        repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
        return len(repeat_video)

    @classmethod
    def download_publish(cls, log_type, crawler, video_dict, strategy, oss_endpoint, env):
        # try:
        # 过滤无效视频
        if video_dict["video_id"] == 0 \
                or video_dict["video_url"] == 0\
                or video_dict["cover_url"] == 0:
            Common.logger(log_type, crawler).warning("无效视频\n")
        # 抓取规则
        elif cls.download_rule(video_dict) is False:
            Common.logger(log_type, crawler).info("不满足抓取规则")
        # 去重
        elif cls.repeat_video(log_type, crawler, video_dict['video_id'], env) != 0:
            Common.logger(log_type, crawler).info("视频已下载\n")
        # 过滤词库
        elif any(str(word) if str(word) in video_dict['video_title'] else False for word in cls.filter_words(log_type, crawler)) is True:
            Common.logger(log_type, crawler).info("视频已中过滤词\n")
        else:
            # 下载封面
            Common.download_method(log_type=log_type, crawler=crawler, text="cover", title=video_dict["video_title"], url=video_dict["cover_url"])
            # 下载视频
            Common.download_method(log_type=log_type, crawler=crawler, text="video", title=video_dict["video_title"], url=video_dict["video_url"])
            # 保存视频信息至 "./videos/{download_video_title}/info.txt"
            Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_dict)

            # 上传视频
            Common.logger(log_type, crawler).info("开始上传视频...")
            our_video_id = Publish.upload_and_publish(log_type=log_type,
                                                      crawler=crawler,
                                                      strategy=strategy,
                                                      our_uid="play",
                                                      env=env,
                                                      oss_endpoint=oss_endpoint)
            if env == "dev":
                our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
            else:
                our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
            Common.logger(log_type, crawler).info("视频上传完成")

            if our_video_id is None:
                # 删除视频文件夹
                shutil.rmtree(f"./{crawler}/videos/{video_dict['video_title']}")
                return

            # 视频信息保存数据库
            rule_dict = {
                "duration": {"min": 40},
                "play_cnt": {"min": 80000},
                "min_publish_day": {"min": 60}
            }

            insert_sql = f""" insert into crawler_video(video_id,
                                                        out_user_id,
                                                        platform,
                                                        strategy,
                                                        out_video_id,
                                                        video_title,
                                                        cover_url,
                                                        video_url,
                                                        duration,
                                                        publish_time,
                                                        play_cnt,
                                                        crawler_rule,
                                                        width,
                                                        height)
                                                        values({our_video_id},
                                                        "{video_dict['profile_id']}",
                                                        "{cls.platform}",
                                                        "播放量榜爬虫策略",
                                                        "{video_dict['video_id']}",
                                                        "{video_dict['video_title']}",
                                                        "{video_dict['cover_url']}",
                                                        "{video_dict['video_url']}",
                                                        {int(video_dict['duration'])},
                                                        "{video_dict['publish_time_str']}",
                                                        {int(video_dict['play_cnt'])},
                                                        '{json.dumps(rule_dict)}',
                                                        {int(video_dict['video_width'])},
                                                        {int(video_dict['video_height'])}) """
            Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
            MysqlHelper.update_values(log_type, crawler, insert_sql, env)
            Common.logger(log_type, crawler).info('视频信息插入数据库成功！')

            # 视频写入飞书
            Feishu.insert_columns(log_type, crawler, "c85k1C", "ROWS", 1, 2)
            # 视频ID工作表，首行写入数据
            upload_time = int(time.time())
            values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)),
                       "播放量榜爬虫策略",
                       str(video_dict['video_id']),
                       str(video_dict['video_title']),
                       our_video_link,
                       video_dict['play_cnt'],
                       video_dict['comment_cnt'],
                       video_dict['like_cnt'],
                       video_dict['share_cnt'],
                       video_dict['duration'],
                       f"{video_dict['video_width']}*{video_dict['video_height']}",
                       str(video_dict['publish_time_str']),
                       str(video_dict['user_name']),
                       str(video_dict['profile_id']),
                       str(video_dict['profile_mid']),
                       str(video_dict['avatar_url']),
                       str(video_dict['cover_url']),
                       str(video_dict['video_url'])]]
            time.sleep(1)
            Feishu.update_values(log_type, crawler, "c85k1C", "F2:Z2", values)
            Common.logger(log_type, crawler).info('视频信息写入飞书成功\n')

        # except Exception as e:
        #     Common.logger(log_type, crawler).error('download_publish异常:{}', e)


if __name__ == '__main__':
    XiaoniangaoPlay.get_videoList("play", "xiaoniangao", "播放量榜爬虫策略", "out", "dev")

    pass