crawler_xiaoniangao/main/xiaoniangao_play_list.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/9/7
import os
import random
import sys
import time
import requests
import urllib3

sys.path.append(os.getcwd())
from main.common import Common
from main.feishu_lib import Feishu
from main.publish import Publish

proxies = {"http": None, "https": None}


class PlayList:
    # 配置微信
    time.sleep(1)
    wechat_sheet = Feishu.get_values_batch("play", "xiaoniangao", "dzcWHw")
    play_x_b3_traceid = wechat_sheet[2][2]
    play_x_token_id = wechat_sheet[3][2]
    play_referer = wechat_sheet[4][2]
    play_uid = wechat_sheet[5][2]
    play_token = wechat_sheet[6][2]

    # 生成 uid、token
    @classmethod
    def get_uid_token(cls, log_type):
        uid_token = "abcdefghijklmnopqrstuvwxyz0123456789"
        uid = ("".join(random.sample(uid_token, 8))) + "-" + ("".join(random.sample(uid_token, 4))) + "-" + (
            "".join(random.sample(uid_token, 4))) + "-" + ("".join(random.sample(uid_token, 4))) + "-" + (
                  "".join(random.sample(uid_token, 12)))
        token = ("".join(random.sample(uid_token, 32)))
        # print(f'uid:{uid}\ntoken:{token}\n')
        Feishu.update_values(log_type, "xiaoniangao", "dzcWHw", "C6:C6", [[uid]])
        time.sleep(1)
        Feishu.update_values(log_type, "xiaoniangao", "dzcWHw", "C7:C7", [[token]])
        Common.logger(log_type).info("生成 uid:{}， token:{}，写入飞书成功\n", uid, token)

    # 过滤敏感词
    @classmethod
    def sensitive_words(cls, log_type):
        # 敏感词库列表
        word_list = []
        # 从云文档读取所有敏感词，添加到词库列表
        time.sleep(1)
        lists = Feishu.get_values_batch(log_type, "xiaoniangao", "DRAnZh")
        for i in lists:
            for j in i:
                # 过滤空的单元格内容
                if j is None:
                    pass
                else:
                    word_list.append(j)
        return word_list

    # 基础门槛规则
    @staticmethod
    def download_rule(d_duration, d_width, d_height, d_play_cnt, d_like_cnt, d_share_cnt, d_send_time):
        """
        下载视频的基本规则
        :param d_duration: 时长
        :param d_width: 宽
        :param d_height: 高
        :param d_play_cnt: 播放量
        :param d_like_cnt: 点赞量
        :param d_share_cnt: 分享量
        :param d_send_time: 发布时间
        :return: 满足规则，返回 True；反之，返回 False
        """
        # 视频时长
        if int(float(d_duration)) >= 40:
            # 宽或高
            if int(d_width) >= 0 or int(d_height) >= 0:
                # 播放量
                if int(d_play_cnt) >= 4000:
                    # 点赞量
                    if int(d_like_cnt) >= 0:
                        # 分享量
                        if int(d_share_cnt) >= 0:
                            # 发布时间 <= 30 天
                            if int(time.time()) - int(d_send_time) / 1000 <= 3600 * 24 * 30:
                                return True
                            else:
                                return False
                        else:
                            return False
                    else:
                        return False
                else:
                    return False
            return False
        return False

    # 获取表情及符号
    @classmethod
    def get_expression(cls, log_type):
        expression_list = []
        char_list = []
        char_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "BhlbST")
        for i in range(len(char_sheet)):
            if char_sheet[i][0] is not None:
                expression_list.append(char_sheet[i][0])
            if char_sheet[i][1] is not None:
                char_list.append(char_sheet[i][1])

        # print(f"expression_list:{expression_list}")
        # print(f"char_list:{char_list}")

        return expression_list, char_list

    # 获取列表
    @classmethod
    def get_play_feeds(cls, log_type):
        """
        1.从列表获取视频，7 天内，播放量>=100000
        2.时长 >=40s
        """
        url = "https://kapi.xiaoniangao.cn/trends/get_recommend_trends"
        headers = {
            "x-b3-traceid": cls.play_x_b3_traceid,
            "X-Token-Id": cls.play_x_token_id,
            "uid": cls.play_uid,
            "content-type": "application/json",
            "Accept-Encoding": "gzip,compress,br,deflate",
            "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
                          ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
                          'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
            "Referer": cls.play_referer
        }
        data = {
            "log_params": {
                "page": "discover_rec",
                "common": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.2",
                    "net": "wifi",
                    "scene": 1089
                }
            },
            "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!750x500r/crop/750x500/interlace/1/format/jpg",
            "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!80x80r/crop/80x80/interlace/1/format/jpg",
            "share_width": 625,
            "share_height": 500,
            "ext": {
                "fmid": 0,
                "items": {}
            },
            "app": "xng",
            "rec_scene": "discover_rec",
            "log_common_params": {
                "e": [{
                    "data": {
                        "page": "discoverIndexPage",
                        "topic": "recommend"
                    },
                    "ab": {}
                }],
                "ext": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.3",
                    "net": "wifi",
                    "scene": "1089"
                },
                "pj": "1",
                "pf": "2",
                "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29"
            },
            "refresh": False,
            "token": cls.play_token,
            "uid": cls.play_uid,
            "proj": "ma",
            "wx_ver": "8.0.20",
            "code_ver": "3.62.0"
        }
        try:
            urllib3.disable_warnings()
            r = requests.post(url=url, headers=headers, json=data, proxies=proxies, verify=False)
            if "data" not in r.json():
                Common.logger(log_type).warning("获取视频feeds错误:{}", r.text)
            elif "list" not in r.json()["data"]:
                Common.logger(log_type).warning("获取视频feeds无数据，休眠10s:{}", r.json()["data"])
            else:
                # 视频列表数据
                feeds = r.json()["data"]["list"]
                for i in range(len(feeds)):
                    # 标题，表情随机加在片头、片尾，或替代句子中间的标点符号
                    if "title" in feeds[i]:
                        befor_video_title = feeds[i]["title"].strip().replace("\n", "") \
                            .replace("/", "").replace("\r", "").replace("#", "") \
                            .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
                            .replace(":", "").replace("*", "").replace("？", "") \
                            .replace("?", "").replace('"', "").replace("<", "") \
                            .replace(">", "").replace("|", "").replace(" ", "").replace("#表情", "").replace("#符号",
                                                                                                             "")

                        expression = cls.get_expression(log_type)
                        expression_list = expression[0]
                        char_list = expression[1]
                        # print(f"expression_list:{expression_list}")
                        # print(f"char_list:{char_list}")
                        # 随机取一个表情
                        expression = random.choice(expression_list)
                        # print(f"expression:{expression}")
                        # 生成标题list[表情+title, title+表情]
                        expression_title_list = [expression + befor_video_title, befor_video_title + expression]
                        # print(f"expression_title_list:{expression_title_list}")
                        # 从标题list中随机取一个标题
                        title_list1 = random.choice(expression_title_list)
                        # print(f"title_list1:{title_list1}")
                        # 生成标题:原标题+符号
                        title_list2 = befor_video_title + random.choice(char_list)
                        # print(f"befor_video_title:{befor_video_title}")

                        # title_list3 = befor_video_title.replace(
                        #     ",", random.choice(expression_list)).replace("，", random.choice(expression_list))
                        # 表情和标题组合，与标题和符号组合，汇总成待使用的标题列表
                        title_list4 = [title_list2, title_list1]
                        # print(f"title_list4:{title_list4}")
                        # 最终标题
                        video_title = random.choice(title_list4)
                        # print(f"video_title:{video_title}\n")
                    else:
                        video_title = 0

                    # 视频 ID
                    if "vid" in feeds[i]:
                        video_id = feeds[i]["vid"]
                    else:
                        video_id = 0

                    # 播放量
                    if "play_pv" in feeds[i]:
                        video_play_cnt = feeds[i]["play_pv"]
                    else:
                        video_play_cnt = 0

                    # 评论量
                    if "comment_count" in feeds[i]:
                        video_comment_cnt = feeds[i]["comment_count"]
                    else:
                        video_comment_cnt = 0

                    # 点赞量
                    if "favor" in feeds[i]:
                        video_like_cnt = feeds[i]["favor"]["total"]
                    else:
                        video_like_cnt = 0

                    # 分享量
                    if "share" in feeds[i]:
                        video_share_cnt = feeds[i]["share"]
                    else:
                        video_share_cnt = 0

                    # 时长
                    if "du" in feeds[i]:
                        video_duration = int(feeds[i]["du"] / 1000)
                    else:
                        video_duration = 0

                    # 宽和高
                    if "w" or "h" in feeds[i]:
                        video_width = feeds[i]["w"]
                        video_height = feeds[i]["h"]
                    else:
                        video_width = 0
                        video_height = 0

                    # 发布时间
                    if "t" in feeds[i]:
                        video_send_time = feeds[i]["t"]
                    else:
                        video_send_time = 0

                    # 用户名 / 头像
                    if "user" in feeds[i]:
                        user_name = feeds[i]["user"]["nick"].strip().replace("\n", "") \
                            .replace("/", "").replace("快手", "").replace(" ", "") \
                            .replace(" ", "").replace("&NBSP", "").replace("\r", "")
                        head_url = feeds[i]["user"]["hurl"]
                    else:
                        user_name = 0
                        head_url = 0

                    # 用户 ID
                    profile_id = feeds[i]["id"]

                    # 用户 mid
                    profile_mid = feeds[i]["user"]["mid"]

                    # 视频封面
                    if "url" in feeds[i]:
                        cover_url = feeds[i]["url"]
                    else:
                        cover_url = 0

                    # 视频播放地址
                    if "v_url" in feeds[i]:
                        video_url = feeds[i]["v_url"]
                    else:
                        video_url = 0

                    Common.logger(log_type).info("标题:{}", video_title)
                    Common.logger(log_type).info("视频ID:{}", video_id)
                    Common.logger(log_type).info("播放量:{}", video_play_cnt)
                    Common.logger(log_type).info("时长:{}秒", video_duration)
                    Common.logger(log_type).info(
                        "视频发布时间:{}", time.strftime(
                            "%Y/%m/%d %H:%M:%S", time.localtime(int(video_send_time) / 1000)))
                    Common.logger(log_type).info("播放地址:{}", video_url)
                    # Common.logger(log_type).info("点赞量:{}", video_like_cnt)
                    # Common.logger(log_type).info("分享量:{}", video_share_cnt)
                    # Common.logger(log_type).info("评论数:{}", video_comment_cnt)
                    # Common.logger(log_type).info("宽高:{}*{}", video_width, video_height)
                    # Common.logger(log_type).info("用户名:{}", user_name)
                    # Common.logger(log_type).info("用户头像:{}", head_url)
                    # Common.logger(log_type).info("封面:{}", cover_url)

                    # 过滤无效视频
                    if video_title == 0 or video_id == 0 or video_duration == 0 \
                            or video_send_time == 0 or user_name == 0 or head_url == 0 \
                            or cover_url == 0 or video_url == 0:
                        Common.logger(log_type).warning("无效视频\n")

                    elif cls.download_rule(video_duration, video_width, video_height, video_play_cnt,
                                           video_like_cnt, video_share_cnt, video_send_time) is False:
                        Common.logger(log_type).info("不满足基础门槛规则\n")

                    # 过滤敏感词
                    elif any(word if word in video_title else False for word in cls.sensitive_words(log_type)) is True:
                        Common.logger(log_type).info("视频已中敏感词:{}\n".format(video_title))
                        time.sleep(1)

                    # 从已下载云文档中去重
                    elif str(video_id) in [j for i in Feishu.get_values_batch(log_type, "xiaoniangao", "yatRv2") for j
                                           in i]:
                        Common.logger(log_type).info("该视频已下载:{}\n", video_title)
                        time.sleep(1)

                    # 从已下载云文档中去重
                    elif str(video_id) in [j for i in Feishu.get_values_batch(log_type, "xiaoniangao", "2zD8cv") for j
                                           in i]:
                        Common.logger(log_type).info("该视频已下载:{}\n", video_title)
                        time.sleep(1)

                    # 从feeds云文档去重
                    elif str(video_id) in [j for i in Feishu.get_values_batch(log_type, "xiaoniangao", "me9Qs4") for j
                                           in i]:
                        Common.logger(log_type).info("该视频已保存过:{}\n", video_title)
                        time.sleep(1)
                    else:
                        Feishu.insert_columns(log_type, "xiaoniangao", "me9Qs4", "ROWS", 1, 2)

                        # 获取当前时间
                        get_feeds_time = int(time.time())
                        values = [[time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(get_feeds_time)),
                                   '播放量榜',
                                   str(video_id),
                                   video_title,
                                   video_play_cnt,
                                   video_comment_cnt,
                                   video_like_cnt,
                                   video_share_cnt,
                                   video_duration,
                                   str(video_width) + '*' + str(video_height),
                                   time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(int(video_send_time) / 1000)),
                                   user_name,
                                   profile_id,
                                   profile_mid,
                                   head_url,
                                   cover_url,
                                   video_url]]
                        # 等待 1s，防止操作云文档太频繁，导致报错
                        time.sleep(1)
                        Feishu.update_values(log_type, "xiaoniangao", "me9Qs4", "F2:Z2", values)
                        Common.logger(log_type).info("视频未下载，添加至feeds成功:{}\n".format(video_title))

        except Exception as e:
            Common.logger(log_type).error("get_play_feeds异常:{}", e)

    @classmethod
    def download_publish(cls, log_type, env):
        try:
            play_feeds_sheet = Feishu.get_values_batch(log_type, 'xiaoniangao', 'me9Qs4')
            for i in range(1, len(play_feeds_sheet)):
                download_video_id = play_feeds_sheet[i][7]
                download_video_title = play_feeds_sheet[i][8]
                download_video_play_cnt = play_feeds_sheet[i][9]
                download_video_comment_cnt = play_feeds_sheet[i][10]
                download_video_like_cnt = play_feeds_sheet[i][11]
                download_video_share_cnt = play_feeds_sheet[i][12]
                download_video_duration = play_feeds_sheet[i][13]
                download_video_resolution = play_feeds_sheet[i][14]
                download_video_send_time = play_feeds_sheet[i][15]
                download_user_name = play_feeds_sheet[i][16]
                download_user_id = play_feeds_sheet[i][17]
                download_user_mid = play_feeds_sheet[i][18]
                download_head_url = play_feeds_sheet[i][19]
                download_cover_url = play_feeds_sheet[i][20]
                download_video_url = play_feeds_sheet[i][21]

                Common.logger(log_type).info("正在判断第{}行，视频:{}", i + 1, download_video_title)

                # 过滤空行及空标题视频
                if download_video_id is None \
                        or download_video_id == "" \
                        or download_video_title is None \
                        or download_video_title == "":
                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range(log_type, "xiaoniangao", "me9Qs4", "ROWS", i + 1, i + 1)
                    Common.logger(log_type).warning("标题为空或空行，删除成功\n")
                    return
                # 播放量不小于 100000
                elif int(download_video_play_cnt) < 100000:
                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range(log_type, "xiaoniangao", "me9Qs4", "ROWS", i + 1, i + 1)
                    Common.logger(log_type).info("播放量:{}<100000，删除成功\n", download_video_play_cnt)
                    return
                # 从已下载视频表中去重
                elif str(download_video_id) in [j for m in Feishu.get_values_batch(
                        log_type, "xiaoniangao", "yatRv2") for j in m]:
                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range(log_type, "xiaoniangao", "me9Qs4", "ROWS", i + 1, i + 1)
                    Common.logger(log_type).info("视频已下载:{}，删除成功\n", download_video_title)
                    return

                # 从已下载视频表中去重
                elif str(download_video_id) in [j for m in Feishu.get_values_batch(
                        log_type, "xiaoniangao", "2zD8cv") for j in m]:
                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range(log_type, "xiaoniangao", "me9Qs4", "ROWS", i + 1, i + 1)
                    Common.logger(log_type).info("视频已下载:{}，删除成功\n", download_video_title)
                    return
                else:
                    # 下载封面
                    Common.download_method(log_type=log_type, text="cover",
                                           d_name=str(download_video_title), d_url=str(download_cover_url))
                    # 下载视频
                    Common.download_method(log_type=log_type, text="video",
                                           d_name=str(download_video_title), d_url=str(download_video_url))
                    # 保存视频信息至 "./videos/{download_video_title}/info.txt"
                    with open("./videos/" + download_video_title + "/" + "info.txt",
                              "a", encoding="UTF-8") as f_a:
                        f_a.write(str(download_video_id) + "\n" +
                                  str(download_video_title) + "\n" +
                                  str(download_video_duration) + "\n" +
                                  str(download_video_play_cnt) + "\n" +
                                  str(download_video_comment_cnt) + "\n" +
                                  str(download_video_like_cnt) + "\n" +
                                  str(download_video_share_cnt) + "\n" +
                                  str(download_video_resolution) + "\n" +
                                  str(int(time.mktime(
                                      time.strptime(download_video_send_time, "%Y/%m/%d %H:%M:%S")))) + "\n" +
                                  str(download_user_name) + "\n" +
                                  str(download_head_url) + "\n" +
                                  str(download_video_url) + "\n" +
                                  str(download_cover_url) + "\n" +
                                  "xiaoniangao_play")
                    Common.logger(log_type).info("==========视频信息已保存至info.txt==========")

                    # 上传视频
                    Common.logger(log_type).info("开始上传视频:{}".format(download_video_title))
                    our_video_id = Publish.upload_and_publish(log_type, env, "play")
                    our_video_link = "https://admin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
                    Common.logger(log_type).info("视频上传完成:{}", download_video_title)

                    # 保存视频 ID 到云文档:https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=yatRv2
                    Common.logger(log_type).info("保存视频ID至云文档:{}", download_video_title)
                    # 视频ID工作表，插入首行
                    Feishu.insert_columns(log_type, "xiaoniangao", "yatRv2", "ROWS", 1, 2)
                    # 视频ID工作表，首行写入数据
                    upload_time = int(time.time())
                    values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
                               "播放量榜",
                               str(download_video_id),
                               str(download_video_title),
                               our_video_link,
                               download_video_play_cnt,
                               download_video_comment_cnt,
                               download_video_like_cnt,
                               download_video_share_cnt,
                               download_video_duration,
                               str(download_video_resolution),
                               str(download_video_send_time),
                               str(download_user_name),
                               str(download_user_id),
                               str(download_user_mid),
                               str(download_head_url),
                               str(download_cover_url),
                               str(download_video_url)]]
                    time.sleep(1)
                    Feishu.update_values(log_type, "xiaoniangao", "yatRv2", "F2:Z2", values)

                    # 保存视频信息到监控表
                    Common.logger(log_type).info("添加视频到监控表:{}", download_video_title)
                    # 插入空行
                    time.sleep(1)
                    Feishu.insert_columns(log_type, "monitor", "N7e2yI", "ROWS", 1, 2)
                    # 视频信息写入监控表
                    values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(int(upload_time))),
                               str(download_video_id),
                               download_video_title,
                               our_video_link,
                               download_user_id,
                               download_user_mid,
                               download_user_name,
                               download_video_duration,
                               str(download_video_send_time),
                               download_video_play_cnt]]
                    time.sleep(1)
                    Feishu.update_values(log_type, "monitor", "N7e2yI", "F2:O2", values)

                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range(log_type, "xiaoniangao", "me9Qs4", "ROWS", i + 1, i + 1)
                    Common.logger(log_type).info("视频:{}，下载/上传成功\n", download_video_title)
                    return

        except Exception as e:
            Feishu.dimension_range(log_type, "xiaoniangao", "me9Qs4", "ROWS", 2, 2)
            Common.logger(log_type).error('download_publish异常:{}', e)

    @classmethod
    def run_download_publish(cls, log_type, env):
        try:
            while True:
                if len(Feishu.get_values_batch(log_type, 'xiaoniangao', 'me9Qs4')) == 1:
                    Common.logger(log_type).info('上传/下载完成\n')
                    break
                else:
                    cls.download_publish(log_type, env)

        except Exception as e:
            Common.logger(log_type).error('run_download_publish异常:{}', e)


if __name__ == '__main__':
    # PlayList.get_uid_token('play')
    # print(PlayList.sensitive_words('play'))
    # PlayList.get_expression('play')
    PlayList.get_play_feeds('play')
    #
    # print((int(time.time())-1661854250) / (3600*24))