crawler_weishi/main/recommend.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/4/8
import json
import os
import sys
import time
import requests
import urllib3
sys.path.append(os.getcwd())
from main.common import Common
from main.feishu_lib import Feishu
from main.publish import Publish
proxies = {"http": None, "https": None}


class DownloadRecommend:

    # 配置微信号
    Referer = Feishu.get_range_value("recommend", "9fTK1f", "C3:C3")[0]
    wesee_openid = Feishu.get_range_value("recommend", "9fTK1f", "C4:C4")[0]
    wesee_openkey = Feishu.get_range_value("recommend", "9fTK1f", "C5:C5")[0]
    wesee_personid = Feishu.get_range_value("recommend", "9fTK1f", "C6:C6")[0]
    wesee_access_token = Feishu.get_range_value("recommend", "9fTK1f", "C7:C7")[0]
    wesee_thr_appid = Feishu.get_range_value("recommend", "9fTK1f", "C8:C8")[0]

    # 过滤词库
    @classmethod
    def sensitive_words(cls):
        # 敏感词库列表
        word_list = []
        # 从云文档读取所有敏感词，添加到词库列表
        lists = Feishu.get_values_batch("recommend", "2Oxf8C")
        for a in lists:
            for j in a:
                # 过滤空的单元格内容
                if j is None:
                    pass
                else:
                    word_list.append(j)
        return word_list

    # 抓取基础规则
    @staticmethod
    def download_rule(d_duration, d_width, d_height, d_play_cnt, d_like_cnt, d_share_cnt):
        """
        下载视频的基本规则
        :param d_duration: 时长
        :param d_width: 宽
        :param d_height: 高
        :param d_play_cnt: 播放量
        :param d_like_cnt: 点赞量
        :param d_share_cnt: 分享量
        :return: 满足规则，返回 True；反之，返回 False
        """
        if int(float(d_duration)) >= 30:
            if int(d_width) >= 720 or int(d_height) >= 720:
                if int(d_play_cnt) >= 0:
                    if int(d_like_cnt) >= 0:
                        if int(d_share_cnt) >= 0:
                            return True
                        else:
                            return False
                    else:
                        return False
                else:
                    return False
            return False
        return False

    # 抓取列表
    @classmethod
    def get_feeds(cls):
        """
        1.从微视小程序首页推荐，获取视频列表
        2.先在 https://w42nne6hzg.feishu.cn/sheets/shtcn5YSWg91JfVGzj0SFZIRRPh?sheet=caa3fa 中去重
        3.再从 https://w42nne6hzg.feishu.cn/sheets/shtcn5YSWg91JfVGzj0SFZIRRPh?sheet=O7fCzr 中去重
        4.添加视频信息至 https://w42nne6hzg.feishu.cn/sheets/shtcn5YSWg91JfVGzj0SFZIRRPh?sheet=O7fCzr
        """
        url = "https://api.weishi.qq.com/trpc.weishi.weishi_h5_proxy.weishi_h5_proxy/WxminiGetFeedList"
        headers = {
            "content-type": "application/json",
            "Accept-Encoding": "gzip,compress,br,deflate",
            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)"
                          " AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"
                          " MicroMessenger/8.0.20(0x18001442) NetType/WIFI Language/zh_CN",
            "Referer": str(cls.Referer)
        }
        cookies = {
            "wesee_authtype": "3",
            "wesee_openid": str(cls.wesee_openid),
            "wesee_openkey": str(cls.wesee_openkey),
            "wesee_personid": str(cls.wesee_personid),
            "wesee_refresh_token": "",
            "wesee_access_token": str(cls.wesee_access_token),
            "wesee_thr_appid": str(cls.wesee_thr_appid),
            "wesee_ichid": "8"
        }
        json_data = {
            "req_body": {
                "requestType": 16,
                "isrefresh": 1,
                "isfirst": 1,
                "attachInfo": "",
                "scene_id": 22,
                "requestExt": {
                    "mini_openid": str(cls.wesee_openid),
                    "notLogin-personid": str(cls.wesee_personid)
                }
            },
            "req_header": {
                "mapExt": "{\"imageSize\":\"480\",\"adaptScene\":\"PicHDWebpLimitScene\"}"
            }
        }

        try:
            urllib3.disable_warnings()
            r = requests.post(headers=headers, url=url, cookies=cookies, json=json_data, proxies=proxies, verify=False)
            response = json.loads(r.content.decode("utf8"))
            feeds = response["rsp_body"]["feeds"]
            for i in range(len(feeds)):
                # 视频标题过滤话题及处理特殊字符
                weishi_title = feeds[i]["desc"]
                title_split1 = weishi_title.split(" #")
                if title_split1[0] != "":
                    title1 = title_split1[0]
                else:
                    title1 = title_split1[-1]

                title_split2 = title1.split(" #")
                if title_split2[0] != "":
                    title2 = title_split2[0]
                else:
                    title2 = title_split2[-1]

                title_split3 = title2.split("@")
                if title_split3[0] != "":
                    title3 = title_split3[0]
                else:
                    title3 = title_split3[-1]
                # 视频标题
                video_title = title3.strip().replace("\n", "") \
                    .replace("/", "").replace("快手", "").replace(" ", "") \
                    .replace(" ", "").replace("&NBSP", "").replace("\r", "") \
                    .replace("#", "").replace(".", "。").replace("\\", "") \
                    .replace(":", "").replace("*", "").replace("？", "") \
                    .replace("?", "").replace('"', "").replace("<", "") \
                    .replace(">", "").replace("|", "").replace("微视", "")

                # 视频 ID
                if "id" not in feeds[i]["video"]:
                    video_id = 0
                else:
                    video_id = feeds[i]["video"]["id"]

                # 播放数
                if "playNum" not in feeds[i]["ugcData"]:
                    video_play_cnt = 0
                else:
                    video_play_cnt = feeds[i]["ugcData"]["playNum"]

                # 点赞数
                if "dingCount" not in feeds[i]["ugcData"]:
                    video_like_cnt = 0
                else:
                    video_like_cnt = feeds[i]["ugcData"]["dingCount"]

                # 分享数
                if "shareNum" not in feeds[i]["ugcData"]:
                    video_share_cnt = 0
                else:
                    video_share_cnt = feeds[i]["ugcData"]["shareNum"]

                # 评论数
                if "totalCommentNum" not in feeds[i]["ugcData"]:
                    video_comment_cnt = 0
                else:
                    video_comment_cnt = feeds[i]["ugcData"]["totalCommentNum"]

                # 视频时长
                if "duration" not in feeds[i]["video"]:
                    video_duration = 0
                else:
                    video_duration = int(int(feeds[i]["video"]["duration"]) / 1000)

                # 视频宽高
                if "width" not in feeds[i]["video"] or "height" not in feeds[i]["video"]:
                    video_width = 0
                    video_height = 0
                    video_resolution = str(video_width) + "*" + str(video_height)
                else:
                    video_width = feeds[i]["video"]["width"]
                    video_height = feeds[i]["video"]["height"]
                    video_resolution = str(video_width) + "*" + str(video_height)

                # 视频发布时间
                if "createTime" not in feeds[i]:
                    video_send_time = 0
                else:
                    video_send_time = int(feeds[i]["createTime"]) * 1000

                # 用户昵称
                user_name = feeds[i]["poster"]["nick"].strip().replace("\n", "") \
                    .replace("/", "").replace("快手", "").replace(" ", "") \
                    .replace(" ", "").replace("&NBSP", "").replace("\r", "").replace("微视", "")

                # 用户 ID
                user_id = feeds[i]["poster"]["id"]

                # 用户头像地址
                if "thumbURL" not in feeds[i]["material"] and "avatar" not in feeds[i]["poster"]:
                    head_url = 0
                elif "thumbURL" in feeds[i]["material"]:
                    head_url = feeds[i]["material"]["thumbURL"]
                else:
                    head_url = feeds[i]["poster"]["avatar"]

                # 视频封面地址
                if len(feeds[i]["images"]) == 0:
                    cover_url = 0
                else:
                    cover_url = feeds[i]["images"][0]["url"]

                # 视频播放地址
                if "url" not in feeds[i]["video"]:
                    video_url = 0
                else:
                    video_url = feeds[i]["video"]["url"]

                Common.logger("recommend").info("video_title:{}".format(video_title))
                Common.logger("recommend").info("video_id:{}".format(video_id))
                Common.logger("recommend").info("video_play_cnt:{}".format(video_play_cnt))
                Common.logger("recommend").info("video_like_cnt:{}".format(video_like_cnt))
                Common.logger("recommend").info("video_share_cnt:{}".format(video_share_cnt))
                # Common.logger("recommend").info("video_comment_cnt:{}".format(video_comment_cnt))
                Common.logger("recommend").info("video_duration:{}秒".format(video_duration))
                # Common.logger("recommend").info("video_resolution:{}".format(video_resolution))
                Common.logger("recommend").info(
                    "video_send_time:{}".format(time.strftime(
                        "%Y/%m/%d %H:%M:%S", time.localtime(int(video_send_time) / 1000))))
                Common.logger("recommend").info("user_name:{}".format(user_name))
                # Common.logger("recommend").info("user_id:{}".format(user_id))
                # Common.logger("recommend").info("head_url:{}".format(head_url))
                # Common.logger("recommend").info("cover_url:{}".format(cover_url))
                Common.logger("recommend").info("video_url:{}".format(video_url))

                # 过滤无效视频
                if video_id == 0 or video_duration == 0 or video_send_time == 0 or head_url == 0 \
                        or cover_url == 0 or video_url == 0:
                    Common.logger("recommend").info("无效视频")
                # 判断基础规则
                elif cls.download_rule(video_duration, video_width, video_height,
                                       video_play_cnt, video_like_cnt, video_share_cnt) is False:
                    Common.logger("recommend").info("不满足基础规则")
                # 判断敏感词
                elif any(word if word in weishi_title else False for word in cls.sensitive_words()) is True:
                    Common.logger("recommend").info("视频已中敏感词:{}".format(weishi_title))
                # 从 云文档 去重:https://w42nne6hzg.feishu.cn/sheets/shtcn5YSWg91JfVGzj0SFZIRRPh?sheet=caa3fa
                elif video_id in [j for m in Feishu.get_values_batch("recommend", "caa3fa") for j in m]:
                    Common.logger("recommend").info("该视频已下载:{}", video_title)
                # 从 云文档 去重:https://w42nne6hzg.feishu.cn/sheets/shtcn5YSWg91JfVGzj0SFZIRRPh?sheet=O7fCzr
                elif video_id in [j for n in Feishu.get_values_batch("recommend", "O7fCzr") for j in n]:
                    Common.logger("recommend").info("该视频已在feeds中:{}", video_title)
                else:
                    Common.logger("recommend").info("该视频未下载，添加至feeds中:{}".format(video_title))
                    # feeds工作表，插入首行
                    time.sleep(1)
                    Feishu.insert_columns("recommend", "O7fCzr", "ROWS", 1, 2)
                    # 获取当前时间
                    get_feeds_time = int(time.time())
                    # 工作表 feeds 中写入数据
                    values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(int(get_feeds_time))),
                               "推荐榜",
                               video_id,
                               video_title,
                               video_play_cnt,
                               video_comment_cnt,
                               video_like_cnt,
                               video_share_cnt,
                               video_duration,
                               video_resolution,
                               time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(int(video_send_time / 1000))),
                               user_name,
                               user_id,
                               head_url,
                               cover_url,
                               video_url]]
                    # 等待 1s，防止操作云文档太频繁，导致报错
                    time.sleep(1)
                    Feishu.update_values("recommend", "O7fCzr", "A2:P2", values)
        except Exception as e:
            Common.logger("recommend").error("获取微视视频list异常:{}".format(e))

    # 下载/上传视频
    @classmethod
    def download_publish(cls):
        try:
            for i in range(1, len(Feishu.get_values_batch("recommend", "O7fCzr")) + 1):
                time.sleep(1)
                download_video_id = Feishu.get_values_batch("recommend", "O7fCzr")[i][2]
                download_video_title = Feishu.get_values_batch("recommend", "O7fCzr")[i][3]
                download_video_play_cnt = Feishu.get_values_batch("recommend", "O7fCzr")[i][4]
                download_video_comment_cnt = Feishu.get_values_batch("recommend", "O7fCzr")[i][5]
                download_video_like_cnt = Feishu.get_values_batch("recommend", "O7fCzr")[i][6]
                download_video_share_cnt = Feishu.get_values_batch("recommend", "O7fCzr")[i][7]
                download_video_duration = Feishu.get_values_batch("recommend", "O7fCzr")[i][8]
                download_video_resolution = Feishu.get_values_batch("recommend", "O7fCzr")[i][9]
                # download_video_width = download_video_resolution.split("*")[0]
                # download_video_height = download_video_resolution.split("*")[-1]
                download_video_send_time = Feishu.get_values_batch("recommend", "O7fCzr")[i][10]
                download_user_name = Feishu.get_values_batch("recommend", "O7fCzr")[i][11]
                download_user_id = Feishu.get_values_batch("recommend", "O7fCzr")[i][12]
                download_head_url = Feishu.get_values_batch("recommend", "O7fCzr")[i][13]
                download_cover_url = Feishu.get_values_batch("recommend", "O7fCzr")[i][14]
                download_video_url = Feishu.get_values_batch("recommend", "O7fCzr")[i][15]

                # Common.logger("recommend").info("download_video_id:{}", download_video_id)
                # Common.logger("recommend").info("download_video_title:{}", download_video_title)
                # Common.logger("recommend").info("download_video_play_cnt:{}", download_video_play_cnt)
                # Common.logger("recommend").info("download_video_comment_cnt:{}", download_video_comment_cnt)
                # Common.logger("recommend").info("download_video_like_cnt:{}", download_video_like_cnt)
                # Common.logger("recommend").info("download_video_share_cnt:{}", download_video_share_cnt)
                # Common.logger("recommend").info("download_video_duration:{}", download_video_duration)
                # Common.logger("recommend").info("download_video_resolution:{}", download_video_resolution)
                # Common.logger("recommend").info("download_video_send_time:{}", download_video_send_time)
                # Common.logger("recommend").info("download_user_name:{}", download_user_name)
                # Common.logger("recommend").info("download_user_id:{}", download_user_id)
                # Common.logger("recommend").info("download_head_url:{}", download_head_url)
                # Common.logger("recommend").info("download_cover_url:{}", download_cover_url)
                # Common.logger("recommend").info("download_video_url:{}", download_video_url)

                Common.logger("recommend").info("正在判断第{}行，视频:{}", i, download_video_title)

                # 过滤空行
                if download_video_id is None \
                        or download_video_id == "" \
                        or download_video_title is None \
                        or download_video_title == "":
                    Common.logger("recommend").warning("空行，删除")
                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range("recommend", "O7fCzr", "ROWS", i + 1, i + 1)
                    return
                # 分享量>=1000
                elif int(download_video_share_cnt) < 1000:
                    Common.logger("recommend").info("分享量:{} < 1000", download_video_share_cnt)
                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range("recommend", "O7fCzr", "ROWS", i + 1, i + 1)
                    return
                # 去重
                elif download_video_id in [j for m in Feishu.get_values_batch("recommend", "caa3fa") for j in m]:
                    Common.logger("recommend").info("该视频已下载:{}", download_video_title)
                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range("recommend", "O7fCzr", "ROWS", i + 1, i + 1)
                    return
                else:
                    Common.logger("recommend").info("开始下载视频:{}", download_video_title)
                    # 下载封面
                    Common.download_method(job="recommend", text="cover",
                                           d_name=str(download_video_title), d_url=str(download_cover_url))
                    # 下载视频
                    Common.download_method(job="recommend", text="video",
                                           d_name=str(download_video_title), d_url=str(download_video_url))
                    # 保存视频信息至 "./videos/{download_video_title}/info.txt"
                    with open("./videos/" + download_video_title
                              + "/" + "info.txt", "a", encoding="UTF-8") as f_a:
                        f_a.write(str(download_video_id) + "\n" +
                                  str(download_video_title) + "\n" +
                                  str(download_video_duration) + "\n" +
                                  str(download_video_play_cnt) + "\n" +
                                  str(download_video_comment_cnt) + "\n" +
                                  str(download_video_like_cnt) + "\n" +
                                  str(download_video_share_cnt) + "\n" +
                                  str(download_video_resolution) + "\n" +
                                  str(int(time.mktime(
                                      time.strptime(download_video_send_time, "%Y/%m/%d %H:%M:%S")))) + "\n" +
                                  str(download_user_name) + "\n" +
                                  str(download_head_url) + "\n" +
                                  str(download_video_url) + "\n" +
                                  str(download_cover_url) + "\n" +
                                  str(cls.wesee_access_token))
                    Common.logger("recommend").info("==========视频信息已保存至info.txt==========")

                    # 上传视频
                    Common.logger("recommend").info("开始上传视频:{}".format(download_video_title))
                    Publish.upload_and_publish("recommend", "prod", "play")

                    # 保存视频 ID 到云文档:https://w42nne6hzg.feishu.cn/sheets/shtcn5YSWg91JfVGzj0SFZIRRPh?sheet=caa3fa
                    Common.logger("recommend").info("保存视频ID至云文档:{}", download_video_title)
                    # 视频ID工作表，插入首行
                    Feishu.insert_columns("recommend", "caa3fa", "ROWS", 1, 2)
                    # 视频ID工作表，首行写入数据
                    upload_time = int(time.time())
                    values = [[str(time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time))),
                               "推荐榜",
                               str(download_video_id),
                               str(download_video_title),
                               download_video_play_cnt,
                               download_video_comment_cnt,
                               download_video_like_cnt,
                               download_video_share_cnt,
                               download_video_duration,
                               str(download_video_resolution),
                               str(download_video_send_time),
                               str(download_user_name),
                               str(download_user_id),
                               str(download_head_url),
                               str(download_cover_url),
                               str(download_video_url)]]
                    time.sleep(1)
                    Feishu.update_values("recommend", "caa3fa", "A2:Q2", values)

                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range("recommend", "O7fCzr", "ROWS", i + 1, i + 1)
                    return
        except Exception as e:
            Common.logger("recommend").error("下载/上传视频异常:{}", e)
            Feishu.dimension_range("recommend", "O7fCzr", "ROWS", 2, 2)


if __name__ == "__main__":
    weishi = DownloadRecommend()
    for n in range(2):
        Common.logger("recommend").info("正在抓取第{}页视频", n + 1)
        weishi.get_feeds()

    # print(weishi.Referer)
    # print(weishi.wesee_openid)
    # print(weishi.wesee_openkey)
    # print(weishi.wesee_personid)
    # print(weishi.wesee_access_token)
    # print(weishi.wesee_thr_appid)
    # print(weishi.json_text)