crawler_xiaoniangao/main/person_list.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/5/18
import time
import requests
import urllib3
from main.common import Common
from main.feishu_lib import Feishu
from main.publish import Publish

proxies = {"http": None, "https": None}


class Person:
    next_t = -1

    # 过滤敏感词
    @classmethod
    def sensitive_words(cls):
        # 敏感词库列表
        word_list = []
        # 从云文档读取所有敏感词，添加到词库列表
        lists = Feishu.get_values_batch("person-logs", "xiaoniangao", "DRAnZh")
        for i in lists:
            for j in i:
                # 过滤空的单元格内容
                if j is None:
                    pass
                else:
                    word_list.append(j)
        return word_list

    # 获取用户列表
    @classmethod
    def person_list(cls):
        try:
            if len(Feishu.get_values_batch("person-logs", "xiaoniangao", "oNpThi")) == 1:
                Common.person_logger().info("暂无定向爬取账号")
            else:
                person_list = []
                nick_list = []
                for i in range(2, len(Feishu.get_values_batch("person-logs", "xiaoniangao", "oNpThi")) + 1):
                    time.sleep(0.5)
                    profile_mid = Feishu.get_range_value(
                        "person-logs", "xiaoniangao", "oNpThi", "B" + str(i) + ":" + "B" + str(i))[0]
                    time.sleep(0.5)
                    nick = \
                        Feishu.get_range_value("person-logs", "xiaoniangao", "oNpThi",
                                               "C" + str(i) + ":" + "C" + str(i))[0]
                    nick_list.append(nick)
                    person_list.append(profile_mid)

                Common.person_logger().info("已获取用户列表:{}", nick_list)
                return person_list

        except Exception as e:
            Common.person_logger().error("获取用户列表异常:{}", e)

    # 关注列表中的用户
    @classmethod
    def sub_persons(cls):
        profile_mids = cls.person_list()
        for profile_mid in profile_mids:
            url = "https://api.xiaoniangao.cn/V1/account/sub_user"
            headers = {
                "X-Mid": "1164637358",
                "X-Token-Id": "af9c47bb6c942236ff35ee10d355f3b0-1164637358",
                "content-type": "application/json",
                "uuid": "3d460a1b-ab85-426b-bd80-62029acaa2c0",
                "Accept-Encoding": "gzip,compress,br,deflate",
                "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)"
                              " AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 "
                              "MicroMessenger/8.0.20(0x18001435) NetType/WIFI Language/zh_CN",
                "Referer": "https://servicewechat.com/wxd7911e4c177690e4/617/page-frame.html"
            }
            data = {
                "visited_mid": int(profile_mid),
                "log_common_params": {
                    "e": [{
                        "data": {
                            "page": "profilePage",
                            "topic": "public",
                            "type": "follow",
                            "name": "user",
                            "smid": str(profile_mid)
                        },
                        "ab": {}
                    }],
                    "ext": {
                        "brand": "iPhone",
                        "device": "iPhone 11",
                        "os": "iOS 14.7.1",
                        "weixinver": "8.0.20",
                        "srcver": "2.24.2",
                        "net": "wifi",
                        "scene": "1089"
                    },
                    "pj": "1",
                    "pf": "2",
                    "session_id": "d53b6125-942b-4ec1-8d22-f9451a35e9f9"
                },
                "token": "451273638af2c8bb90266bcfaf601a68",
                "uid": "3d460a1b-ab85-426b-bd80-62029acaa2c0",
                "proj": "ma",
                "wx_ver": "8.0.20",
                "code_ver": "3.62.0"
            }

            try:
                urllib3.disable_warnings()
                r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False)
                Common.person_logger().info("关注用户:{}，{}", profile_mid, r)
            except Exception as e:
                Common.person_logger().error("关注用户异常:{}", e)

    # 从关注列表获取视频，并下载符合规则的视频，再进行上传
    @classmethod
    def download_from_sub(cls, endtime):
        url = "https://api.xiaoniangao.cn/album/get_user_trends"
        headers = {
            "X-Mid": "1164637358",
            "X-Token-Id": "af9c47bb6c942236ff35ee10d355f3b0-1164637358",
            "content-type": "application/json",
            "uuid": "3d460a1b-ab85-426b-bd80-62029acaa2c0",
            "Accept-Encoding": "gzip,compress,br,deflate",
            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)"
                          " AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 "
                          "MicroMessenger/8.0.20(0x18001435) NetType/WIFI Language/zh_CN",
            "Referer": "https://servicewechat.com/wxd7911e4c177690e4/617/page-frame.html"
        }
        data = {
            "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!750x500r/crop/750x500/interlace/1/format/jpg",
            "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!80x80r/crop/80x80/interlace/1/format/jpg",
            "start_t": int(cls.next_t),
            "limit": 5,
            "share_width": 625,
            "share_height": 500,
            "token": "451273638af2c8bb90266bcfaf601a68",
            "uid": "3d460a1b-ab85-426b-bd80-62029acaa2c0",
            "proj": "ma",
            "wx_ver": "8.0.20",
            "code_ver": "3.62.0",
            "log_common_params": {
                "e": [{
                    "data": {
                        "page": "discoverIndexPage",
                        "topic": "follow"
                    }
                }],
                "ext": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.2",
                    "net": "wifi",
                    "scene": "1089"
                },
                "pj": "1",
                "pf": "2",
                "session_id": "18da9157-5aa6-4955-a849-9160f07ee912"
            }
        }
        try:
            urllib3.disable_warnings()
            r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False)
            cls.next_t = r.json()["data"]["next_t"]
            # cls.next_t_list.append(next_t)
            feeds = r.json()["data"]["list"]
            for i in range(len(feeds)):
                # 标题
                video_title = feeds[i]["title"].strip().replace("\n", "") \
                            .replace("/", "").replace("\r", "").replace("#", "") \
                            .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
                            .replace(":", "").replace("*", "").replace("？", "") \
                            .replace("?", "").replace('"', "").replace("<", "") \
                            .replace(">", "").replace("|", "").replace(" ", "")
                Common.person_logger().info("标题:{}", video_title)
                # 用户名
                user_name = feeds[i]["user"]["nick"].strip().replace("\n", "") \
                    .replace("/", "").replace("快手", "").replace(" ", "") \
                    .replace(" ", "").replace("&NBSP", "").replace("\r", "")
                Common.person_logger().info("用户名:{}", user_name)
                # 视频 ID
                video_id = feeds[i]["vid"]
                Common.person_logger().info("视频ID:{}", video_id)
                # 播放量
                video_play_cnt = feeds[i]["play_pv"]
                Common.person_logger().info("播放量:{}", video_play_cnt)
                # 评论数
                video_comment_cnt = feeds[i]["comment_count"]
                # 点赞
                video_like_cnt = feeds[i]["favor"]["total"]
                # 分享
                video_share_cnt = feeds[i]["share"]
                # 时长
                video_duration = int(feeds[i]["du"] / 1000)
                # 发布时间
                video_send_time = feeds[i]["t"]
                Common.person_logger().info(
                    "发布时间:{}", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time) / 1000)))
                # 宽和高
                video_width = feeds[i]["w"]
                video_height = feeds[i]["h"]
                # 头像
                head_url = feeds[i]["user"]["hurl"]
                # 用户 ID
                profile_id = feeds[i]["id"]
                # 用户 mid
                profile_mid = feeds[i]["user"]["mid"]
                # 封面
                cover_url = feeds[i]["url"]
                # 视频播放地址
                video_url = feeds[i]["v_url"]
                Common.person_logger().info("播放地址:{}", video_url)

                # 过滤无效视频
                if video_id == "" or video_url == "" or video_send_time == "":
                    Common.person_logger().info("无效视频")
                # 判断发布时间:2022年5月18日以后发布
                elif int(video_send_time) < endtime:
                    Common.person_logger().info(
                        "发布时间:{}，在2022年5月18日之前",
                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time) / 1000)))
                # 判断视频播放量大于1000
                elif int(video_play_cnt) < 1000:
                    Common.person_logger().info("视频:{}，播放量:{}<1000", video_title, video_play_cnt)
                # 过滤敏感词
                elif any(word if word in video_title else False for word in cls.sensitive_words()) is True:
                    Common.person_logger().info("视频已中敏感词：{}".format(video_title))
                # 从 云文档 去重:https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=yatRv2
                elif video_id in [j for i in Feishu.get_values_batch(
                        "person-logs", "xiaoniangao", "yatRv2") for j in i]:
                    Common.person_logger().info("该视频已下载：{}", video_title)
                # 满足抓取规则
                else:
                    Common.person_logger().info("开始下载视频:{}", video_title)
                    # 下载封面
                    Common.download_method(
                        log_path="person-logs", text="cover", d_name=video_title, d_url=cover_url)
                    # 下载视频
                    Common.download_method(
                        log_path="person-logs", text="video", d_name=video_title, d_url=video_url)
                    # 保存视频信息至 "./videos/{download_video_title}/info.txt"
                    with open(r"./videos/" + video_title
                              + "/" + "info.txt", "a", encoding="UTF-8") as f_a:
                        f_a.write(str(video_id) + "\n" +
                                  str(video_title) + "\n" +
                                  str(video_duration) + "\n" +
                                  str(video_play_cnt) + "\n" +
                                  str(video_comment_cnt) + "\n" +
                                  str(video_like_cnt) + "\n" +
                                  str(video_share_cnt) + "\n" +
                                  str(video_width)+"*"+str(video_height) + "\n" +
                                  str(video_send_time) + "\n" +
                                  str(user_name) + "\n" +
                                  str(head_url) + "\n" +
                                  str(video_url) + "\n" +
                                  str(cover_url) + "\n" +
                                  str("xiaoniangao"))
                    Common.person_logger().info("==========视频信息已保存至info.txt==========")

                    # 上传视频
                    Common.person_logger().info("开始上传视频:{}".format(video_title))
                    Publish.upload_and_publish("dev", "play")
                    Common.person_logger().info("视频上传完成:{}", video_title)
                    # 上传完成时间
                    upload_time = int(time.time())

                    # 保存视频信息到云文档
                    Common.person_logger().info("添加视频到云文档:{}", video_title)
                    # 插入空行
                    time.sleep(1)
                    Feishu.insert_columns("person-logs", "xiaoniangao", "yatRv2", "ROWS", 1, 2)
                    # 视频信息写入云文档
                    values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(upload_time))),
                               "定向账号爬取",
                               video_id,
                               video_title,
                               video_play_cnt,
                               video_comment_cnt,
                               video_like_cnt,
                               video_share_cnt,
                               video_duration,
                               str(video_width)+"*"+str(video_height),
                               time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time)/1000)),
                               user_name,
                               profile_id,
                               profile_mid,
                               head_url,
                               cover_url,
                               video_url]]
                    time.sleep(1)
                    Feishu.update_values("person-logs", "xiaoniangao", "yatRv2", "A2:Q2", values)
                return int(video_send_time)
        except Exception as e:
            Common.person_logger().error("请求关注列表异常:{}", e)


if __name__ == "__main__":
    person = Person()
    # person.person_list()
    # person.download_person_videos()
    person.sub_persons()