crawler_xiaoniangao/main/demo.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/5/16
import datetime
import random
import time
from datetime import timedelta
import requests
import urllib3

from main.common import Common
from main.feishu_lib import Feishu

proxies = {"http": None, "https": None}


class Demo:
    # 查询视频详情的各项数据
    @classmethod
    def video_detail_info(cls, user_id, user_mid, video_id):
        url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id"
        headers = {
            "x-b3-traceid": "bd267349bf41b",
            "X-Token-Id": "86f6d7cc2b2b6870004df5d16c82aaf3-1185665701",
            "uid": "8fde3c6c-c070-4379-bfc4-15c7e85139c9",
            "content-type": "application/json",
            "Accept-Encoding": "gzip,compress,br,deflate",
            "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
                          ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
                          'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
            "Referer": "https://servicewechat.com/wxd7911e4c177690e4/617/page-frame.html"
        }
        data = {
            "play_src": "1",
            "profile_id": int(user_id),
            "profile_mid": int(user_mid),
            "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/"
                  "!400x400r/crop/400x400/interlace/1/format/jpg",
            "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail"
                    "/!80x80r/crop/80x80/interlace/1/format/jpg",
            "share_width": 625,
            "share_height": 500,
            "no_comments": True,
            "no_follow": True,
            "vid": video_id,
            "hot_l1_comment": True,
            "token": "90747742180aeb22c0fe3a3c6a38f3d9",
            "uid": "8fde3c6c-c070-4379-bfc4-15c7e85139c9",
            "proj": "ma",
            "wx_ver": "8.0.20",
            "code_ver": "3.62.0",
            "log_common_params": {
                "e": [{
                    "data": {
                        "page": "dynamicSharePage"
                    }
                }],
                "ext": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.3",
                    "net": "wifi",
                    "scene": "1089"
                },
                "pj": "1",
                "pf": "2",
                "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29"
            }
        }
        try:
            urllib3.disable_warnings()
            r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False)
            video_title = r.json()["data"]["title"]
            video_id = r.json()["data"]["vid"]
            video_play_cnt = r.json()["data"]["play_pv"]
            video_duration = int(r.json()["data"]["du"]) / 1000
            video_url = r.json()["data"]["v_url"]
            video_send_time = r.json()["data"]["t"]
            print(r.text)
            print(f"video_title:{video_title}")
            print(f"video_id:{video_id}")
            print(f"video_play_cnt:{video_play_cnt}")
            print(f"video_duration:{video_duration}")
            print(f'video_send_time:{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time) / 1000))}')
            print(f"video_url:{video_url}")
        except Exception as e:
            print(e)

    # 时间
    @classmethod
    def times(cls):
        # before_yesterday = (datetime.date.today() + timedelta(days=-2)).strftime("%Y/%m/%d %H:%M:%S")
        # before_yesterday = time.strptime(before_yesterday, "%Y/%m/%d %H:%M:%S")
        # before_yesterday = int(time.mktime(before_yesterday))
        # print(before_yesterday)
        # print(type(before_yesterday))

        # v_upload_time = Feishu.get_values_batch("hour", "xiaoniangao", "ba0da4")[7][5]
        # v_send_time = int(time.mktime(time.strptime(v_upload_time, "%Y-%m-%d %H:%M:%S")))
        # print(v_upload_time)
        # print(int(time.time()) - v_send_time)
        pass

    # 微信号
    @classmethod
    def get_sheet(cls):
        hour_sheet = Feishu.get_values_batch("person", "xiaoniangao", "k6ldje")
        # print(hour_sheet)
        print(hour_sheet[2][7])

    @classmethod
    def check_date(cls):
        date = Feishu.get_values_batch("hour", "xiaoniangao", "FEcuil")
        print(f'文档中L1:N1的日期:{date[0][11]}')
        today = datetime.datetime.now().strftime("%Y-%m-%d")
        print(f"today:{today}")
        Common.logger("hour").info("检查今日上升榜日期是否存在")
        if date[0][11] != today:
            # 插入3列 L1:N1，并写入日期和时间数据
            values = [[today], ["10:00", "15:00", "20:00"]]
            time.sleep(1)
            Feishu.insert_columns("hour", "xiaoniangao", "FEcuil", "COLUMNS", 11, 14)
            time.sleep(1)
            Feishu.update_values("hour", "xiaoniangao", "FEcuil", "L1:N2", values)
            time.sleep(1)
            Feishu.merge_cells("hour", "xiaoniangao", "FEcuil", "L1:N1")
            Common.logger("hour").info("插入今天日期成功")
        else:
            Common.logger("hour").info("今日上升榜日期已存在")

    # 获取列表
    @classmethod
    def get_recommend(cls, uid, token):
        """
        1.从列表获取视频，7 天内，播放量>=5000
        2.时长 1-10min
        3.每天10:00、15:00、20:00 把符合规则的视频，写入云文档
        https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=ba0da4
        """
        url = "https://kapi.xiaoniangao.cn/trends/get_recommend_trends"
        headers = {
            "x-b3-traceid": "1c403a4aa72e3c",
            "X-Token-Id": "ab619e96d801f1567388629260aa68ec-1202200806",
            "uid": uid,
            "content-type": "application/json",
            "Accept-Encoding": "gzip,compress,br,deflate",
            "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
                          ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
                          'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
            "Referer": "https://servicewechat.com/wxd7911e4c177690e4/624/page-frame.html"
        }
        data = {
            "log_params": {
                "page": "discover_rec",
                "common": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.2",
                    "net": "wifi",
                    "scene": 1089
                }
            },
            "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!750x500r/crop/750x500/interlace/1/format/jpg",
            "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!80x80r/crop/80x80/interlace/1/format/jpg",
            "share_width": 625,
            "share_height": 500,
            "ext": {
                "fmid": 0,
                "items": {}
            },
            "app": "xng",
            "rec_scene": "discover_rec",
            "log_common_params": {
                "e": [{
                    "data": {
                        "page": "discoverIndexPage",
                        "topic": "recommend"
                    },
                    "ab": {}
                }],
                "ext": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.3",
                    "net": "wifi",
                    "scene": "1089"
                },
                "pj": "1",
                "pf": "2",
                "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29"
            },
            "refresh": False,
            "token": token,
            "uid": uid,
            "proj": "ma",
            "wx_ver": "8.0.20",
            "code_ver": "3.62.0"
        }
        try:
            urllib3.disable_warnings()
            r = requests.post(url=url, headers=headers, json=data, proxies=proxies, verify=False)
            if "data" not in r.json():
                Common.logger("demo").warning("获取视频feeds错误:{}", r.text)
            elif "list" not in r.json()["data"]:
                Common.logger("demo").warning("获取视频feeds无数据，休眠10s:{}", r.text)
            else:
                # 视频列表数据
                feeds = r.json()["data"]["list"]
                for i in range(len(feeds)):
                    # 标题
                    if "title" in feeds[i]:
                        video_title = feeds[i]["title"].strip().replace("\n", "") \
                            .replace("/", "").replace("\r", "").replace("#", "") \
                            .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
                            .replace(":", "").replace("*", "").replace("？", "") \
                            .replace("?", "").replace('"', "").replace("<", "") \
                            .replace(">", "").replace("|", "").replace(" ", "")
                    else:
                        video_title = 0

                    # 视频 ID
                    if "vid" in feeds[i]:
                        video_id = feeds[i]["vid"]
                    else:
                        video_id = 0

                    # 播放量
                    if "play_pv" in feeds[i]:
                        video_play_cnt = feeds[i]["play_pv"]
                    else:
                        video_play_cnt = 0

                    # 点赞量
                    if "favor" in feeds[i]:
                        video_like_cnt = feeds[i]["favor"]["total"]
                    else:
                        video_like_cnt = 0

                    # 分享量
                    if "share" in feeds[i]:
                        video_share_cnt = feeds[i]["share"]
                    else:
                        video_share_cnt = 0

                    # # 评论量
                    # if "comment_count" in feeds[i]:
                    #     video_comment_cnt = feeds[i]["comment_count"]
                    # else:
                    #     video_comment_cnt = 0

                    # 时长
                    if "du" in feeds[i]:
                        video_duration = int(feeds[i]["du"] / 1000)
                    else:
                        video_duration = 0

                    # 宽和高
                    if "w" or "h" in feeds[i]:
                        video_width = feeds[i]["w"]
                        video_height = feeds[i]["h"]
                    else:
                        video_width = 0
                        video_height = 0

                    # 发布时间
                    if "t" in feeds[i]:
                        video_send_time = feeds[i]["t"]
                    else:
                        video_send_time = 0

                    # 用户名 / 头像
                    if "user" in feeds[i]:
                        user_name = feeds[i]["user"]["nick"].strip().replace("\n", "") \
                            .replace("/", "").replace("快手", "").replace(" ", "") \
                            .replace(" ", "").replace("&NBSP", "").replace("\r", "")
                        head_url = feeds[i]["user"]["hurl"]
                    else:
                        user_name = 0
                        head_url = 0

                    # 用户 ID
                    profile_id = feeds[i]["id"]

                    # 用户 mid
                    profile_mid = feeds[i]["user"]["mid"]

                    # 视频封面
                    if "url" in feeds[i]:
                        cover_url = feeds[i]["url"]
                    else:
                        cover_url = 0

                    # 视频播放地址
                    if "v_url" in feeds[i]:
                        video_url = feeds[i]["v_url"]
                    else:
                        video_url = 0

                    Common.logger("demo").info("标题:{}", video_title)
                    Common.logger("demo").info("视频ID:{}", video_id)
                    Common.logger("demo").info("播放量:{}", video_play_cnt)
                    # Common.logger("demo").info("点赞量:{}", video_like_cnt)
                    # Common.logger("demo").info("分享量:{}", video_share_cnt)
                    # Common.logger("demo").info("评论数:{}", video_comment_cnt)
                    Common.logger("demo").info("时长:{}秒", video_duration)
                    # Common.logger("demo").info("宽高:{}*{}", video_width, video_height)
                    Common.logger("demo").info(
                        "视频发布时间:{}", time.strftime(
                            "%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time) / 1000)))
                    Common.logger("demo").info("用户名:{}", user_name)
                    # Common.logger("demo").info("用户头像:{}", head_url)
                    # Common.logger("demo").info("封面:{}", cover_url)
                    Common.logger("demo").info("播放地址:{}", video_url)
        except Exception as e:
            Common.logger("demo").error("get_recommend异常:{}", e)

    # 生成 uid、token
    @classmethod
    def get_uid_token(cls):
        uid_token = "abcdefghijklmnopqrstuvwxyz0123456789"
        uid = ("".join(random.sample(uid_token, 8))) + "-" + ("".join(random.sample(uid_token, 4))) + "-" + (
            "".join(random.sample(uid_token, 4))) + "-" + ("".join(random.sample(uid_token, 4))) + "-" + (
                  "".join(random.sample(uid_token, 12)))
        token = ("".join(random.sample(uid_token, 32)))
        Feishu.update_values("hour", "xiaoniangao", "dzcWHw", "B6:B6", [[uid]])
        time.sleep(1)
        Feishu.update_values("hour", "xiaoniangao", "dzcWHw", "B7:B7", [[token]])


if __name__ == "__main__":
    demo = Demo()
    # demo.video_detail_info("44838076", "116311065", "3941884615")
    # print("\n")
    # demo.video_detail_info("44767103", "116311065", "4167630382")
    # print("\n")
    # demo.video_detail_info("44264441", "329676836", "4505318929")

    # demo.times()

    # demo.video_detail_info("44615236", "460972177", "62a9965e000001609aa588ad")
    # demo.get_sheet()
    # demo.check_date()
    # demo.get_recommend("56e87ba8-dc00-4b05-9fb9-75dbc734eabc", "c84bcac5cdd3893fc4b068e14f1d2456")
    demo.get_uid_token()