crawler_monitor/main/update_xiaoniangao.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/6/23
import datetime
import os
import sys
import time
import requests
import urllib3
sys.path.append(os.getcwd())
from crawler_monitor.main.common import Common
from crawler_monitor.main.feishu_lib import Feishu
proxies = {"http": None, "https": None}


class UpdateXiaoNianGao:
    # 待更新数据列表
    update_list = []

    # 检查是否有今日的上升榜日期
    @classmethod
    def check_data(cls, date):
        # 判断J1单元格的日期是否为今天
        time.sleep(1)
        if Feishu.get_range_value("xiaoniangao", "monitor", "N7e2yI", "P1:P1")[0] != str(date):
            Common.logger("xiaoniangao").info("今天日期不存在")
            # 插入列 I1:J1，并写入日期
            values = [[str(date)]]
            time.sleep(1)
            Feishu.insert_columns("xiaoniangao", "monitor", "N7e2yI", "COLUMNS", 15, 16)
            time.sleep(1)
            Feishu.update_values("xiaoniangao", "monitor", "N7e2yI", "P1:P1", values)
            Common.logger("xiaoniangao").info("插入今天日期成功\n")
        else:
            Common.logger("xiaoniangao").info("今天日期已存在\n")

    # 更新小年糕播放量
    @classmethod
    def update_play_cnt(cls):
        try:
            xiaoniangao_sheet = Feishu.get_values_batch("xiaoniangao", "monitor", "N7e2yI")
            for i in range(1, len(xiaoniangao_sheet)):
                Common.logger("xiaoniangao").info("正在更新第{}行", i+1)
                video_id = xiaoniangao_sheet[i][6]
                user_id = xiaoniangao_sheet[i][9]
                user_mid = xiaoniangao_sheet[i][10]

                if video_id is None or video_id == "" \
                        or user_id is None or user_id == "" \
                        or user_mid is None or user_mid == "":
                    Common.logger("xiaoniangao").info("空行")
                    cls.update_list.append("0")
                else:
                    url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id"
                    headers = {
                        "x-b3-traceid": "96376ab5ed525",
                        "X-Token-Id": "35f5e036e103500b6b51f11adf1b345d-1145266232",
                        "uid": "250e6514-fd83-446c-a880-e274c7f17bce",
                        "content-type": "application/json",
                        "Accept-Encoding": "gzip,compress,br,deflate",
                        "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
                                      ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
                                      'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
                        "Referer": "https://servicewechat.com/wxd7911e4c177690e4/623/page-frame.html"
                    }
                    data = {
                        "play_src": "1",
                        "profile_id": int(user_id),
                        "profile_mid": int(user_mid),
                        "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/"
                              "!400x400r/crop/400x400/interlace/1/format/jpg",
                        "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail"
                                "/!80x80r/crop/80x80/interlace/1/format/jpg",
                        "share_width": 625,
                        "share_height": 500,
                        "no_comments": True,
                        "no_follow": True,
                        "vid": str(video_id),
                        "hot_l1_comment": True,
                        "token": "a7a4cb67d17ceb2e02a6d8bb6854beaa",
                        "uid": "250e6514-fd83-446c-a880-e274c7f17bce",
                        "proj": "ma",
                        "wx_ver": "8.0.23",
                        "code_ver": "3.67.0",
                        "log_common_params": {
                            "e": [{
                                "data": {
                                    "page": "dynamicSharePage"
                                }
                            }],
                            "ext": {
                                "brand": "iPhone",
                                "device": "iPhone 11",
                                "os": "iOS 14.7.1",
                                "weixinver": "8.0.23",
                                "srcver": "2.24.6",
                                "net": "wifi",
                                "scene": "1089"
                            },
                            "pj": "1",
                            "pf": "2",
                            "session_id": "d0c5c363-d571-4c8a-ab55-91a7ad284ee4"
                        }
                    }
                    urllib3.disable_warnings()
                    r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False)
                    video_play_cnt = r.json()["data"]["play_pv"]

                    # 单条更新
                    # Feishu.update_values(
                    #     "xiaoniangao", "monitor", "N7e2yI", "J" + str(i+1) + ":" + "J" + str(i+1), [[video_play_cnt]])
                    # Common.logger("xiaoniangao").info("video_id:{}, play_cnt:{}, 更新成功\n", video_id, video_play_cnt)

                    # 待更新数据列表
                    cls.update_list.append(video_play_cnt)

                    Common.logger("xiaoniangao").info("video_id:{}, 播放量:{}，已添加至待更新数据列表", video_id, video_play_cnt)
                    Common.logger("xiaoniangao").info("待更新列表数量:{}\n", len(cls.update_list))

                if len(cls.update_list) >= 200:
                    Common.logger("xiaoniangao").info("P{}:P{}", str(i-198), str(i+1))
                    Common.logger("xiaoniangao").info("{}\n", cls.update_list)
                    time.sleep(1)
                    Feishu.update_values(
                        "xiaoniangao", "monitor", "N7e2yI",
                        "P"+str(i-198) + ":" + "P"+str(i+1), [[x] for x in cls.update_list])
                    Common.logger("xiaoniangao").info("更新{}条播放量成功\n", len(cls.update_list))
                    cls.update_list = []
                elif i+1 == len(xiaoniangao_sheet):
                    Common.logger("xiaoniangao").info("P{}:P{}", str(i+1-len(cls.update_list)+1), str(i+1))
                    Common.logger("xiaoniangao").info("{}\n", cls.update_list)
                    time.sleep(1)
                    Feishu.update_values(
                        "xiaoniangao", "monitor", "N7e2yI",
                        "P" + str(i+1-len(cls.update_list)+1) + ":" + "P" + str(i+1), [[x] for x in cls.update_list])
                    Common.logger("xiaoniangao").info("更新{}条播放量成功\n", len(cls.update_list))
                    cls.update_list = []
                    return

        except Exception as e:
            Common.logger("xiaoniangao").error("更新小年糕播放量异常:{}", e)


if __name__ == "__main__":
    xng = UpdateXiaoNianGao()
    start_time = time.time()
    today = datetime.datetime.now().strftime("%Y-%m-%d")
    xng.check_data(today)
    xng.update_play_cnt()
    end_time = time.time()
    Common.logger("xiaoniangao").info("全部更新完成，共耗时:{}秒\n", int(end_time - start_time))