优化速度，嘎嘎快

main/demo.py

@@ -7,6 +7,8 @@ from datetime import timedelta
 import requests
 import urllib3
 
+from main.feishu_lib import Feishu
+
 proxies = {"http": None, "https": None}
 
 
@@ -74,6 +76,7 @@ class Demo:
             video_duration = int(r.json()["data"]["du"])/1000
             video_url = r.json()["data"]["v_url"]
             video_send_time = r.json()["data"]["t"]
+            print(r.text)
             print(f"video_title:{video_title}")
             print(f"video_id:{video_id}")
             print(f"video_play_cnt:{video_play_cnt}")
@@ -92,8 +95,22 @@ class Demo:
         print(before_yesterday)
         print(type(before_yesterday))
 
+    # 微信号
+    @classmethod
+    def get_sheet(cls):
+        hour_sheet = Feishu.get_values_batch("hour", "xiaoniangao", "ba0da4")
+        print(hour_sheet[2][8])
+
 
 if __name__ == "__main__":
     demo = Demo()
-    demo.video_detail_info("44504014", "1105334217", "4433151749")
+    # demo.video_detail_info("44838076", "116311065", "3941884615")
+    # print("\n")
+    # demo.video_detail_info("44767103", "116311065", "4167630382")
+    # print("\n")
+    # demo.video_detail_info("44264441", "329676836", "4505318929")
+
     # demo.times()
+
+    # demo.video_detail_info("44615236", "460972177", "62a9965e000001609aa588ad")
+    demo.get_sheet()

main/hour_list.py

@@ -16,11 +16,16 @@ proxies = {"http": None, "https": None}
 
 
 class HourList:
-    hour_x_b3_traceid = Feishu.get_range_value("hour", "xiaoniangao", "dzcWHw", "B3:B3")[0]
-    hour_x_token_id = Feishu.get_range_value("hour", "xiaoniangao", "dzcWHw", "B4:B4")[0]
-    hour_referer = Feishu.get_range_value("hour", "xiaoniangao", "dzcWHw", "B5:B5")[0][0]["link"]
-    hour_uid = Feishu.get_range_value("hour", "xiaoniangao", "dzcWHw", "B6:B6")[0]
-    hour_token = Feishu.get_range_value("hour", "xiaoniangao", "dzcWHw", "B7:B7")[0]
+    # 配置微信
+    wechat_sheet = Feishu.get_values_batch("hour", "xiaoniangao", "dzcWHw")
+    hour_x_b3_traceid = wechat_sheet[2][1]
+    hour_x_token_id = wechat_sheet[3][1]
+    hour_referer = wechat_sheet[4][1]
+    hour_uid = wechat_sheet[5][1]
+    hour_token = wechat_sheet[6][1]
+
+    # 小时级数据表
+    hour_sheet = Feishu.get_values_batch("hour", "xiaoniangao", "ba0da4")
 
     # 过滤敏感词
     @classmethod
@@ -92,7 +97,7 @@ class HourList:
     def check_hour_list_data(cls, log_type, date):
         # 判断J1单元格的日期是否为今天
         time.sleep(1)
-        if Feishu.get_range_value(log_type, "xiaoniangao", "ba0da4", "L1:N1")[0] != date:
+        if cls.hour_sheet[0][11] != date:
             # 插入3列 L1:N1，并写入日期和时间数据
             values = [[date], ["10:00", "15:00", "20:00"]]
             time.sleep(1)
@@ -314,7 +319,7 @@ class HourList:
                         time.sleep(1)
 
                     # 从云文档去重:https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=ba0da4
-                    elif video_id in [j for i in Feishu.get_values_batch(log_type, "xiaoniangao", "ba0da4") for j in i]:
+                    elif video_id in [j for i in cls.hour_sheet for j in i]:
                         Common.logger(log_type).info("该视频已保存过:{}", video_title)
                         time.sleep(1)
                     else:
@@ -352,58 +357,41 @@ class HourList:
         """
         try:
             time.sleep(1)
-            if len(Feishu.get_values_batch(log_type, "xiaoniangao", "ba0da4")) == 2:
+            if len(cls.hour_sheet) == 2:
                 Common.logger(log_type).info("当前工作表无数据")
             else:
                 time.sleep(1)
-                for i in range(3, len(Feishu.get_values_batch(log_type, "xiaoniangao", "ba0da4"))+1):
+                for i in range(3, len(cls.hour_sheet)+1):
                     time.sleep(1)
                     Common.logger(log_type).info("更新第:{}行视频信息", i)
 
                     # 略过空行
-                    if Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "D" + str(i) + ":" + "D" + str(i))[0] is None\
-                            and Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "C"+str(i)+":"+"C"+str(i))[0] is None\
-                            and Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "A"+str(i)+":"+"A"+str(i))[0] is None:
+                    if cls.hour_sheet[i][0] is None or cls.hour_sheet[i][1] is None or cls.hour_sheet[i][2] is None:
                         Common.logger(log_type).info("空行，略过")
                     else:
                         # 视频标题
-                        time.sleep(1)
-                        v_title = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "D" + str(i) + ":" + "D" + str(i))[0]
-                        Common.logger(log_type).info("视频详情，video_title:{}", v_title)
+                        v_title = cls.hour_sheet[i][3]
+                        Common.logger(log_type).info("video_title:{}", v_title)
 
                         # 视频 ID
-                        time.sleep(1)
-                        v_id = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "C" + str(i) + ":" + "C" + str(i))[0]
-                        Common.logger(log_type).info("视频详情，video_id:{}", v_id)
+                        v_id = cls.hour_sheet[i][2]
+                        Common.logger(log_type).info("video_id:{}", v_id)
 
                         # profile_id，用户 ID
-                        time.sleep(1)
-                        p_id = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "A" + str(i) + ":" + "A" + str(i))[0]
-                        Common.logger(log_type).info("视频详情，profile_id:{}", p_id)
+                        p_id = cls.hour_sheet[i][0]
+                        Common.logger(log_type).info("profile_id:{}", p_id)
 
                         # profile_mid
-                        time.sleep(1)
-                        p_mid = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "B" + str(i) + ":" + "B" + str(i))[0]
-                        Common.logger(log_type).info("视频详情，profile_mid:{}", p_mid)
+                        p_mid = cls.hour_sheet[i][1]
+                        Common.logger(log_type).info("profile_mid:{}", p_mid)
 
                         # 抓取时的播放量
-                        time.sleep(1)
-                        v_play_cnt = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "K" + str(i) + ":" + "K" + str(i))[0]
-                        Common.logger(log_type).info("视频详情，video_play_cnt:{}", v_play_cnt)
+                        v_play_cnt = cls.hour_sheet[i][10]
+                        Common.logger(log_type).info("video_play_cnt:{}", v_play_cnt)
 
                         # 抓取时间
-                        time.sleep(1)
-                        v_upload_time = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "J" + str(i) + ":" + "J" + str(i))[0]
-                        Common.logger(log_type).info("视频详情，video_send_time:{}", v_upload_time)
+                        v_upload_time = cls.hour_sheet[i][9]
+                        Common.logger(log_type).info("video_send_time:{}", v_upload_time)
                         # 抓取时间的时间戳格式（秒为单位）
                         v_time = int(time.mktime(time.strptime(v_upload_time, "%Y-%m-%d %H:%M:%S")))
 
@@ -495,13 +483,10 @@ class HourList:
                                 Common.logger(log_type).info("当天 15:00 视频播放量:{}", fifteen_hour_play_cnt)
 
                                 # 当天 10:00 上升的数据
-                                time.sleep(1)
-                                if Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4", "L"+str(i) + ":" + "L"+str(i))[0] is None:
+                                if cls.hour_sheet[i][11] is None:
                                     ten_up_cnt = 0
                                 else:
-                                    ten_up_cnt = Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4", "L"+str(i) + ":" + "L"+str(i))[0]
+                                    ten_up_cnt = cls.hour_sheet[i][11]
 
                                 # 15:00 的上升榜写入数据
                                 values = int(fifteen_hour_play_cnt) - (int(v_play_cnt) + int(ten_up_cnt))
@@ -533,23 +518,17 @@ class HourList:
 
                                 # 当天 10:00 上升的数据
                                 time.sleep(1)
-                                if Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4",
-                                        "L" + str(i) + ":" + "L" + str(i))[0] is None:
+                                if cls.hour_sheet[i][11] is None:
                                     ten_up_cnt = 0
                                 else:
-                                    ten_up_cnt = Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0]
+                                    ten_up_cnt = cls.hour_sheet[i][11]
 
                                 # 当天 15:00 上升的数据
                                 time.sleep(1)
-                                if Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4",
-                                        "M" + str(i) + ":" + "M" + str(i))[0] is None:
+                                if cls.hour_sheet[i][12] is None:
                                     fifteen_up_cnt = 0
                                 else:
-                                    fifteen_up_cnt = Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0]
+                                    fifteen_up_cnt = cls.hour_sheet[i][12]
 
                                 # 20:00 的上升榜写入数据
                                 values = int(twenty_hour_play_cnt) - (
@@ -568,13 +547,10 @@ class HourList:
 
                                 # 当天 15:00 上升的数据
                                 time.sleep(1)
-                                if Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4",
-                                        "M" + str(i) + ":" + "M" + str(i))[0] is None:
+                                if cls.hour_sheet[i][12] is None:
                                     fifteen_up_cnt = 0
                                 else:
-                                    fifteen_up_cnt = Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0]
+                                    fifteen_up_cnt = cls.hour_sheet[i][12]
 
                                 # 20:00 的上升榜写入数据
                                 values = int(twenty_hour_play_cnt) - (int(v_play_cnt) + int(fifteen_up_cnt))
@@ -622,13 +598,10 @@ class HourList:
 
                                 # 当天 10:00 上升的数据
                                 time.sleep(1)
-                                if Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4",
-                                        "L" + str(i) + ":" + "L" + str(i))[0] is None:
+                                if cls.hour_sheet[i][11] is None:
                                     ten_up_cnt = 0
                                 else:
-                                    ten_up_cnt = Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0]
+                                    ten_up_cnt = cls.hour_sheet[i][11]
 
                                 # 15:00 的上升榜写入数据
                                 values = int(fifteen_hour_play_cnt) - (int(v_play_cnt) + int(ten_up_cnt))
@@ -647,23 +620,17 @@ class HourList:
 
                                 # 当天 10:00 上升的数据
                                 time.sleep(1)
-                                if Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4",
-                                        "L" + str(i) + ":" + "L" + str(i))[0] is None:
+                                if cls.hour_sheet[i][11] is None:
                                     ten_up_cnt = 0
                                 else:
-                                    ten_up_cnt = Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0]
+                                    ten_up_cnt = cls.hour_sheet[i][11]
 
                                 # 当天 15:00 上升的数据
                                 time.sleep(1)
-                                if Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4",
-                                        "M" + str(i) + ":" + "M" + str(i))[0] is None:
+                                if cls.hour_sheet[i][12] is None:
                                     fifteen_up_cnt = 0
                                 else:
-                                    fifteen_up_cnt = Feishu.get_range_value(
-                                        log_type, "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0]
+                                    fifteen_up_cnt = cls.hour_sheet[i][12]
 
                                 # 20:00 的上升榜写入数据
                                 values = int(twenty_hour_play_cnt) - (
@@ -692,89 +659,66 @@ class HourList:
         """
         try:
             time.sleep(1)
-            if len(Feishu.get_values_batch(log_type, "xiaoniangao", "ba0da4")) == 2:
+            if len(cls.hour_sheet) == 2:
                 Common.logger(log_type).info("当前工作表无数据")
             else:
                 time.sleep(1)
-                for i in range(3, len(Feishu.get_values_batch(log_type, "xiaoniangao", "ba0da4"))+1):
+                for i in range(3, len(cls.hour_sheet)+1):
                     time.sleep(1)
                     Common.logger(log_type).info("分析第:{}行视频信息是否符合下载规则", i)
 
                     # 略过空行
-                    if Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "D" + str(i) + ":" + "D" + str(i))[0] is None\
-                            and Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "C"+str(i)+":"+"C"+str(i))[0] is None\
-                            and Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "A"+str(i)+":"+"A"+str(i))[0] is None:
+                    if cls.hour_sheet[i][0] is None or cls.hour_sheet[i][1] is None or cls.hour_sheet[i][2] is None:
                         Common.logger(log_type).info("空行，略过")
                     else:
                         # 今日 10:00 数据上升量
                         time.sleep(1)
-                        if Feishu.get_range_value(
-                                log_type, "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0] is None:
+                        if cls.hour_sheet[i][11] is None:
                             ten_cnt = 0
                         else:
-                            ten_cnt = Feishu.get_range_value(
-                                log_type, "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0]
+                            ten_cnt = cls.hour_sheet[i][11]
                         # 今日 15:00 数据上升量
                         time.sleep(1)
-                        if Feishu.get_range_value(
-                                log_type, "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0] is None:
+                        if cls.hour_sheet[i][12] is None:
                             fifteen_cnt = 0
                         else:
-                            fifteen_cnt = Feishu.get_range_value(
-                                log_type, "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0]
+                            fifteen_cnt = cls.hour_sheet[i][12]
                         # 今日 20:00 数据上升量
                         time.sleep(1)
-                        if Feishu.get_range_value(
-                                log_type, "xiaoniangao", "ba0da4", "N" + str(i) + ":" + "N" + str(i))[0] is None:
+                        if cls.hour_sheet[i][13] is None:
                             twenty_cnt = 0
                         else:
-                            twenty_cnt = Feishu.get_range_value(
-                                log_type, "xiaoniangao", "ba0da4", "N" + str(i) + ":" + "N" + str(i))[0]
+                            twenty_cnt = cls.hour_sheet[i][13]
                         # 昨日 20:00 数据上升量
                         time.sleep(1)
-                        if Feishu.get_range_value(
-                                log_type, "xiaoniangao", "ba0da4", "Q" + str(i) + ":" + "Q" + str(i))[0] is None:
+                        if cls.hour_sheet[i][16] is None:
                             yesterday_twenty_cnt = 0
                         else:
-                            yesterday_twenty_cnt = Feishu.get_range_value(
-                                log_type, "xiaoniangao", "ba0da4", "Q" + str(i) + ":" + "Q" + str(i))[0]
+                            yesterday_twenty_cnt = cls.hour_sheet[i][16]
 
                         # 视频标题
-                        time.sleep(1)
-                        v_title = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "D" + str(i) + ":" + "D" + str(i))[0]
-                        Common.logger(log_type).info("视频详情，video_title:{}", v_title)
+                        v_title = cls.hour_sheet[i][3]
+                        Common.logger(log_type).info("video_title:{}", v_title)
 
                         # 视频 ID
-                        time.sleep(1)
-                        v_id = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "C" + str(i) + ":" + "C" + str(i))[0]
-                        Common.logger(log_type).info("视频详情，video_id:{}", v_id)
+                        v_id = cls.hour_sheet[i][2]
+                        Common.logger(log_type).info("video_id:{}", v_id)
 
                         # profile_id，用户 ID
-                        time.sleep(1)
-                        p_id = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "A" + str(i) + ":" + "A" + str(i))[0]
+                        p_id = cls.hour_sheet[i][0]
 
                         # 视频时长
-                        time.sleep(1)
-                        v_duration = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "F" + str(i) + ":" + "F" + str(i))[0]
+                        v_duration = cls.hour_sheet[i][5]
 
                         # profile_mid
-                        time.sleep(1)
-                        p_mid = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "B" + str(i) + ":" + "B" + str(i))[0]
+                        p_mid = cls.hour_sheet[i][1]
+
                         Common.logger(log_type).info("10:00 / 15:00 / 20:00 上升量: {} / {} / {}",
                                                      ten_cnt, fifteen_cnt, twenty_cnt)
 
                         # 发布时间
                         time.sleep(1)
-                        v_upload_time = Feishu.get_range_value(
-                            log_type, "xiaoniangao", "ba0da4", "J" + str(i) + ":" + "J" + str(i))[0]
+                        v_upload_time = cls.hour_sheet[i][8]
                         v_send_time = int(time.mktime(time.strptime(v_upload_time, "%Y-%m-%d %H:%M:%S")))
 
                         # # 判断视频 ID 长度，不大于 13 位

main/person_list.py

@@ -15,11 +15,17 @@ class Person:
     # 翻页初始值
     next_t_list = [-1]
 
-    person_x_b3_traceid = Feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C3:C3")[0]
-    person_x_token_id = Feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C4:C4")[0]
-    person_referer = Feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C5:C5")[0][0]["link"]
-    person_uid = Feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C6:C6")[0]
-    person_token = Feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C7:C7")[0]
+    # person_x_b3_traceid = Feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C3:C3")[0]
+    # person_x_token_id = Feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C4:C4")[0]
+    # person_referer = Feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C5:C5")[0][0]["link"]
+    # person_uid = Feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C6:C6")[0]
+    # person_token = Feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C7:C7")[0]
+    wechat_sheet = Feishu.get_values_batch("person", "xiaoniangao", "dzcWHw")
+    person_x_b3_traceid = wechat_sheet[2][2]
+    person_x_token_id = wechat_sheet[3][2]
+    person_referer = wechat_sheet[4][2]
+    person_uid = wechat_sheet[5][2]
+    person_token = wechat_sheet[6][2]
 
     # 过滤敏感词
     @classmethod

main/run_hour_list.py

@@ -48,7 +48,7 @@ def hour_list_job():
                 Common.logger("hour").info("开始下载上升榜")
                 HourList.download_and_publish("hour")
 
-            elif hour_list_job_time.hour == 23 and hour_list_job_time.minute >= 50:
+            elif 1 >= hour_list_job_time.hour >= 0:
 
                 break
 

main/run_person_list.py

@@ -36,9 +36,10 @@ def person_list_job():
 
             if Person.download_from_sub("person", endtime) >= endtime:
                 Person.download_from_sub("person", endtime)
-            elif person_list_time.hour == 23 and person_list_time.minute >= 50:
+            elif 1 >= person_list_time.hour >= 0:
                 Common.logger("person").info("结束今日抓取任务")
                 Person.next_t_list = [-1]
+                time.sleep(300)
                 break
             else:
                 Common.logger("person").info("发布时间大于48小时，结束抓取任务")