Merge branch 'master' of https://git.yishihui.com/Server/piaoquan_crawler

common/public.py

@@ -7,6 +7,9 @@ import random
 sys.path.append(os.getcwd())
 from common.common import Common
 from common.scheduling_db import MysqlHelper
+# from common import Common
+# from scheduling_db import MysqlHelper
+
 
 
 # 过滤词库
@@ -41,10 +44,11 @@ def get_user_from_mysql(log_type, crawler, source, env, action=''):
 
 
 def get_config_from_mysql(log_type, source, env, text, action=''):
-    select_sql = f"""select * from crawler_config where source='{source}' """
+    select_sql = f"""select * from crawler_config where source="{source}" """
     contents = MysqlHelper.get_values(log_type, source, select_sql, env, action=action)
     title_list = []
     filter_list = []
+    emoji_list = []
     for content in contents:
         config = content['config']
         config_dict = eval(config)
@@ -57,10 +61,16 @@ def get_config_from_mysql(log_type, source, env, text, action=''):
                 filter_list_config = v.split(",")
                 for filter_word in filter_list_config:
                     filter_list.append(filter_word)
+            if k == "emoji":
+                emoji_list_config = v.split(",")
+                for emoji in emoji_list_config:
+                    emoji_list.append(emoji)
     if text == "title":
         return title_list
     elif text == "filter":
         return filter_list
+    elif text == "emoji":
+        return emoji_list
 
 
 def random_title(log_type, crawler, env, text):
@@ -68,6 +78,37 @@ def random_title(log_type, crawler, env, text):
     return random.choice(random_title_list)
 
 
+def task_fun(task_str):
+    task_str = task_str.replace("'[{", '[{').replace("}}]'", '}}]')
+    task_dict = dict(eval(task_str))
+
+    rule = task_dict['rule']
+    task_dict['rule'] = dict()
+    for item in rule:
+        for k, val in item.items():
+            task_dict['rule'][k] = val
+    rule_dict = task_dict['rule']
+
+    task_dict = {
+        "task_dict": task_dict,
+        "rule_dict": rule_dict
+    }
+    return task_dict
+
 if __name__ == "__main__":
-    print(filter_word('public', 'xiaoniangao', '小年糕', 'prod'))
-    # get_config_from_mysql('author', 'xigua', 'prod')
+    # print(filter_word('public', 'xiaoniangao', '小年糕', 'prod'))
+    print(get_config_from_mysql('hour', 'xiaoniangao', 'dev', 'emoji'))
+    # task_str = "[('task_id','11')," \
+    #            "('task_name','小年糕小时榜')," \
+    #            "('source','xiaoniangao')," \
+    #            "('start_time','1681834560000')," \
+    #            "('interval','1'),('mode','hour')," \
+    #            "('rule','[{'duration':{'min':40,'max':0}},{'playCnt':{'min':4000,'max':0}},{'period':{'min':10,'max':0}},{'fans':{'min':0,'max':0}},{'videos':{'min':0,'max':0}},{'like':{'min':0,'max':0}},{'videoWidth':{'min':0,'max':0}},{'videoHeight':{'min':0,'max':0}}]')," \
+    #            "('spider_name','')," \
+    #            "('machine','')," \
+    #            "('status','0')," \
+    #            "('create_time','1681889875288')," \
+    #            "('update_time','1681889904908')," \
+    #            "('operator','王坤')]"
+    # print(task(task_str))
+    pass

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow.py

@@ -348,8 +348,8 @@ class GongzhonghaoFollow:
                     }
                     for k, v in video_dict.items():
                         Common.logger(log_type, crawler).info(f"{k}:{v}")
-                    if int(time.time()) - publish_time_stamp > 3600 * 24 * 3:
-                        Common.logger(log_type, crawler).info(f'发布时间{publish_time_str} > 3 天\n')
+                    if int(time.time()) - publish_time_stamp > 3600 * 24 * 2:
+                        Common.logger(log_type, crawler).info(f'发布时间{publish_time_str} > 2 天\n')
                         cls.begin = 0
                         return
                     cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)

scheduling/scheduling_v3/crawler_scheduling_v3.py

@@ -4,7 +4,6 @@
 import os
 import sys
 import time
-from hashlib import md5
 
 sys.path.append(os.getcwd())
 from common.common import Common

scheduling/scheduling_v3/demo.py

@@ -1,3 +1,105 @@
 # -*- coding: utf-8 -*-
 # @Author: wangkun
 # @Time: 2023/4/19
+import time
+from datetime import date, timedelta
+
+# import ast
+# task_str = "[('task_id','11')," \
+#            "('task_name','小年糕小时榜')," \
+#            "('source','xiaoniangao')," \
+#            "('start_time','1681834560000')," \
+#            "('interval','1'),('mode','hour')," \
+#            "('rule','[{'duration':{'min':40,'max':0}},{'playCnt':{'min':4000,'max':0}},{'period':{'min':10,'max':0}},{'fans':{'min':0,'max':0}},{'videos':{'min':0,'max':0}},{'like':{'min':0,'max':0}},{'videoWidth':{'min':0,'max':0}},{'videoHeight':{'min':0,'max':0}}]')," \
+#            "('spider_name','')," \
+#            "('machine','')," \
+#            "('status','0')," \
+#            "('create_time','1681889875288')," \
+#            "('update_time','1681889904908')," \
+#            "('operator','王坤')]"
+# task_str = task_str.replace("'[{", '[{').replace("}}]'", '}}]')
+# print(task_str)
+# task_list = eval(task_str)
+# print(task_list)
+# print(type(task_list))
+# task_dict = dict(task_list)
+# print(task_dict)
+# print(type(task_dict))
+#
+#
+# rule = task_dict['rule']
+# print(type(rule))
+# print(rule)
+# print(task_dict)
+# task_dict['rule'] = dict()
+# for item in rule:
+#     for k, val in item.items():
+#         task_dict['rule'][k] = val
+# print('\n')
+# print(task_dict['rule'])
+# print('\n')
+# print(task_dict)
+
+# str1 = task_str.split(",('rule',")[0]+"]"
+# print(type(str1))
+# print(str1)
+# eval1 = eval(str1)
+# print(type(eval1), eval1)
+# dict1 = dict(eval1)
+# print(type(dict1), dict1)
+# print("============\n")
+#
+# "[{'duration':{'min':40,'max':0}},{'playCnt':{'min':4000,'max':0}},{'period':{'min':10,'max':0}},{'fans':{'min':0,'max':0}},{'videos':{'min':0,'max':0}},{'like':{'min':0,'max':0}},{'videoWidth':{'min':0,'max':0}},{'videoHeight':{'min':0,'max':0}}]"
+# print(task_str.split(",('rule',")[-1].split(",('spider_name'")[0].replace(")", ""))
+# # rule_str = task_str.split(",('rule',")[-1].split(",('spider_name'")[0].replace(")", "").replace("'[{", '[{').replace("}}]'", '}}]')
+# rule_str = task_str.split(",('rule',")[-1].split(",('spider_name'")[0].replace(")", "")[1:-1]
+# print(type(rule_str))
+# print(rule_str)
+#
+#
+# rule_list = eval(rule_str)
+# print(type(rule_list))
+# print(rule_list)
+
+# rule_dict = {'duration': {'min': 40, 'max': 0}, 'playCnt': {'min': 4000, 'max': 0}, 'period': {'min': 10, 'max': 0}, 'fans': {'min': 0, 'max': 0}, 'videos': {'min': 0, 'max': 0}, 'like': {'min': 0, 'max': 0}, 'videoWidth': {'min': 0, 'max': 0}, 'videoHeight': {'min': 0, 'max': 0}}
+# rule_dict = {}
+#
+# for k, v in rule_dict.items():
+#     print(f"{k}:{v}")
+#
+# rule_duration_min = rule_dict.get('duration', {}).get('min', 0)
+# rule_duration_max = rule_dict.get('duration', {}).get('max', 100000000)
+# rule_playCnt_min = rule_dict.get('playCnt', {}).get('min', 0)
+# rule_playCnt_max = rule_dict.get('playCnt', {}).get('max', 100000000)
+# rule_period_min = rule_dict.get('period', {}).get('min', 0)
+# rule_period_max = rule_dict.get('period', {}).get('max', 100000000)
+# rule_fans_min = rule_dict.get('fans', {}).get('min', 0)
+# rule_fans_max = rule_dict.get('fans', {}).get('max', 100000000)
+# rule_videos_min = rule_dict.get('videos', {}).get('min', 0)
+# rule_videos_max = rule_dict.get('videos', {}).get('max', 100000000)
+# rule_like_min = rule_dict.get('like', {}).get('min', 0)
+# rule_like_max = rule_dict.get('like', {}).get('max', 100000000)
+# rule_videoWidth_min = rule_dict.get('videoWidth', {}).get('min', 0)
+# rule_videoWidth_max = rule_dict.get('videoWidth', {}).get('max', 100000000)
+# rule_videoHeight_min = rule_dict.get('videoWidth', {}).get('min', 0)
+# rule_videoHeight_max = rule_dict.get('videoWidth', {}).get('max', 100000000)
+#
+# print(f"rule_duration_min:{rule_duration_min}")
+# print(f"rule_duration_max:{rule_duration_max}")
+# print(f"rule_playCnt_min:{rule_playCnt_min}")
+# print(f"rule_playCnt_max:{rule_playCnt_max}")
+# print(f"rule_period_min:{rule_period_min}")
+# print(f"rule_period_max:{rule_period_max}")
+# print(f"rule_fans_min:{rule_fans_min}")
+# print(f"rule_fans_max:{rule_fans_max}")
+# print(f"rule_videos_min:{rule_videos_min}")
+# print(f"rule_videos_max:{rule_videos_max}")
+# print(f"rule_videoWidth_min:{rule_videoWidth_min}")
+# print(f"rule_videoWidth_max:{rule_videoWidth_max}")
+# print(f"rule_videoHeight_min:{rule_videoHeight_min}")
+# print(f"rule_videoHeight_max:{rule_videoHeight_max}")
+
+time_str = (date.today() + timedelta(days=-10)).strftime("%Y-%m-%d %H:%M:%S")
+time_stamp = int(time.mktime(time.strptime(time_str, "%Y-%m-%d %H:%M:%S")))
+print(time_str)
+print(time_stamp)

xiaoniangao/xiaoniangao_author/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/4/20

xiaoniangao/xiaoniangao_follow/xiaoniangao_follow_scheduling.py → xiaoniangao/xiaoniangao_author/xiaoniangao_author_scheduling.py

@@ -20,7 +20,7 @@ from common.public import filter_word
 proxies = {"http": None, "https": None}
 
 
-class XiaoniangaoFollow:
+class XiaoniangaoAuthorScheduling:
     platform = "小年糕"
     # 小程序个人主页视频列表翻页参数
     next_t = None
@@ -415,9 +415,6 @@ class XiaoniangaoFollow:
             Feishu.update_values(log_type, crawler, "Wu0CeL", "F2:Z2", values)
             Common.logger(log_type, crawler).info('视频信息写入飞书成功\n')
 
-        # except Exception as e:
-        #     Common.logger(log_type, crawler).error("下载/上传异常:{}", e)
-
     # 获取所有关注列表的用户视频
     @classmethod
     def get_follow_videos(cls, log_type, crawler, strategy, oss_endpoint, env):
@@ -441,7 +438,7 @@ class XiaoniangaoFollow:
 
 
 if __name__ == "__main__":
-    # print(XiaoniangaoFollow.repeat_video("follow", "xiaoniangao", "4919087666", "prod", "aliyun"))
-    # print(XiaoniangaoFollow.repeat_video("follow", "xiaoniangao", "4919087666", "dev"))
-    XiaoniangaoFollow.get_users()
+    # print(XiaoniangaoAuthorScheduling.repeat_video("follow", "xiaoniangao", "4919087666", "prod", "aliyun"))
+    # print(XiaoniangaoAuthorScheduling.repeat_video("follow", "xiaoniangao", "4919087666", "dev"))
+    XiaoniangaoAuthorScheduling.get_users()
     pass

xiaoniangao/xiaoniangao_hour/xiaoniangao_hour_scheduling.py

@@ -10,18 +10,16 @@ import sys
 import time
 import requests
 import urllib3
-
 sys.path.append(os.getcwd())
 from common.common import Common
 from common.feishu import Feishu
 from common.publish import Publish
 from common.scheduling_db import MysqlHelper
-from common.public import filter_word
-
+from common.public import get_config_from_mysql
 proxies = {"http": None, "https": None}
 
 
-class XiaoniangaoHour:
+class XiaoniangaoHourScheduling:
     platform = "小年糕"
 
     # 生成 uid、token
@@ -38,45 +36,76 @@ class XiaoniangaoHour:
 
     # 基础门槛规则
     @staticmethod
-    def download_rule(video_dict):
+    def download_rule(log_type, crawler, video_dict, rule_dict):
         """
         下载视频的基本规则
+        :param log_type: 日志
+        :param crawler: 哪款爬虫
         :param video_dict: 视频信息，字典格式
+        :param rule_dict: 规则信息，字典格式
         :return: 满足规则，返回 True；反之，返回 False
         """
-        # 视频时长
-        if int(float(video_dict["duration"])) >= 40:
-            # 宽或高
-            if int(video_dict["video_width"]) >= 0 or int(video_dict["video_height"]) >= 0:
-                # 播放量
-                if int(video_dict["play_cnt"]) >= 4000:
-                    # 点赞量
-                    if int(video_dict["like_cnt"]) >= 0:
-                        # 分享量
-                        if int(video_dict["share_cnt"]) >= 0:
-                            # 发布时间 <= 10 天
-                            if int(time.time()) - int(video_dict["publish_time_stamp"]) <= 3600 * 24 * 10:
-                                return True
-                            else:
-                                return False
-                        else:
-                            return False
-                    else:
-                        return False
-                else:
-                    return False
+        # rule_period_max = rule_dict.get('period', {}).get('max', 100000000)
+        # rule_fans_min = rule_dict.get('fans', {}).get('min', 0)
+        # rule_fans_max = rule_dict.get('fans', {}).get('max', 100000000)
+        # rule_videos_min = rule_dict.get('videos', {}).get('min', 0)
+        # rule_videos_max = rule_dict.get('videos', {}).get('max', 100000000)
+        rule_duration_min = rule_dict.get('duration', {}).get('min', 0)
+        rule_duration_max = rule_dict.get('duration', {}).get('max', 100000000)
+        if rule_duration_max == 0:
+            rule_duration_max = 100000000
+        rule_playCnt_min = rule_dict.get('playCnt', {}).get('min', 0)
+        rule_playCnt_max = rule_dict.get('playCnt', {}).get('max', 100000000)
+        if rule_playCnt_max == 0:
+            rule_playCnt_max = 100000000
+        rule_period_min = rule_dict.get('period', {}).get('min', 0)
+        rule_like_min = rule_dict.get('like', {}).get('min', 0)
+        rule_like_max = rule_dict.get('like', {}).get('max', 100000000)
+        if rule_like_max == 0:
+            rule_like_max = 100000000
+        rule_videoWidth_min = rule_dict.get('videoWidth', {}).get('min', 0)
+        rule_videoWidth_max = rule_dict.get('videoWidth', {}).get('max', 100000000)
+        if rule_videoWidth_max == 0:
+            rule_videoWidth_max = 100000000
+        rule_videoHeight_min = rule_dict.get('videoWidth', {}).get('min', 0)
+        rule_videoHeight_max = rule_dict.get('videoWidth', {}).get('max', 100000000)
+        if rule_videoHeight_max == 0:
+            rule_videoHeight_max = 100000000
+        Common.logger(log_type, crawler).info(f'rule_duration_max:{rule_duration_max} >= duration:{int(float(video_dict["duration"]))} >= rule_duration_min:{int(rule_duration_min)}')
+        Common.logger(log_type, crawler).info(f'rule_playCnt_max:{int(rule_playCnt_max)} >= play_cnt:{int(video_dict["play_cnt"])} >= rule_playCnt_min:{int(rule_playCnt_min)}')
+        Common.logger(log_type, crawler).info(f'now:{int(time.time())} - publish_time_stamp:{int(video_dict["publish_time_stamp"])} <= {3600 * 24 * int(rule_period_min)}')
+        Common.logger(log_type, crawler).info(f'rule_like_max:{int(rule_like_max)} >= like_cnt:{int(video_dict["like_cnt"])} >= rule_like_min:{int(rule_like_min)}')
+        Common.logger(log_type, crawler).info(f'rule_videoWidth_max:{int(rule_videoWidth_max)} >= video_width:{int(video_dict["video_width"])} >= rule_videoWidth_min:{int(rule_videoWidth_min)}')
+        Common.logger(log_type, crawler).info(f'rule_videoHeight_max:{int(rule_videoHeight_max)} >= video_height:{int(video_dict["video_height"])} >= rule_videoHeight_min:{int(rule_videoHeight_min)}')
+        if int(rule_duration_max) >= int(float(video_dict["duration"])) >= int(rule_duration_min) \
+                and int(rule_playCnt_max) >= int(video_dict['play_cnt']) >= int(rule_playCnt_min) \
+                and int(time.time()) - int(video_dict["publish_time_stamp"]) <= 3600 * 24 * int(rule_period_min)\
+                and int(rule_like_max) >= int(video_dict['like_cnt']) >= int(rule_like_min)\
+                and int(rule_videoWidth_max) >= int(video_dict['video_width']) >= int(rule_videoWidth_min)\
+                and int(rule_videoHeight_max) >= int(video_dict['video_height']) >= int(rule_videoHeight_min):
+            return True
+        else:
             return False
-        return False
 
-    # 获取表情及符号
-    @classmethod
-    def get_expression(cls):
-        # 表情列表
-        expression_list = ['📍', '⭕️', '🔥', '📣', '🎈', '⚡', '🔔', '🚩', '💢', '💎', '👉', '💓', '❗️', '🔴', '🔺', '♦️', '♥️', '👉',
-                           '👈', '🏆', '❤️\u200d🔥']
-        # 符号列表
-        char_list = ['...', '~~']
-        return expression_list, char_list
+        # if int(rule_duration_max) >= int(float(video_dict["duration"])) >= int(rule_duration_min):
+        #     if int(rule_playCnt_max) >= int(video_dict['play_cnt']) >= 0:
+        #         if int(time.time()) - int(video_dict["publish_time_stamp"]) <= 3600 * 24 * 365:
+        #             if int(rule_like_max) >= int(video_dict['like_cnt']) >= int(rule_like_min):
+        #                 if int(rule_videoWidth_max) >= int(video_dict['video_width']) >= int(rule_videoWidth_min):
+        #                     if int(rule_videoHeight_max) >= int(video_dict['video_height']) >= int(rule_videoHeight_min):
+        #                         return True
+        #                     else:
+        #                         return False
+        #                 else:
+        #                     return False
+        #             else:
+        #                 return False
+        #         else:
+        #             return False
+        #     else:
+        #         return False
+        # else:
+        #     return False
 
     @classmethod
     def repeat_video(cls, log_type, crawler, video_id, env):
@@ -92,23 +121,18 @@ class XiaoniangaoHour:
 
     # 获取列表
     @classmethod
-    def get_videoList(cls, log_type, crawler, env):
-        # try:
+    def get_videoList(cls, log_type, crawler, rule_dict, env):
         uid_token_dict = cls.get_uid_token()
         url = "https://kapi.xiaoniangao.cn/trends/get_recommend_trends"
         headers = {
-            # "x-b3-traceid": cls.hour_x_b3_traceid,
             "x-b3-traceid": '1c403a4aa72e3c',
-            # "X-Token-Id": cls.hour_x_token_id,
             "X-Token-Id": 'ab619e96d801f1567388629260aa68ec-1202200806',
-            # "uid": cls.hour_uid,
             "uid": uid_token_dict['uid'],
             "content-type": "application/json",
             "Accept-Encoding": "gzip,compress,br,deflate",
             "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
                           ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
                           'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
-            # "Referer": cls.hour_referer
             "Referer": 'https://servicewechat.com/wxd7911e4c177690e4/624/page-frame.html'
         }
         data = {
@@ -181,127 +205,63 @@ class XiaoniangaoHour:
             feeds = r.json()["data"]["list"]
             for i in range(len(feeds)):
                 # 标题，表情随机加在片头、片尾，或替代句子中间的标点符号
-                if "title" in feeds[i]:
-                    befor_video_title = feeds[i]["title"].strip().replace("\n", "") \
+                xiaoniangao_title = feeds[i].get("title", "").strip().replace("\n", "") \
                         .replace("/", "").replace("\r", "").replace("#", "") \
                         .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
                         .replace(":", "").replace("*", "").replace("？", "") \
                         .replace("?", "").replace('"', "").replace("<", "") \
-                        .replace(">", "").replace("|", "").replace(" ", "").replace("#表情", "").replace("#符号","").replace(
-                        '"', '').replace("'", '').replace('"', '').replace("'", '')
-
-                    expression = cls.get_expression()
-                    expression_list = expression[0]
-                    char_list = expression[1]
-                    # 随机取一个表情
-                    expression = random.choice(expression_list)
-                    # 生成标题list[表情+title, title+表情]
-                    expression_title_list = [expression + befor_video_title, befor_video_title + expression]
-                    # 从标题list中随机取一个标题
-                    title_list1 = random.choice(expression_title_list)
-                    # 生成标题:原标题+符号
-                    title_list2 = befor_video_title + random.choice(char_list)
-                    # 表情和标题组合，与标题和符号组合，汇总成待使用的标题列表
-                    title_list4 = [title_list2, title_list1]
-                    # 最终标题
-                    video_title = random.choice(title_list4)
-                else:
-                    video_title = 0
-
+                        .replace(">", "").replace("|", "").replace(" ", "")\
+                        .replace('"', '').replace("'", '')
+                # 随机取一个表情/符号
+                emoji = random.choice(get_config_from_mysql(log_type, crawler, env, "emoji"))
+                # 生成最终标题，标题list[表情+title, title+表情]随机取一个
+                video_title = random.choice([f"{emoji}{xiaoniangao_title}", f"{xiaoniangao_title}{emoji}"])
                 # 视频 ID
-                if "vid" in feeds[i]:
-                    video_id = feeds[i]["vid"]
-                else:
-                    video_id = 0
-
+                video_id = feeds[i].get("vid", "")
                 # 播放量
-                if "play_pv" in feeds[i]:
-                    video_play_cnt = feeds[i]["play_pv"]
-                else:
-                    video_play_cnt = 0
-
+                play_cnt = feeds[i].get("play_pv", 0)
                 # 点赞量
-                if "favor" in feeds[i]:
-                    video_like_cnt = feeds[i]["favor"]["total"]
-                else:
-                    video_like_cnt = 0
-
+                like_cnt = feeds[i].get("favor", {}).get("total", 0)
                 # 评论数
-                if "comment_count" in feeds[i]:
-                    video_comment_cnt = feeds[i]["comment_count"]
-                else:
-                    video_comment_cnt = 0
-
+                comment_cnt = feeds[i].get("comment_count", 0)
                 # 分享量
-                if "share" in feeds[i]:
-                    video_share_cnt = feeds[i]["share"]
-                else:
-                    video_share_cnt = 0
-
+                share_cnt = feeds[i].get("share", 0)
                 # 时长
-                if "du" in feeds[i]:
-                    video_duration = int(feeds[i]["du"] / 1000)
-                else:
-                    video_duration = 0
-
+                duration = int(feeds[i].get("du", 0)/1000)
                 # 宽和高
-                if "w" or "h" in feeds[i]:
-                    video_width = feeds[i]["w"]
-                    video_height = feeds[i]["h"]
-                else:
-                    video_width = 0
-                    video_height = 0
-
+                video_width = int(feeds[i].get("w", 0))
+                video_height = int(feeds[i].get("h", 0))
                 # 发布时间
-                if "t" in feeds[i]:
-                    video_send_time = feeds[i]["t"]
-                else:
-                    video_send_time = 0
-                publish_time_stamp = int(int(video_send_time) / 1000)
+                publish_time_stamp = int(int(feeds[i].get("t", 0))/1000)
                 publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp))
-
                 # 用户名 / 头像
-                if "user" in feeds[i]:
-                    user_name = feeds[i]["user"]["nick"].strip().replace("\n", "") \
+                user_name = feeds[i].get("user", {}).get("nick", "").strip().replace("\n", "") \
                         .replace("/", "").replace("快手", "").replace(" ", "") \
                         .replace(" ", "").replace("&NBSP", "").replace("\r", "")
-                    head_url = feeds[i]["user"]["hurl"]
-                else:
-                    user_name = 0
-                    head_url = 0
-
+                avatar_url = feeds[i].get("user", {}).get("hurl", "")
                 # 用户 ID
                 profile_id = feeds[i]["id"]
-
                 # 用户 mid
                 profile_mid = feeds[i]["user"]["mid"]
-
                 # 视频封面
-                if "url" in feeds[i]:
-                    cover_url = feeds[i]["url"]
-                else:
-                    cover_url = 0
-
+                cover_url = feeds[i].get("url", "")
                 # 视频播放地址
-                if "v_url" in feeds[i]:
-                    video_url = feeds[i]["v_url"]
-                else:
-                    video_url = 0
+                video_url = feeds[i].get("v_url", "")
 
                 video_dict = {
                     "video_title": video_title,
                     "video_id": video_id,
-                    "duration": video_duration,
-                    "play_cnt": video_play_cnt,
-                    "like_cnt": video_like_cnt,
-                    "comment_cnt": video_comment_cnt,
-                    "share_cnt": video_share_cnt,
+                    "duration": duration,
+                    "play_cnt": play_cnt,
+                    "like_cnt": like_cnt,
+                    "comment_cnt": comment_cnt,
+                    "share_cnt": share_cnt,
                     "user_name": user_name,
                     "publish_time_stamp": publish_time_stamp,
                     "publish_time_str": publish_time_str,
                     "video_width": video_width,
                     "video_height": video_height,
-                    "avatar_url": head_url,
+                    "avatar_url": avatar_url,
                     "profile_id": profile_id,
                     "profile_mid": profile_mid,
                     "cover_url": cover_url,
@@ -312,20 +272,16 @@ class XiaoniangaoHour:
                     Common.logger(log_type, crawler).info(f"{k}:{v}")
 
                 # 过滤无效视频
-                if video_title == 0 or video_id == 0 or video_duration == 0 \
-                        or video_send_time == 0 or user_name == 0 or head_url == 0 \
-                        or cover_url == 0 or video_url == 0:
+                if video_title == "" or video_id == "" or video_url == "":
                     Common.logger(log_type, crawler).warning("无效视频\n")
                 # 抓取基础规则过滤
-                elif cls.download_rule(video_dict) is False:
-                    Common.logger(log_type, crawler).info("不满足基础门槛规则\n")
+                elif cls.download_rule(log_type, crawler, video_dict, rule_dict) is False:
+                    Common.logger(log_type, crawler).info("不满足抓取规则\n")
                 elif cls.repeat_video(log_type, crawler, video_dict['video_id'], env) != 0:
                     Common.logger(log_type, crawler).info('视频已下载\n')
                 # 过滤敏感词
-                elif any(str(word) if str(word) in video_title else False for word in
-                         filter_word(log_type, crawler, "小年糕", env)) is True:
+                elif any(str(word) if str(word) in video_title else False for word in get_config_from_mysql(log_type, crawler, env, "filter", action="")) is True:
                     Common.logger(log_type, crawler).info("视频已中过滤词\n")
-                    time.sleep(1)
                 else:
                     # 写入飞书小时级feeds数据库表
                     insert_sql = f""" insert into crawler_xiaoniangao_hour(profile_id,
@@ -349,27 +305,22 @@ class XiaoniangaoHour:
                     "{user_name}",
                     "{cover_url}",
                     "{video_url}",
-                    {video_duration},
+                    {duration},
                     "{publish_time_str}",
-                    {video_play_cnt},
+                    {play_cnt},
                     {int(time.time())},
                     "{time.strftime("%Y-%y-%d %H:%M:%S", time.localtime(int(time.time())))}"
                     )"""
                     Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
                     MysqlHelper.update_values(log_type, crawler, insert_sql, env)
                     Common.logger(log_type, crawler).info('视频信息插入数据库成功！\n')
-        # except Exception as e:
-        #     Common.logger(log_type, crawler).error(f"get_videoList:{e}\n")
 
     @classmethod
     def get_video_info(cls, log_type, crawler, p_id, p_mid, v_title, v_id):
-        # try:
         uid_token_dict = cls.get_uid_token()
         url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id"
         headers = {
-            # "x-b3-traceid": cls.hour_x_b3_traceid,
             "x-b3-traceid": '1c403a4aa72e3c',
-            # "X-Token-Id": cls.hour_x_token_id,
             "X-Token-Id": 'ab619e96d801f1567388629260aa68ec-1202200806',
             "uid": uid_token_dict['uid'],
             "content-type": "application/json",
@@ -377,7 +328,6 @@ class XiaoniangaoHour:
             "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
                           ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
                           'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
-            # "Referer": cls.hour_referer
             "Referer": 'https://servicewechat.com/wxd7911e4c177690e4/624/page-frame.html'
         }
         data = {
@@ -394,9 +344,7 @@ class XiaoniangaoHour:
             "no_follow": True,
             "vid": v_id,
             "hot_l1_comment": True,
-            # "token": cls.hour_token,
             "token": uid_token_dict['token'],
-            # "uid": cls.hour_uid,
             "uid": uid_token_dict['uid'],
             "proj": "ma",
             "wx_ver": "8.0.20",
@@ -462,16 +410,12 @@ class XiaoniangaoHour:
             }
             return video_info_dict
 
-        # except Exception as e:
-        #     Common.logger(log_type, crawler).error(f"download_video:{e}\n")
-
     # 更新小时榜数据
     @classmethod
     def update_videoList(cls, log_type, crawler, strategy, oss_endpoint, env):
         """
         更新小时榜数据
         """
-        # try:
         befor_yesterday = (datetime.date.today() + datetime.timedelta(days=-3)).strftime("%Y-%m-%d %H:%M:%S")
         update_time_stamp = int(time.mktime(time.strptime(befor_yesterday, "%Y-%m-%d %H:%M:%S")))
         select_sql = f""" select * from crawler_xiaoniangao_hour where crawler_time_stamp >= {update_time_stamp} GROUP BY out_video_id """
@@ -528,8 +472,6 @@ class XiaoniangaoHour:
                                      env)
             else:
                 pass
-        # except Exception as e:
-        #     Common.logger(log_type, crawler).error(f"update_videoList:{e}\n")
 
     @classmethod
     def download(cls, log_type, crawler, video_info_dict, strategy, oss_endpoint, env):
@@ -629,7 +571,6 @@ class XiaoniangaoHour:
     # 下载/上传
     @classmethod
     def download_publish(cls, log_type, crawler, video_info_dict, update_video_info, strategy, oss_endpoint, env):
-        # try:
         if cls.repeat_video(log_type, crawler, video_info_dict["video_id"], env) != 0:
             Common.logger(log_type, crawler).info('视频已下载\n')
         # 播放量大于 50000，直接下载
@@ -666,12 +607,10 @@ class XiaoniangaoHour:
 
         else:
             Common.logger(log_type, crawler).info("上升量不满足下载规则")
-    # except Exception as e:
-    #     Common.logger(log_type, crawler).error(f"download_publish:{e}\n")
 
 
 if __name__ == "__main__":
-    print(XiaoniangaoHour.get_expression())
+    print(get_config_from_mysql(log_type='hour', source='xiaoniangao', env='dev', text='filter'))
     # print(XiaoniangaoHour.get_uid_token())
     # XiaoniangaoHour.get_videoList("test", "xiaoniangao", "dev")
     # XiaoniangaoHour.update_videoList("test", "xiaoniangao", "小时榜爬虫策略", "out", "dev")

xiaoniangao/xiaoniangao_main/run_xiaoniangao_author_scheduling.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/4/20
+import argparse
+import os
+import sys
+sys.path.append(os.getcwd())
+from common.public import task_fun
+from common.common import Common
+from xiaoniangao.xiaoniangao_author.xiaoniangao_author_scheduling import XiaoniangaoAuthorScheduling
+
+
+def main(log_type, crawler, task, oss_endpoint, env):
+    task_dict = task_fun(task)['task_dict']
+    rule_dict = task_fun(task)['rule_dict']
+    Common.logger(log_type, crawler).info(f"调度任务:\n{task_dict}")
+    Common.logger(log_type, crawler).info(f"抓取规则:\n{rule_dict}")
+    Common.logger(log_type, crawler).info('开始抓取 小年糕 定向榜\n')
+    XiaoniangaoAuthorScheduling.get_follow_videos(log_type=log_type,
+                                        crawler=crawler,
+                                        strategy="定向爬虫策略",
+                                        oss_endpoint=oss_endpoint,
+                                        env=env)
+    Common.del_logs(log_type, crawler)
+    Common.logger(log_type, crawler).info('抓取完一轮\n')
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()  ## 新建参数解释器对象
+    parser.add_argument('--log_type', type=str)  ## 添加参数,注明参数类型
+    parser.add_argument('--crawler')  ## 添加参数
+    parser.add_argument('--task')  ## 添加参数
+    parser.add_argument('--oss_endpoint')  ## 添加参数
+    parser.add_argument('--env')  ## 添加参数
+    args = parser.parse_args()  ### 参数赋值，也可以通过终端赋值
+    main(log_type=args.log_type,
+         crawler=args.crawler,
+         task=args.task,
+         oss_endpoint=args.oss_endpoint,
+         env=args.env)

xiaoniangao/xiaoniangao_main/run_xiaoniangao_hour_scheduling.py

@@ -7,43 +7,42 @@ import os
 import sys
 sys.path.append(os.getcwd())
 from common.common import Common
-from xiaoniangao.xiaoniangao_hour.xiaoniangao_hour_scheduling import XiaoniangaoHour
+from common.public import task_fun
+from xiaoniangao.xiaoniangao_hour.xiaoniangao_hour_scheduling import XiaoniangaoHourScheduling
 
 
 def main(log_type, crawler, task, oss_endpoint, env):
-    Common.logger(log_type, crawler).info(f"{type(task)}:{task}")
-    task = eval(task)
-    Common.logger(log_type, crawler).info(f"{type(task)}\n")
-    Common.logger(log_type, crawler).info(f"{task}\n")
-    Common.logger(log_type, crawler).info(f"{oss_endpoint}")
-    Common.logger(log_type, crawler).info(f"{env}")
+    task_dict = task_fun(task)['task_dict']
+    rule_dict = task_fun(task)['rule_dict']
+    Common.logger(log_type, crawler).info(f"调度任务:\n{task_dict}")
+    Common.logger(log_type, crawler).info(f"抓取规则:\n{rule_dict}")
     # 获取符合规则的视频，写入小时级数据_feeds
-    # XiaoniangaoHour.get_videoList(log_type, crawler, env)
-    # now = datetime.datetime.now()
-    # if now.hour == 10 and 0 <= now.minute <= 10:
-    #     Common.logger(log_type, crawler).info("开始更新/下载上升榜")
-    #     XiaoniangaoHour.update_videoList(log_type=log_type,
-    #                                      crawler=crawler,
-    #                                      strategy="小时榜爬虫策略",
-    #                                      oss_endpoint=oss_endpoint,
-    #                                      env=env)
-    #
-    # elif now.hour == 15 and now.minute <= 10:
-    #     Common.logger(log_type, crawler).info("开始更新/下载上升榜")
-    #     XiaoniangaoHour.update_videoList(log_type=log_type,
-    #                                      crawler=crawler,
-    #                                      strategy="小时榜爬虫策略",
-    #                                      oss_endpoint=oss_endpoint,
-    #                                      env=env)
-    #
-    # elif now.hour == 20 and now.minute <= 10:
-    #     Common.logger(log_type, crawler).info("开始更新/下载上升榜")
-    #     XiaoniangaoHour.update_videoList(log_type=log_type,
-    #                                      crawler=crawler,
-    #                                      strategy="小时榜爬虫策略",
-    #                                      oss_endpoint=oss_endpoint,
-    #                                      env=env)
-    # Common.del_logs(log_type, crawler)
+    XiaoniangaoHourScheduling.get_videoList(log_type, crawler, rule_dict, env)
+    now = datetime.datetime.now()
+    if now.hour == 10 and 0 <= now.minute <= 10:
+        Common.logger(log_type, crawler).info("开始更新/下载上升榜")
+        XiaoniangaoHourScheduling.update_videoList(log_type=log_type,
+                                         crawler=crawler,
+                                         strategy="小时榜爬虫策略",
+                                         oss_endpoint=oss_endpoint,
+                                         env=env)
+
+    elif now.hour == 15 and now.minute <= 10:
+        Common.logger(log_type, crawler).info("开始更新/下载上升榜")
+        XiaoniangaoHourScheduling.update_videoList(log_type=log_type,
+                                         crawler=crawler,
+                                         strategy="小时榜爬虫策略",
+                                         oss_endpoint=oss_endpoint,
+                                         env=env)
+
+    elif now.hour == 20 and now.minute <= 10:
+        Common.logger(log_type, crawler).info("开始更新/下载上升榜")
+        XiaoniangaoHourScheduling.update_videoList(log_type=log_type,
+                                         crawler=crawler,
+                                         strategy="小时榜爬虫策略",
+                                         oss_endpoint=oss_endpoint,
+                                         env=env)
+    Common.del_logs(log_type, crawler)
 
 
 if __name__ == "__main__":