|
|
@@ -224,12 +224,12 @@ class DyFollow(object):
|
|
|
@classmethod
|
|
|
def download_publish(cls, log_type, crawler, strategy, video_dict, rule_dict, our_uid, oss_endpoint, env, machine):
|
|
|
try:
|
|
|
- # if cls.download_rule(video_dict, rule_dict) is False:
|
|
|
- # Common.logger(log_type, crawler).info('不满足抓取规则\n')
|
|
|
- # elif any(word if word in video_dict['video_title'] else False for word in
|
|
|
- # cls.filter_words(log_type, crawler)) is True:
|
|
|
- # Common.logger(log_type, crawler).info('标题已中过滤词:{}\n', video_dict['video_title'])
|
|
|
- if cls.repeat_video(log_type, crawler, video_dict['video_id'], env, machine) != 0:
|
|
|
+ if cls.download_rule(video_dict, rule_dict) is False:
|
|
|
+ Common.logger(log_type, crawler).info('不满足抓取规则\n')
|
|
|
+ elif any(word if word in video_dict['video_title'] else False for word in
|
|
|
+ cls.filter_words(log_type, crawler)) is True:
|
|
|
+ Common.logger(log_type, crawler).info('标题已中过滤词:{}\n', video_dict['video_title'])
|
|
|
+ elif cls.repeat_video(log_type, crawler, video_dict['video_id'], env, machine) != 0:
|
|
|
Common.logger(log_type, crawler).info('视频已下载\n')
|
|
|
else:
|
|
|
# 下载视频
|
|
|
@@ -346,7 +346,7 @@ class DyFollow(object):
|
|
|
@classmethod
|
|
|
def get_follow_videos(cls, log_type, crawler, strategy, oss_endpoint, env, machine):
|
|
|
user_list = get_user_from_mysql(log_type, crawler, crawler, env)
|
|
|
- rule_dict = {}#cls.get_rule(log_type, crawler)
|
|
|
+ rule_dict = cls.get_rule(log_type, crawler)
|
|
|
for user in user_list:
|
|
|
spider_link = user["spider_link"]
|
|
|
out_uid = spider_link
|
