# -*- coding: utf-8 -*- # @Author: wangkun # @Time: 2023/3/27 import os, sys import random sys.path.append(os.getcwd()) from common.common import Common from common.scheduling_db import MysqlHelper # from common import Common # from scheduling_db import MysqlHelper # 过滤词库 def filter_word(log_type, crawler, source, env): """ 过滤词库 :param log_type: 日志 :param crawler: 哪款爬虫,如:xiaoniangao :param source: 哪款爬虫,如:小年糕 :param env: 环境 :return: word_list """ select_sql = f""" select * from crawler_filter_word where source="{source}" """ words = MysqlHelper.get_values(log_type, crawler, select_sql, env, action='') word_list = [] if len(words) == 0: return word_list for word in words: word_list.append(word['filter_word']) return word_list def get_user_from_mysql(log_type, crawler, source, env, action=''): sql = f"select * from crawler_user_v3 where source='{source}' and mode='{log_type}'" results = MysqlHelper.get_values(log_type, crawler, sql, env, action=action) if results: return results else: Common.logger(log_type, crawler).warning(f"爬虫:{crawler},没有查到抓取名单") return [] def get_config_from_mysql(log_type, source, env, text, action=''): select_sql = f"""select * from crawler_config where source="{source}" """ contents = MysqlHelper.get_values(log_type, source, select_sql, env, action=action) title_list = [] filter_list = [] emoji_list = [] for content in contents: config = content['config'] config_dict = eval(config) for k, v in config_dict.items(): if k == "title": title_list_config = v.split(",") for title in title_list_config: title_list.append(title) if k == "filter": filter_list_config = v.split(",") for filter_word in filter_list_config: filter_list.append(filter_word) if k == "emoji": emoji_list_config = v.split(",") for emoji in emoji_list_config: emoji_list.append(emoji) if text == "title": return title_list elif text == "filter": return filter_list elif text == "emoji": return emoji_list def random_title(log_type, crawler, env, text): random_title_list = get_config_from_mysql(log_type, crawler, env, text) return random.choice(random_title_list) def task_fun(task_str): task_str = task_str.replace("'[{", '[{').replace("}}]'", '}}]') task_dict = dict(eval(task_str)) rule = task_dict['rule'] task_dict['rule'] = dict() for item in rule: for k, val in item.items(): task_dict['rule'][k] = val rule_dict = task_dict['rule'] task_dict = { "task_dict": task_dict, "rule_dict": rule_dict } return task_dict if __name__ == "__main__": # print(filter_word('public', 'xiaoniangao', '小年糕', 'prod')) print(get_config_from_mysql('hour', 'xiaoniangao', 'dev', 'emoji')) # task_str = "[('task_id','11')," \ # "('task_name','小年糕小时榜')," \ # "('source','xiaoniangao')," \ # "('start_time','1681834560000')," \ # "('interval','1'),('mode','hour')," \ # "('rule','[{'duration':{'min':40,'max':0}},{'playCnt':{'min':4000,'max':0}},{'period':{'min':10,'max':0}},{'fans':{'min':0,'max':0}},{'videos':{'min':0,'max':0}},{'like':{'min':0,'max':0}},{'videoWidth':{'min':0,'max':0}},{'videoHeight':{'min':0,'max':0}}]')," \ # "('spider_name','')," \ # "('machine','')," \ # "('status','0')," \ # "('create_time','1681889875288')," \ # "('update_time','1681889904908')," \ # "('operator','王坤')]" # print(task(task_str)) pass