|
|
@@ -2,6 +2,7 @@
|
|
|
# @Author: wangkun
|
|
|
# @Time: 2023/3/27
|
|
|
import os, sys
|
|
|
+import random
|
|
|
|
|
|
sys.path.append(os.getcwd())
|
|
|
from common.common import Common
|
|
|
@@ -39,5 +40,34 @@ def get_user_from_mysql(log_type, crawler, source, env, action=''):
|
|
|
return []
|
|
|
|
|
|
|
|
|
+def get_config_from_mysql(log_type, source, env, text, action=''):
|
|
|
+ select_sql = f"""select * from crawler_config where source='{source}' """
|
|
|
+ contents = MysqlHelper.get_values(log_type, source, select_sql, env, action=action)
|
|
|
+ title_list = []
|
|
|
+ filter_list = []
|
|
|
+ for content in contents:
|
|
|
+ config = content['config']
|
|
|
+ config_dict = eval(config)
|
|
|
+ for k, v in config_dict.items():
|
|
|
+ if k == "title":
|
|
|
+ title_list_config = v.split(",")
|
|
|
+ for title in title_list_config:
|
|
|
+ title_list.append(title)
|
|
|
+ if k == "filter":
|
|
|
+ filter_list_config = v.split(",")
|
|
|
+ for filter_word in filter_list_config:
|
|
|
+ filter_list.append(filter_word)
|
|
|
+ if text == "title":
|
|
|
+ return title_list
|
|
|
+ elif text == "filter":
|
|
|
+ return filter_list
|
|
|
+
|
|
|
+
|
|
|
+def random_title(log_type, crawler, env, text):
|
|
|
+ random_title_list = get_config_from_mysql(log_type, crawler, env, text)
|
|
|
+ return random.choice(random_title_list)
|
|
|
+
|
|
|
+
|
|
|
if __name__ == "__main__":
|
|
|
print(filter_word('public', 'xiaoniangao', '小年糕', 'prod'))
|
|
|
+ # get_config_from_mysql('author', 'xigua', 'prod')
|
