import json import os import sys import time from datetime import date, timedelta sys.path.append(os.getcwd()) from common.scheduling_db import MysqlHelper from common.feishu import Feishu class Demo: @classmethod def get_config(cls, log_type, crawler, text, env): select_sql = f"""select * from crawler_config where source="xigua" """ contents = MysqlHelper.get_values(log_type, crawler, select_sql, env, action='') title_list = [] filter_list = [] for content in contents: config = content['config'] config_dict = eval(config) for k, v in config_dict.items(): if k == "title": title_list_config = v.split(",") for title in title_list_config: title_list.append(title) if k == "filter": filter_list_config = v.split(",") for filter_word in filter_list_config: filter_list.append(filter_word) if text == "title": return title_list elif text == "filter": return filter_list @classmethod def before_day(cls): publish_time_str_rule = (date.today() + timedelta(days=-30)).strftime("%Y-%m-%d %H:%M:%S") publish_time_stamp_rule = int(time.mktime(time.strptime(publish_time_str_rule, "%Y-%m-%d %H:%M:%S"))) print(publish_time_str_rule) print(publish_time_stamp_rule) @classmethod def insert_config(cls, log_type, crawler, env): filter_sheet = Feishu.get_values_batch(log_type, crawler, "KGB4Hc") title_sheet = Feishu.get_values_batch(log_type, crawler, "bHSW1p") filter_list = [] title_list = [] for x in filter_sheet: for y in x: if y is None: pass else: filter_list.append(y) for x in title_sheet: for y in x: if y is None: pass else: title_list.append(y) str_title = ','.join(title_list) str_filter = ','.join(filter_list) config_dict = { "title": str_title, "filter": str_filter } str_config_dict = str(config_dict) # print(f"config_dict:{config_dict}") # print(f"str_config_dict:{str_config_dict}") insert_sql = f""" insert into crawler_config(title, source, config) values("西瓜视频", "xigua", "{str_config_dict}") """ MysqlHelper.update_values(log_type, crawler, insert_sql, env) if __name__ == "__main__": # Demo.get_config("demo", "xiaoniangao", "dev") # Demo.before_day() Demo.insert_config("demo", "xigua", "prod") pass