""" @author: luojunhui """ import datetime from tqdm import tqdm from applications import AIDTApi, DeNetMysql, PQMySQL, Functions, longArticlesMySQL from config import poolTagMap from stratrgy import ArticlePoolStrategy class SendToMultiLevels(object): """ 流量池任务 """ AidApi = AIDTApi() DeMysql = DeNetMysql() PqMysql = PQMySQL() Fun = Functions() # Ad = @classmethod def getYesterdayData(cls): """ 获取前一天数据表现 :return: """ sql = f""" select article_id, read_count from changwen_article_datastat where article_id in ( select id from changwen_article where publish_timestamp >= 1722268800000 ) and read_count > 100; """ result = cls.Ad.select(sql=sql) response_list = [ { "id": line[0], "read_count": line[1] } for line in result ] return response_list @classmethod def splitToDifferentPools(cls, yesterday_data): """ 分类至Pools :return: """ S = ArticlePoolStrategy() detail_list = S.getData(article_list=yesterday_data) result = S.splitByStrategy(detail_list=detail_list) return result @classmethod def sendToEachCrawlerPlan(cls, key, result_list): """ :param result_list: :param key: :return: """ # print(key) # print(len(result_list)) # for index, i in enumerate(result_list): # print(index, "\t", i['level_rate'], "\t", i['title'], "\t", i['avg_read'], "\t", i['article_read'], "\t", i['key']) # print(url_list) # daily自动创建新抓取计划 cls.AidApi.updateArticleIntoCrawlerPlan( plan_id=None, plan_name="流量池晋级--{}--{}".format(datetime.datetime.today().__str__().split(" ")[0], key), plan_tag=poolTagMap[key], url_list=[i['url'] for i in result_list] ) @classmethod def sendToDifferentPools(cls, pool_info): """ 获取文章url :return: """ for key in pool_info: cls.sendToEachCrawlerPlan(key, pool_info[key]) @classmethod def deal(cls): """ Dealing function :return: """ yesterday_data = cls.getYesterdayData() level_url_list_map = cls.splitToDifferentPools(yesterday_data) cls.sendToDifferentPools(pool_info=level_url_list_map) if __name__ == '__main__': S = SendToMultiLevels() S.deal() # yesterday_data = S.getYesterdayData() # for line in tqdm(yesterday_data): # print(line)