""" @author: luojunhui """ import datetime from tqdm import tqdm from applications import AIDTApi, DeNetMysql, PQMySQL, Functions, ODPSApi from config import poolTagMap from stratrgy import ArticlePoolStrategy class SendToMultiLevels(object): """ 流量池任务 """ AidApi = AIDTApi() DeMysql = DeNetMysql() PqMysql = PQMySQL() Fun = Functions() OA = ODPSApi() @classmethod def getYesterdayData(cls): """ 获取前一天数据表现 :return: """ odps_sql = "select * from loghubods.changwen_article_datastat where dt = '20240729';" result = cls.OA.select(sql=odps_sql) response_list = [ { "article_id": record["article_id"], "increase_read_count": record["increase_read_count"], "read_count": record["read_count"], "increase_income": record["increase_income"], "income": record["income"], "increase_share_count": record["increase_share_count"], "share_count": record["share_count"], "update_timestamp": record["update_timestamp"] } for record in result if record['increase_read_count'] >= 1000 ] return response_list @classmethod def splitToDifferentPools(cls, yesterday_data): """ 分类至Pools :return: """ S = ArticlePoolStrategy() detail_list = S.getData(article_list=yesterday_data) result = S.splitByStrategy(detail_list=detail_list) return result @classmethod def sendToEachCrawlerPlan(cls, key, url_list): """ :param key: :param url_list: :return: """ print(key) print(len(url_list)) print(url_list) # daily自动创建新抓取计划 # cls.AidApi.updateArticleIntoCrawlerPlan( # plan_id=None, # plan_name="{}--{}".format(datetime.datetime.today().__str__().split(" ")[0], key), # plan_tag=poolTagMap[key], # url_list=url_list # ) @classmethod def sendToDifferentPools(cls, pool_info): """ 获取文章url :return: """ for key in pool_info: cls.sendToEachCrawlerPlan(key, pool_info[key]) @classmethod def deal(cls): """ Dealing function :return: """ yesterday_data = cls.getYesterdayData() level_url_list_map = cls.splitToDifferentPools(yesterday_data) cls.sendToDifferentPools(pool_info=level_url_list_map) if __name__ == '__main__': ST = SendToMultiLevels() ST.deal()