|
@@ -13,6 +13,7 @@ from applications import aiditApi, log, bot
|
|
|
from config import apolloConfig
|
|
from config import apolloConfig
|
|
|
|
|
|
|
|
apollo = apolloConfig()
|
|
apollo = apolloConfig()
|
|
|
|
|
+DAILY_CRAWLER_MAX_NUM = 1000
|
|
|
|
|
|
|
|
|
|
|
|
|
class CategoryColdStartTask(object):
|
|
class CategoryColdStartTask(object):
|
|
@@ -268,7 +269,7 @@ class CategoryColdStartTask(object):
|
|
|
},
|
|
},
|
|
|
mention=False
|
|
mention=False
|
|
|
)
|
|
)
|
|
|
- return filter_df[:1000]
|
|
|
|
|
|
|
+ return filter_df[:DAILY_CRAWLER_MAX_NUM]
|
|
|
|
|
|
|
|
def filter_toutiao_articles(self, articles_df, category):
|
|
def filter_toutiao_articles(self, articles_df, category):
|
|
|
"""
|
|
"""
|
|
@@ -377,7 +378,7 @@ class CategoryColdStartTask(object):
|
|
|
for category in category_list:
|
|
for category in category_list:
|
|
|
try:
|
|
try:
|
|
|
# 已发布标题去重
|
|
# 已发布标题去重
|
|
|
- self.published_articles_title_filter(category=category)
|
|
|
|
|
|
|
+ self.published_articles_title_filter()
|
|
|
|
|
|
|
|
category_df = self.get_articles_from_meta_table(category=category, article_source=article_source)
|
|
category_df = self.get_articles_from_meta_table(category=category, article_source=article_source)
|
|
|
self.publish_filter_articles(
|
|
self.publish_filter_articles(
|
