change coldStartTasks -> cold_start

account_association_task.py

@@ -8,7 +8,7 @@ from argparse import ArgumentParser
 from datetime import datetime
 
 from applications import bot
-from coldStartTasks.crawler.weixin_account_association_crawler import AccountAssociationCrawler
+from cold_start.crawler.weixin_account_association_crawler import AccountAssociationCrawler
 
 
 def main():

account_cold_start_daily.py

@@ -7,9 +7,9 @@ import traceback
 from argparse import ArgumentParser
 
 from applications import longArticlesMySQL, bot
-from coldStartTasks.crawler.weixinCategoryCrawler import weixinCategory
-from coldStartTasks.publish.publishCategoryArticles import CategoryColdStartTask
-from coldStartTasks.filter.title_similarity_task import ColdStartTitleSimilarityTask
+from cold_start.crawler.weixinCategoryCrawler import weixinCategory
+from cold_start.publish.publishCategoryArticles import CategoryColdStartTask
+from cold_start.filter.title_similarity_task import ColdStartTitleSimilarityTask
 
 DEFAULT_CATEGORY_LIST = ['1030-手动挑号', 'account_association']
 

coldStartTasks/ai_pipeline/extract_video_best_frame.py → cold_start/ai_pipeline/extract_video_best_frame.py

@@ -14,12 +14,12 @@ from applications.api import GoogleAIAPI
 from applications.const import GoogleVideoUnderstandTaskConst
 from applications.db import DatabaseConnector
 from config import long_articles_config
-from coldStartTasks.ai_pipeline.basic import download_file
-from coldStartTasks.ai_pipeline.basic import update_task_queue_status
-from coldStartTasks.ai_pipeline.basic import roll_back_lock_tasks
-from coldStartTasks.ai_pipeline.basic import extract_best_frame_prompt
-from coldStartTasks.ai_pipeline.basic import get_video_cover
-from coldStartTasks.ai_pipeline.basic import normalize_time_str
+from cold_start.ai_pipeline.basic import download_file
+from cold_start.ai_pipeline.basic import update_task_queue_status
+from cold_start.ai_pipeline.basic import roll_back_lock_tasks
+from cold_start.ai_pipeline.basic import extract_best_frame_prompt
+from cold_start.ai_pipeline.basic import get_video_cover
+from cold_start.ai_pipeline.basic import normalize_time_str
 
 const = GoogleVideoUnderstandTaskConst()
 google_ai = GoogleAIAPI()

coldStartTasks/crawler/baidu/video_crawler.py → cold_start/crawler/baidu/video_crawler.py

@@ -18,8 +18,8 @@ from applications.const import BaiduVideoCrawlerConst
 from applications.db import DatabaseConnector
 from applications.exception import SpiderError
 from config import long_articles_config
-from coldStartTasks.crawler.baidu.baidu_spider import baidu_account_video_crawler
-from coldStartTasks.crawler.baidu.baidu_spider import baidu_single_video_crawler
+from cold_start.crawler.baidu.baidu_spider import baidu_account_video_crawler
+from cold_start.crawler.baidu.baidu_spider import baidu_single_video_crawler
 
 const = BaiduVideoCrawlerConst()
 empty_list = []

coldStartTasks/crawler/sohu/get_hot_point.py → cold_start/crawler/sohu/get_hot_point.py

@@ -4,9 +4,9 @@ from tenacity import retry
 
 from applications import log
 from applications.utils import proxy, request_retry
-from coldStartTasks.crawler.sohu.basic import generate_random_strings
-from coldStartTasks.crawler.sohu.basic import generate_random_digits
-from coldStartTasks.crawler.sohu.basic import get_ms_timestamp
+from cold_start.crawler.sohu.basic import generate_random_strings
+from cold_start.crawler.sohu.basic import generate_random_digits
+from cold_start.crawler.sohu.basic import get_ms_timestamp
 
 
 retry_desc = request_retry(retry_times=3, min_retry_delay=2, max_retry_delay=30)

coldStartTasks/crawler/sohu/get_recommedation.py → cold_start/crawler/sohu/get_recommedation.py

@@ -4,8 +4,8 @@ from tenacity import retry
 
 from applications import log
 from applications.utils import proxy, request_retry
-from coldStartTasks.crawler.sohu.basic import generate_random_strings
-from coldStartTasks.crawler.sohu.basic import get_ms_timestamp
+from cold_start.crawler.sohu.basic import generate_random_strings
+from cold_start.crawler.sohu.basic import get_ms_timestamp
 
 
 retry_desc = request_retry(retry_times=3, min_retry_delay=2, max_retry_delay=30)

coldStartTasks/crawler/sohu/get_user_homepage.py → cold_start/crawler/sohu/get_user_homepage.py

@@ -4,9 +4,9 @@ from tenacity import retry
 
 from applications import log
 from applications.utils import proxy, request_retry
-from coldStartTasks.crawler.sohu.basic import generate_random_strings
-from coldStartTasks.crawler.sohu.basic import generate_random_digits
-from coldStartTasks.crawler.sohu.basic import get_ms_timestamp
+from cold_start.crawler.sohu.basic import generate_random_strings
+from cold_start.crawler.sohu.basic import generate_random_digits
+from cold_start.crawler.sohu.basic import get_ms_timestamp
 
 retry_desc = request_retry(retry_times=3, min_retry_delay=2, max_retry_delay=30)
 

coldStartTasks/crawler/toutiao_recommend_crawler.py → cold_start/crawler/toutiao_recommend_crawler.py

@@ -15,7 +15,7 @@ from applications import log
 from applications import Functions
 from applications.db import DatabaseConnector
 from config import long_articles_config
-from coldStartTasks.filter import article_crawler_duplicate_filter
+from cold_start.filter import article_crawler_duplicate_filter
 
 functions = Functions()
 

coldStartTasks/crawler/weixinCategoryCrawler.py → cold_start/crawler/weixinCategoryCrawler.py

@@ -9,7 +9,7 @@ from tqdm import tqdm
 from pymysql.cursors import DictCursor
 
 from applications import WeixinSpider, Functions, log
-from coldStartTasks.filter import article_crawler_duplicate_filter
+from cold_start.filter import article_crawler_duplicate_filter
 from config import apolloConfig
 
 # 常量

coldStartTasks/crawler/weixin_video_crawler.py → cold_start/crawler/weixin_video_crawler.py

@@ -17,7 +17,7 @@ from applications import Functions
 from applications import WeixinSpider
 from applications import longArticlesMySQL
 from applications.const import WeixinVideoCrawlerConst
-from coldStartTasks.filter import video_crawler_duplicate_filter
+from cold_start.filter import video_crawler_duplicate_filter
 
 spider = WeixinSpider()
 functions = Functions()

coldStartTasks/publish/publish_article_association_articles.py → cold_start/publish/publish_article_association_articles.py

@@ -9,15 +9,15 @@ from applications.const import ColdStartTaskConst
 from applications.db import DatabaseConnector
 from config import long_articles_config
 
-from coldStartTasks.publish.basic import filter_by_status
-from coldStartTasks.publish.basic import filter_by_sensitive_words
-from coldStartTasks.publish.basic import filter_by_title_length
-from coldStartTasks.publish.basic import update_published_articles_status
-from coldStartTasks.publish.basic import get_article_from_meta_table
-from coldStartTasks.publish.basic import update_article_status_after_publishing
-from coldStartTasks.publish.basic import create_crawler_plan
-from coldStartTasks.publish.basic import insert_into_article_crawler_plan
-from coldStartTasks.publish.basic import bind_to_generate_plan
+from cold_start.publish.basic import filter_by_status
+from cold_start.publish.basic import filter_by_sensitive_words
+from cold_start.publish.basic import filter_by_title_length
+from cold_start.publish.basic import update_published_articles_status
+from cold_start.publish.basic import get_article_from_meta_table
+from cold_start.publish.basic import update_article_status_after_publishing
+from cold_start.publish.basic import create_crawler_plan
+from cold_start.publish.basic import insert_into_article_crawler_plan
+from cold_start.publish.basic import bind_to_generate_plan
 
 const = ColdStartTaskConst()
 

run_baidu_video_crawler.py

@@ -1,7 +1,7 @@
 """
 @author: luojunhui
 """
-from coldStartTasks.crawler.baidu import BaiduVideoCrawler
+from cold_start.crawler.baidu import BaiduVideoCrawler
 
 if __name__ == '__main__':
     task = BaiduVideoCrawler()

run_toutiao_recommend.py

@@ -2,7 +2,7 @@
 @author: luojunhui
 @description: 今日头条推荐流文章抓取任务
 """
-from coldStartTasks.crawler.toutiao_recommend_crawler import ToutiaoRecommendCrawler
+from cold_start.crawler.toutiao_recommend_crawler import ToutiaoRecommendCrawler
 
 
 if __name__ == "__main__":

run_video_account_crawler.py

@@ -8,7 +8,7 @@ from datetime import datetime
 from argparse import ArgumentParser
 
 from applications import bot
-from coldStartTasks.crawler import WeixinAccountCrawler, WeixinVideoCrawler
+from cold_start.crawler import WeixinAccountCrawler, WeixinVideoCrawler
 
 account_crawler = WeixinAccountCrawler()
 video_crawler = WeixinVideoCrawler()

run_video_publish_and_audit.py

@@ -3,7 +3,7 @@
 """
 from argparse import ArgumentParser
 
-from coldStartTasks.publish import PublishVideosForAudit
+from cold_start.publish import PublishVideosForAudit
 
 pub = PublishVideosForAudit()
 

run_video_understanding_with_google.py

@@ -6,7 +6,7 @@ import datetime
 import multiprocessing
 
 from applications import log
-from coldStartTasks.ai_pipeline import ExtractVideoBestFrame
+from cold_start.ai_pipeline import ExtractVideoBestFrame
 
 PROCESS_EXIT_TIMEOUT = 10 * 60
 

tasks/crawler_tasks/crawler_account/crawler_accounts_by_association.py

@@ -16,15 +16,15 @@ from applications.db import DatabaseConnector
 from applications.pipeline import scrape_account_entities_process
 from applications.utils import Item
 from applications.utils import insert_into_candidate_account_pool_table
-from coldStartTasks.crawler.baidu import haokan_search_videos
-from coldStartTasks.crawler.baidu.baidu_spider import baidu_account_video_crawler
-from coldStartTasks.crawler.channels import search_in_wechat_channel
-from coldStartTasks.crawler.channels import get_channel_account_videos
-from coldStartTasks.crawler.toutiao import get_associated_recommendation
-from coldStartTasks.crawler.toutiao import get_toutiao_account_video_list
-from coldStartTasks.crawler.wechat import get_article_detail
-from coldStartTasks.crawler.wechat import get_article_list_from_account
-from coldStartTasks.crawler.wechat import get_source_account_from_article
+from cold_start.crawler.baidu import haokan_search_videos
+from cold_start.crawler.baidu.baidu_spider import baidu_account_video_crawler
+from cold_start.crawler.channels import search_in_wechat_channel
+from cold_start.crawler.channels import get_channel_account_videos
+from cold_start.crawler.toutiao import get_associated_recommendation
+from cold_start.crawler.toutiao import get_toutiao_account_video_list
+from cold_start.crawler.wechat import get_article_detail
+from cold_start.crawler.wechat import get_article_list_from_account
+from cold_start.crawler.wechat import get_source_account_from_article
 from config import apolloConfig, long_articles_config
 
 config = apolloConfig()

tasks/crawler_tasks/crawler_video/crawler_sohu_videos.py

@@ -14,10 +14,10 @@ from applications.utils import Item
 from applications.utils import str_to_md5
 from applications.utils import insert_into_single_video_source_table
 
-from coldStartTasks.crawler.sohu import get_video_detail
-from coldStartTasks.crawler.sohu import get_hot_point_videos
-from coldStartTasks.crawler.sohu import get_recommendation_video_list
-from coldStartTasks.crawler.sohu import get_user_homepage_videos
+from cold_start.crawler.sohu import get_video_detail
+from cold_start.crawler.sohu import get_hot_point_videos
+from cold_start.crawler.sohu import get_recommendation_video_list
+from cold_start.crawler.sohu import get_user_homepage_videos
 
 from config import long_articles_config
 

tasks/crawler_tasks/crawler_video/crawler_sph_videos.py

@@ -20,7 +20,7 @@ from applications.utils import Item
 from applications.utils import str_to_md5
 from applications.utils import upload_to_oss
 from config import long_articles_config
-from coldStartTasks.crawler.channels import get_channel_account_videos
+from cold_start.crawler.channels import get_channel_account_videos
 
 const = ChannelVideoCrawlerConst()
 

tasks/crawler_tasks/crawler_video/crawler_toutiao_videos.py

@@ -17,7 +17,7 @@ from applications.pipeline import scrape_video_entities_process
 from applications.utils import Item
 from applications.utils import str_to_md5
 from applications.utils import insert_into_single_video_source_table
-from coldStartTasks.crawler.toutiao import get_toutiao_account_video_list
+from cold_start.crawler.toutiao import get_toutiao_account_video_list
 from config import apolloConfig, long_articles_config
 
 const = ToutiaoVideoCrawlerConst()

tasks/not_used_tasks/article_association_task.py

@@ -5,8 +5,8 @@ import traceback
 from argparse import ArgumentParser
 
 from applications import bot
-from coldStartTasks.crawler.wechat import ArticleAssociationCrawler
-from coldStartTasks.publish.publish_article_association_articles import ArticleAssociationPublish
+from cold_start.crawler.wechat import ArticleAssociationCrawler
+from cold_start.publish.publish_article_association_articles import ArticleAssociationPublish
 
 
 def main():

tasks/publish_tasks/cold_start_publish_daily.py

@@ -4,7 +4,7 @@ import traceback
 from applications import bot
 from applications.db import DatabaseConnector
 from config import long_articles_config
-from coldStartTasks.publish.publish_single_video_pool_videos import PublishSingleVideoPoolVideos
+from cold_start.publish.publish_single_video_pool_videos import PublishSingleVideoPoolVideos
 
 
 class ColdStartPublishDailyTask:

title_similarity_score_task.py

@@ -3,7 +3,7 @@
 """
 import traceback
 from applications import bot
-from coldStartTasks.filter.title_similarity_task import ColdStartTitleSimilarityTask
+from cold_start.filter.title_similarity_task import ColdStartTitleSimilarityTask
 
 
 if __name__ == '__main__':