Merge branch 'master' into add-celery-app

merge master

account_association_task.py

@@ -8,7 +8,7 @@ from argparse import ArgumentParser
 from datetime import datetime
 
 from applications import bot
-from coldStartTasks.crawler.weixin_account_association_crawler import AccountAssociationCrawler
+from cold_start.crawler.weixin_account_association_crawler import AccountAssociationCrawler
 
 
 def main():

account_cold_start_daily.py

@@ -7,9 +7,9 @@ import traceback
 from argparse import ArgumentParser
 
 from applications import longArticlesMySQL, bot
-from coldStartTasks.crawler.weixinCategoryCrawler import weixinCategory
-from coldStartTasks.publish.publishCategoryArticles import CategoryColdStartTask
-from coldStartTasks.filter.title_similarity_task import ColdStartTitleSimilarityTask
+from cold_start.crawler.weixinCategoryCrawler import weixinCategory
+from cold_start.publish.publishCategoryArticles import CategoryColdStartTask
+from cold_start.filter.title_similarity_task import ColdStartTitleSimilarityTask
 
 DEFAULT_CATEGORY_LIST = ['1030-手动挑号', 'account_association']
 

account_explore_task.py

@@ -32,7 +32,7 @@ def deal_each_platform(platform: str) -> None:
 
 
 if __name__ == "__main__":
-    # get_association_title_list_in_multi_threads()
+    get_association_title_list_in_multi_threads()
 
     # get each platform
     platform_list = ["sph", "hksp", "toutiao", "gzh"]

coldStartTasks/crawler/weixinAccountCrawler.py

@@ -1,67 +0,0 @@
-"""
-@author: luojunhui
-输入文章连接，输出账号信息，并且把账号存储到数据库中
-"""
-import datetime
-
-from tqdm import tqdm
-from applications import WeixinSpider, longArticlesMySQL
-
-
-class weixinAccountCrawler(object):
-    """
-    weixinAccountCrawler
-    """
-
-    def __init__(self, target_url_list):
-        self.db_client = longArticlesMySQL()
-        self.spider = WeixinSpider()
-        self.url_list = target_url_list
-
-    def get_account_detail(self, url):
-        """
-        通过文章链接获取账号信息
-        :param url:
-        :return:
-        """
-        account_detail = self.spider.get_account_by_url(content_url=url)
-        account_obj = account_detail['data']['data']
-        account_name = account_obj['account_name']
-        gh_id = account_obj['wx_gh']
-        self.insert_account_into_database(account_name, gh_id)
-
-    def insert_account_into_database(self, account_name, gh_id, category=None):
-        """
-        :param category:
-        :param account_name:
-        :param gh_id:
-        :return:
-        """
-        if not category:
-            category = "daily-account-mining"
-        insert_sql = f"""
-            INSERT INTO long_articles_accounts
-            (gh_id, account_source, account_name, account_category, init_date)
-            values 
-            (%s, %s, %s, %s, %s)
-        """
-        self.db_client.update(
-            sql=insert_sql,
-            params=(gh_id, "weixin", account_name, category, datetime.date.today().__str__())
-        )
-
-    def deal(self):
-        """
-        entrance of this code
-        :return:
-        """
-        for url in tqdm(self.url_list):
-            self.get_account_detail(url)
-
-
-if __name__ == '__main__':
-    url_list = [
-        'https://mp.weixin.qq.com/s/Q9Je-eNKcHNjh8S-NqQLgg'
-    ]
-    wac = weixinAccountCrawler(url_list)
-    wac.deal()

coldStartTasks/ai_pipeline/extract_video_best_frame.py → cold_start/ai_pipeline/extract_video_best_frame.py

@@ -14,12 +14,12 @@ from applications.api import GoogleAIAPI
 from applications.const import GoogleVideoUnderstandTaskConst
 from applications.db import DatabaseConnector
 from config import long_articles_config
-from coldStartTasks.ai_pipeline.basic import download_file
-from coldStartTasks.ai_pipeline.basic import update_task_queue_status
-from coldStartTasks.ai_pipeline.basic import roll_back_lock_tasks
-from coldStartTasks.ai_pipeline.basic import extract_best_frame_prompt
-from coldStartTasks.ai_pipeline.basic import get_video_cover
-from coldStartTasks.ai_pipeline.basic import normalize_time_str
+from cold_start.ai_pipeline.basic import download_file
+from cold_start.ai_pipeline.basic import update_task_queue_status
+from cold_start.ai_pipeline.basic import roll_back_lock_tasks
+from cold_start.ai_pipeline.basic import extract_best_frame_prompt
+from cold_start.ai_pipeline.basic import get_video_cover
+from cold_start.ai_pipeline.basic import normalize_time_str
 
 const = GoogleVideoUnderstandTaskConst()
 google_ai = GoogleAIAPI()

coldStartTasks/crawler/baidu/video_crawler.py → cold_start/crawler/baidu/video_crawler.py

@@ -18,8 +18,8 @@ from applications.const import BaiduVideoCrawlerConst
 from applications.db import DatabaseConnector
 from applications.exception import SpiderError
 from config import long_articles_config
-from coldStartTasks.crawler.baidu.baidu_spider import baidu_account_video_crawler
-from coldStartTasks.crawler.baidu.baidu_spider import baidu_single_video_crawler
+from cold_start.crawler.baidu.baidu_spider import baidu_account_video_crawler
+from cold_start.crawler.baidu.baidu_spider import baidu_single_video_crawler
 
 const = BaiduVideoCrawlerConst()
 empty_list = []

coldStartTasks/crawler/sohu/get_hot_point.py → cold_start/crawler/sohu/get_hot_point.py

@@ -4,9 +4,9 @@ from tenacity import retry
 
 from applications import log
 from applications.utils import proxy, request_retry
-from coldStartTasks.crawler.sohu.basic import generate_random_strings
-from coldStartTasks.crawler.sohu.basic import generate_random_digits
-from coldStartTasks.crawler.sohu.basic import get_ms_timestamp
+from cold_start.crawler.sohu.basic import generate_random_strings
+from cold_start.crawler.sohu.basic import generate_random_digits
+from cold_start.crawler.sohu.basic import get_ms_timestamp
 
 
 retry_desc = request_retry(retry_times=3, min_retry_delay=2, max_retry_delay=30)

coldStartTasks/crawler/sohu/get_recommedation.py → cold_start/crawler/sohu/get_recommedation.py

@@ -4,8 +4,8 @@ from tenacity import retry
 
 from applications import log
 from applications.utils import proxy, request_retry
-from coldStartTasks.crawler.sohu.basic import generate_random_strings
-from coldStartTasks.crawler.sohu.basic import get_ms_timestamp
+from cold_start.crawler.sohu.basic import generate_random_strings
+from cold_start.crawler.sohu.basic import get_ms_timestamp
 
 
 retry_desc = request_retry(retry_times=3, min_retry_delay=2, max_retry_delay=30)

coldStartTasks/crawler/sohu/get_user_homepage.py → cold_start/crawler/sohu/get_user_homepage.py

@@ -4,9 +4,9 @@ from tenacity import retry
 
 from applications import log
 from applications.utils import proxy, request_retry
-from coldStartTasks.crawler.sohu.basic import generate_random_strings
-from coldStartTasks.crawler.sohu.basic import generate_random_digits
-from coldStartTasks.crawler.sohu.basic import get_ms_timestamp
+from cold_start.crawler.sohu.basic import generate_random_strings
+from cold_start.crawler.sohu.basic import generate_random_digits
+from cold_start.crawler.sohu.basic import get_ms_timestamp
 
 retry_desc = request_retry(retry_times=3, min_retry_delay=2, max_retry_delay=30)
 

coldStartTasks/crawler/toutiao_recommend_crawler.py → cold_start/crawler/toutiao_recommend_crawler.py

@@ -15,7 +15,7 @@ from applications import log
 from applications import Functions
 from applications.db import DatabaseConnector
 from config import long_articles_config
-from coldStartTasks.filter import article_crawler_duplicate_filter
+from cold_start.filter import article_crawler_duplicate_filter
 
 functions = Functions()
 

coldStartTasks/crawler/weixinCategoryCrawler.py → cold_start/crawler/weixinCategoryCrawler.py

@@ -9,7 +9,7 @@ from tqdm import tqdm
 from pymysql.cursors import DictCursor
 
 from applications import WeixinSpider, Functions, log
-from coldStartTasks.filter import article_crawler_duplicate_filter
+from cold_start.filter import article_crawler_duplicate_filter
 from config import apolloConfig
 
 # 常量

coldStartTasks/crawler/weixin_video_crawler.py → cold_start/crawler/weixin_video_crawler.py

@@ -17,7 +17,7 @@ from applications import Functions
 from applications import WeixinSpider
 from applications import longArticlesMySQL
 from applications.const import WeixinVideoCrawlerConst
-from coldStartTasks.filter import video_crawler_duplicate_filter
+from cold_start.filter import video_crawler_duplicate_filter
 
 spider = WeixinSpider()
 functions = Functions()

coldStartTasks/publish/publish_article_association_articles.py → cold_start/publish/publish_article_association_articles.py

@@ -9,15 +9,15 @@ from applications.const import ColdStartTaskConst
 from applications.db import DatabaseConnector
 from config import long_articles_config
 
-from coldStartTasks.publish.basic import filter_by_status
-from coldStartTasks.publish.basic import filter_by_sensitive_words
-from coldStartTasks.publish.basic import filter_by_title_length
-from coldStartTasks.publish.basic import update_published_articles_status
-from coldStartTasks.publish.basic import get_article_from_meta_table
-from coldStartTasks.publish.basic import update_article_status_after_publishing
-from coldStartTasks.publish.basic import create_crawler_plan
-from coldStartTasks.publish.basic import insert_into_article_crawler_plan
-from coldStartTasks.publish.basic import bind_to_generate_plan
+from cold_start.publish.basic import filter_by_status
+from cold_start.publish.basic import filter_by_sensitive_words
+from cold_start.publish.basic import filter_by_title_length
+from cold_start.publish.basic import update_published_articles_status
+from cold_start.publish.basic import get_article_from_meta_table
+from cold_start.publish.basic import update_article_status_after_publishing
+from cold_start.publish.basic import create_crawler_plan
+from cold_start.publish.basic import insert_into_article_crawler_plan
+from cold_start.publish.basic import bind_to_generate_plan
 
 const = ColdStartTaskConst()
 

flow_pool/__init__.py

@@ -1,3 +0,0 @@
-"""
-@author: luojunhui
-"""

flow_pool/upLevel.py

@@ -1,296 +0,0 @@
-"""
-@author: luojunhui
-"""
-import json
-
-from pandas import DataFrame
-from datetime import datetime
-from applications import longArticlesMySQL
-
-lam = longArticlesMySQL()
-
-
-class articleLevelUp(object):
-    """
-    文章晋级
-    """
-    columns = [
-        "位置",
-        "粉丝量",
-        "阅读量",
-        "平均阅读量",
-        "头条阅读量",
-        "头条平均阅读量",
-        "阅读均值倍数",
-        "阅读率",
-        "小程序打开率",
-        "T+0裂变率",
-        "标题",
-        "链接"
-    ]
-    statMapThreeToEight = {
-        "阅读均值倍数": {
-            "mean": 1.1388723507368606,
-            "max": 62.50000000000001,
-            "min": 0.0,
-            "median": 0.8890469416785206,
-            "75%": 1.2617516081147946,
-            "80%": 1.37797320398902,
-            "90%": 1.8733429945338946,
-            "95%": 2.6455874825730517,
-            "99%": 6.252251764489181
-        },
-        "阅读率": {
-            "mean": 0.0006051220910642054,
-            "max": 0.06252537555826228,
-            "min": 0.0,
-            "median": 0.0002241206067691894,
-            "75%": 0.0005117154674215644,
-            "80%": 0.0006449975188817015,
-            "90%": 0.001255232384471895,
-            "95%": 0.002233845658277497,
-            "99%": 0.00633843067255787
-        },
-        "小程序打开率": {
-            "mean": 0.062085135696479415,
-            "max": 1.0,
-            "min": 0.0,
-            "median": 0.045454545454545456,
-            "75%": 0.08695652173913043,
-            "80%": 0.1,
-            "90%": 0.14285714285714285,
-            "95%": 0.18518518518518517,
-            "99%": 0.310463054187192
-        },
-        "T+0裂变率": {
-            "mean": 0.35277482885383377,
-            "max": 181.0,
-            "min": 0.0,
-            "median": 0.0,
-            "75%": 0.0,
-            "80%": 0.09090909090909091,
-            "90%": 0.6666666666666666,
-            "95%": 1.5,
-            "99%": 6.0
-        }
-    }
-    statMapTwoToOne = {
-        "阅读均值倍数": {
-            "mean": 1.0242728432910957,
-            "max": 4.921632060507756,
-            "min": 0.04236315118498048,
-            "median": 0.9604958720021857,
-            "75%": 1.237352622811623,
-            "80%": 1.3131587863024974,
-            "90%": 1.5778563945144477,
-            "95%": 1.8312064951656155,
-            "99%": 2.5125234834603165
-        },
-        "阅读率": {
-            "mean": 0.0073535037464145655,
-            "max": 0.05265662356955502,
-            "min": 0.00020895172629276676,
-            "median": 0.005941952332154309,
-            "75%": 0.009324205525316574,
-            "80%": 0.010420614811741105,
-            "90%": 0.013728137204835086,
-            "95%": 0.01704242661483454,
-            "99%": 0.02622215995438508
-        },
-        "小程序打开率": {
-            "mean": 0.14893695109764848,
-            "max": 2.5,
-            "min": 0.0,
-            "median": 0.1360318513603185,
-            "75%": 0.1875,
-            "80%": 0.20230028849345147,
-            "90%": 0.25449906489537877,
-            "95%": 0.3051369784478383,
-            "99%": 0.4016107123469446
-        },
-        "T+0裂变率": {
-            "mean": 0.6465295965706923,
-            "max": 12.804878048780488,
-            "min": 0.0,
-            "median": 0.48770491803278687,
-            "75%": 0.8011363636363636,
-            "80%": 0.9144722345551121,
-            "90%": 1.317362236032163,
-            "95%": 1.792137476827772,
-            "99%": 3.277849462365585
-        }
-    }
-    firstLevelMap = {
-        "阅读均值倍数": {
-            "mean": 1.0469541000103093,
-            "max": 25.719380724649426,
-            "min": 0.037429819089207735,
-            "median": 0.9521466355025219,
-            "75%": 1.2800839124458492,
-            "80%": 1.370275508982941,
-            "90%": 1.674800845262867,
-            "95%": 1.995613204168999,
-            "99%": 2.9869225601165135
-        },
-        "阅读率": {
-            "mean": 0.016311355353310464,
-            "max": 0.7427434456928839,
-            "min": 0.0006011082360982278,
-            "median": 0.01255841121495327,
-            "75%": 0.020080845617803843,
-            "80%": 0.022950649260452458,
-            "90%": 0.03136776141996209,
-            "95%": 0.0398727631704118,
-            "99%": 0.05986584275411923
-        },
-        "小程序打开率": {
-            "mean": 0.20655535828501095,
-            "max": 0.8,
-            "min": 0.0,
-            "median": 0.19921326215228996,
-            "75%": 0.25838983436476154,
-            "80%": 0.27586206896551724,
-            "90%": 0.32290043225754594,
-            "95%": 0.3709317026683608,
-            "99%": 0.4685840031614304
-        },
-        "T+0裂变率": {
-            "mean": 0.6660929834568661,
-            "max": 46.0,
-            "min": 0.0,
-            "median": 0.5434782608695652,
-            "75%": 0.7940509083886685,
-            "80%": 0.8776439089692103,
-            "90%": 1.159075752014066,
-            "95%": 1.62348848368522,
-            "99%": 2.785400696864109
-        }
-    }
-
-    @classmethod
-    def readRateDebias(cls, row):
-        """
-        阅读均值倍数通过头条消偏
-        :param row:
-        :return:
-        """
-        if row["位置"] != 1:
-            return row["阅读量"] / (
-                    max(1.0, row["头条阅读量"] / row["头条阅读均值"]) * row["阅读均值"]
-            )
-        else:
-            return row["阅读均值倍数"]
-
-    @classmethod
-    def getBaseData(cls):
-        """
-
-        :return:
-        """
-        sql = f"""
-        SELECT
-            position, fans, view_count, avg_view_count, first_view_count, first_avg_view_count, read_rate, read_fans_rate, first_read_rate, fission0_first_rate, title, link
-        FROM 
-            datastat_sort_strategy;
-        """
-        response = lam.select(sql)
-        df = DataFrame(response, columns=cls.columns)
-        df = df.sort_values(by=["阅读均值倍数"], ascending=[False]).reset_index(drop=True)
-        df = df[df["粉丝量"] > 10000].reset_index(drop=True)
-        return df
-
-    @classmethod
-    def analysisDF(cls, indexList):
-        """
-        分析 dataframe 中数据占比
-        :return:
-        """
-        DF = cls.getBaseData()
-        DF = DF[(DF["位置"].isin(indexList))]
-        print(len(DF))
-        avg_read_times = DF['阅读均值倍数'].sort_values(ascending=False)
-        read_rate = DF['阅读率'].sort_values(ascending=False)
-        mini_open_rate = DF['小程序打开率'].sort_values(ascending=False)
-        t_plus_0_fission = DF['T+0裂变率'].sort_values(ascending=False)
-        detail = {
-            "阅读均值倍数": {
-                "mean": avg_read_times.mean(),
-                "max": avg_read_times.max(),
-                "min": avg_read_times.min(),
-                "median": avg_read_times.median(),
-                "75%": avg_read_times.quantile(0.75),
-                "80%": avg_read_times.quantile(0.8),
-                "90%": avg_read_times.quantile(0.9),
-                "95%": avg_read_times.quantile(0.95),
-                "99%": avg_read_times.quantile(0.99)
-            },
-            "阅读率": {
-                "mean": read_rate.mean(),
-                "max": read_rate.max(),
-                "min": read_rate.min(),
-                "median": read_rate.median(),
-                "75%": read_rate.quantile(0.75),
-                "80%": read_rate.quantile(0.8),
-                "90%": read_rate.quantile(0.9),
-                "95%": read_rate.quantile(0.95),
-                "99%": read_rate.quantile(0.99)
-            },
-            "小程序打开率": {
-                "mean": mini_open_rate.mean(),
-                "max": mini_open_rate.max(),
-                "min": mini_open_rate.min(),
-                "median": mini_open_rate.median(),
-                "75%": mini_open_rate.quantile(0.75),
-                "80%": mini_open_rate.quantile(0.8),
-                "90%": mini_open_rate.quantile(0.9),
-                "95%": mini_open_rate.quantile(0.95),
-                "99%": mini_open_rate.quantile(0.99)
-            },
-            "T+0裂变率": {
-                "mean": t_plus_0_fission.mean(),
-                "max": t_plus_0_fission.max(),
-                "min": t_plus_0_fission.min(),
-                "median": t_plus_0_fission.median(),
-                "75%": t_plus_0_fission.quantile(0.75),
-                "80%": t_plus_0_fission.quantile(0.8),
-                "90%": t_plus_0_fission.quantile(0.9),
-                "95%": t_plus_0_fission.quantile(0.95),
-                "99%": t_plus_0_fission.quantile(0.99)
-            }
-        }
-        print(json.dumps(detail, ensure_ascii=False, indent=4))
-
-    @classmethod
-    def upLevel38To2(cls):
-        """
-        :return:
-        """
-        dataThreeToEight = cls.getBaseData()
-        dataThreeToEight = dataThreeToEight[dataThreeToEight['位置'].isin([3, 4, 5, 6, 7, 8])]
-        filter_data = dataThreeToEight[
-            (dataThreeToEight['T+0裂变率'] > cls.statMapThreeToEight['T+0裂变率']['95%'])
-            & (dataThreeToEight['阅读均值倍数'] > cls.statMapThreeToEight['阅读均值倍数']['80%'])
-            ]
-        return filter_data
-
-    @classmethod
-    def upLevel2To1(cls):
-        """
-        :return:
-        """
-        dataThreeToEight = cls.getBaseData()
-        dataThreeToEight = dataThreeToEight[dataThreeToEight['位置'].isin([2])]
-        filter_data = dataThreeToEight[
-            (dataThreeToEight['T+0裂变率'] > cls.statMapThreeToEight['T+0裂变率']['90%'])
-            & (dataThreeToEight['阅读均值倍数'] > cls.statMapThreeToEight['阅读均值倍数']['90%'])
-            ]
-        return filter_data
-
-
-U = articleLevelUp()
-U.analysisDF(indexList=[1])
-f_d = U.upLevel2To1()
-for line in list(zip(f_d['标题'], f_d['链接'])):
-    print(line[0])
-    print(line[1])
-    print("\n")

run_article_title_exit_v1.py

@@ -1,7 +1,7 @@
 """
 @author: luojunhui
 """
-from flow_pool.exit_article_with_title import main
+from tasks.flow_pool_tasks.exit_article_with_title import main
 
 
 if __name__ == '__main__':

run_baidu_video_crawler.py

@@ -1,7 +1,7 @@
 """
 @author: luojunhui
 """
-from coldStartTasks.crawler.baidu import BaiduVideoCrawler
+from cold_start.crawler.baidu import BaiduVideoCrawler
 
 if __name__ == '__main__':
     task = BaiduVideoCrawler()

run_toutiao_recommend.py

@@ -2,7 +2,7 @@
 @author: luojunhui
 @description: 今日头条推荐流文章抓取任务
 """
-from coldStartTasks.crawler.toutiao_recommend_crawler import ToutiaoRecommendCrawler
+from cold_start.crawler.toutiao_recommend_crawler import ToutiaoRecommendCrawler
 
 
 if __name__ == "__main__":

run_video_account_crawler.py

@@ -8,7 +8,7 @@ from datetime import datetime
 from argparse import ArgumentParser
 
 from applications import bot
-from coldStartTasks.crawler import WeixinAccountCrawler, WeixinVideoCrawler
+from cold_start.crawler import WeixinAccountCrawler, WeixinVideoCrawler
 
 account_crawler = WeixinAccountCrawler()
 video_crawler = WeixinVideoCrawler()

run_video_publish_and_audit.py

@@ -3,7 +3,7 @@
 """
 from argparse import ArgumentParser
 
-from coldStartTasks.publish import PublishVideosForAudit
+from cold_start.publish import PublishVideosForAudit
 
 pub = PublishVideosForAudit()
 

run_video_understanding_with_google.py

@@ -6,7 +6,7 @@ import datetime
 import multiprocessing
 
 from applications import log
-from coldStartTasks.ai_pipeline import ExtractVideoBestFrame
+from cold_start.ai_pipeline import ExtractVideoBestFrame
 
 PROCESS_EXIT_TIMEOUT = 10 * 60
 

sh/run_account_explore.sh

@@ -14,7 +14,7 @@ then
 else
     echo "$(date '+%Y-%m-%d %H:%M:%S') - trying to restart account_explore_task.py"
     # 切换到指定目录
-    cd /root/luojunhui/dev/LongArticlesJob
+    cd /root/luojunhui/LongArticlesJob
 
     # 激活 Conda 环境
     source /root/miniconda3/etc/profile.d/conda.sh

sh/run_account_quality_analysis.sh

@@ -14,7 +14,7 @@ then
 else
     echo "$(date '+%Y-%m-%d %H:%M:%S') - trying to restart account_quality_analysis.py"
     # 切换到指定目录
-    cd /root/luojunhui/dev/LongArticlesJob
+    cd /root/luojunhui/LongArticlesJob
 
     # 激活 Conda 环境
     source /root/miniconda3/etc/profile.d/conda.sh

tasks/crawler_tasks/crawler_account/crawler_accounts_by_association.py

@@ -16,15 +16,15 @@ from applications.db import DatabaseConnector
 from applications.pipeline import scrape_account_entities_process
 from applications.utils import Item
 from applications.utils import insert_into_candidate_account_pool_table
-from coldStartTasks.crawler.baidu import haokan_search_videos
-from coldStartTasks.crawler.baidu.baidu_spider import baidu_account_video_crawler
-from coldStartTasks.crawler.channels import search_in_wechat_channel
-from coldStartTasks.crawler.channels import get_channel_account_videos
-from coldStartTasks.crawler.toutiao import get_associated_recommendation
-from coldStartTasks.crawler.toutiao import get_toutiao_account_video_list
-from coldStartTasks.crawler.wechat import get_article_detail
-from coldStartTasks.crawler.wechat import get_article_list_from_account
-from coldStartTasks.crawler.wechat import get_source_account_from_article
+from cold_start.crawler.baidu import haokan_search_videos
+from cold_start.crawler.baidu.baidu_spider import baidu_account_video_crawler
+from cold_start.crawler.channels import search_in_wechat_channel
+from cold_start.crawler.channels import get_channel_account_videos
+from cold_start.crawler.toutiao import get_associated_recommendation
+from cold_start.crawler.toutiao import get_toutiao_account_video_list
+from cold_start.crawler.wechat import get_article_detail
+from cold_start.crawler.wechat import get_article_list_from_account
+from cold_start.crawler.wechat import get_source_account_from_article
 from config import apolloConfig, long_articles_config
 
 config = apolloConfig()

tasks/crawler_tasks/crawler_video/crawler_sohu_videos.py

@@ -14,10 +14,10 @@ from applications.utils import Item
 from applications.utils import str_to_md5
 from applications.utils import insert_into_single_video_source_table
 
-from coldStartTasks.crawler.sohu import get_video_detail
-from coldStartTasks.crawler.sohu import get_hot_point_videos
-from coldStartTasks.crawler.sohu import get_recommendation_video_list
-from coldStartTasks.crawler.sohu import get_user_homepage_videos
+from cold_start.crawler.sohu import get_video_detail
+from cold_start.crawler.sohu import get_hot_point_videos
+from cold_start.crawler.sohu import get_recommendation_video_list
+from cold_start.crawler.sohu import get_user_homepage_videos
 
 from config import long_articles_config
 

tasks/crawler_tasks/crawler_video/crawler_sph_videos.py

@@ -20,7 +20,7 @@ from applications.utils import Item
 from applications.utils import str_to_md5
 from applications.utils import upload_to_oss
 from config import long_articles_config
-from coldStartTasks.crawler.channels import get_channel_account_videos
+from cold_start.crawler.channels import get_channel_account_videos
 
 const = ChannelVideoCrawlerConst()
 

tasks/crawler_tasks/crawler_video/crawler_toutiao_videos.py

@@ -17,7 +17,7 @@ from applications.pipeline import scrape_video_entities_process
 from applications.utils import Item
 from applications.utils import str_to_md5
 from applications.utils import insert_into_single_video_source_table
-from coldStartTasks.crawler.toutiao import get_toutiao_account_video_list
+from cold_start.crawler.toutiao import get_toutiao_account_video_list
 from config import apolloConfig, long_articles_config
 
 const = ToutiaoVideoCrawlerConst()

flow_pool/exit_article_with_title.py → tasks/flow_pool_tasks/exit_article_with_title.py

@@ -222,7 +222,3 @@ def main():
         mention=False
     )
 
-
-if __name__ == '__main__':
-    main()
-

not_used_tasks/article_association_task.py → tasks/not_used_tasks/article_association_task.py

@@ -5,8 +5,8 @@ import traceback
 from argparse import ArgumentParser
 
 from applications import bot
-from coldStartTasks.crawler.wechat import ArticleAssociationCrawler
-from coldStartTasks.publish.publish_article_association_articles import ArticleAssociationPublish
+from cold_start.crawler.wechat import ArticleAssociationCrawler
+from cold_start.publish.publish_article_association_articles import ArticleAssociationPublish
 
 
 def main():

tasks/publish_tasks/cold_start_publish_daily.py

@@ -4,7 +4,7 @@ import traceback
 from applications import bot
 from applications.db import DatabaseConnector
 from config import long_articles_config
-from coldStartTasks.publish.publish_single_video_pool_videos import PublishSingleVideoPoolVideos
+from cold_start.publish.publish_single_video_pool_videos import PublishSingleVideoPoolVideos
 
 
 class ColdStartPublishDailyTask:

title_similarity_score_task.py

@@ -3,7 +3,7 @@
 """
 import traceback
 from applications import bot
-from coldStartTasks.filter.title_similarity_task import ColdStartTitleSimilarityTask
+from cold_start.filter.title_similarity_task import ColdStartTitleSimilarityTask
 
 
 if __name__ == '__main__':