Browse Source

整体改为进程+协程运行模式

zhangliang 1 week ago
parent
commit
e777b7209a

+ 0 - 0
functions/__init__.py → application/functions/__init__.py


+ 0 - 0
functions/clean_title.py → application/functions/clean_title.py


+ 0 - 0
functions/get_redirect_url.py → application/functions/get_redirect_url.py


+ 0 - 0
functions/mysql_service.py → application/functions/mysql_service.py


+ 0 - 0
functions/read_mysql_config.py → application/functions/read_mysql_config.py


+ 0 - 0
functions/zqkd_db_redis.py → application/functions/zqkd_db_redis.py


+ 1 - 1
application/items/item.py

@@ -1,5 +1,5 @@
 import time
-from functions import clean_title
+from application.functions import clean_title
 
 
 class VideoItem(object):

+ 1 - 1
application/spiders/universal_crawler.py

@@ -7,7 +7,7 @@ from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_t
 
 from application.config.common import MQ
 from configs.config import base_url
-from functions import MysqlService
+from application.functions import MysqlService
 from application.items import VideoItem
 from application.pipeline import PiaoQuanPipeline
 from utils.extractors import safe_extract

+ 1 - 1
main.py

@@ -8,7 +8,7 @@ import asyncio
 from application.config.common import LoggerManager
 from utils.trace_utils import generate_trace_id
 from application.config.common import get_consumer, ack_message
-from functions import MysqlService
+from application.functions import MysqlService
 from application.spiders.spider_registry import get_spider_class, SPIDER_CLASS_MAP
 
 

+ 0 - 0
pipelines/__init__.py