|
@@ -4,7 +4,6 @@
|
|
|
import json
|
|
import json
|
|
|
import os
|
|
import os
|
|
|
import random
|
|
import random
|
|
|
-import shutil
|
|
|
|
|
import sys
|
|
import sys
|
|
|
import time
|
|
import time
|
|
|
import oss2
|
|
import oss2
|
|
@@ -285,7 +284,7 @@ class Publish:
|
|
|
return "CRAWLER"
|
|
return "CRAWLER"
|
|
|
|
|
|
|
|
@classmethod
|
|
@classmethod
|
|
|
- def local_file_path(cls, crawler):
|
|
|
|
|
|
|
+ def local_file_path(cls):
|
|
|
# local_file_path = f'./{crawler}/videos'
|
|
# local_file_path = f'./{crawler}/videos'
|
|
|
local_file_path = f'./videos'
|
|
local_file_path = f'./videos'
|
|
|
video_file = 'video'
|
|
video_file = 'video'
|
|
@@ -313,11 +312,11 @@ class Publish:
|
|
|
Common.logger(log_type).info("upload_and_publish starting...")
|
|
Common.logger(log_type).info("upload_and_publish starting...")
|
|
|
today = time.strftime("%Y%m%d", time.localtime())
|
|
today = time.strftime("%Y%m%d", time.localtime())
|
|
|
# videos 目录下的所有视频文件夹
|
|
# videos 目录下的所有视频文件夹
|
|
|
- files = os.listdir(cls.local_file_path(crawler)["local_file_path"])
|
|
|
|
|
|
|
+ files = os.listdir(cls.local_file_path()["local_file_path"])
|
|
|
for fv in files:
|
|
for fv in files:
|
|
|
try:
|
|
try:
|
|
|
# 单个视频文件夹
|
|
# 单个视频文件夹
|
|
|
- fi_d = os.path.join(cls.local_file_path(crawler)["local_file_path"], fv)
|
|
|
|
|
|
|
+ fi_d = os.path.join(cls.local_file_path()["local_file_path"], fv)
|
|
|
# 确认为视频文件夹
|
|
# 确认为视频文件夹
|
|
|
if os.path.isdir(fi_d):
|
|
if os.path.isdir(fi_d):
|
|
|
Common.logger(log_type).info('dir = {}'.format(fi_d))
|
|
Common.logger(log_type).info('dir = {}'.format(fi_d))
|
|
@@ -336,7 +335,7 @@ class Publish:
|
|
|
fi_path = fi_d + '/' + fi
|
|
fi_path = fi_d + '/' + fi
|
|
|
Common.logger(log_type).info('dir fi_path = {}'.format(fi_path))
|
|
Common.logger(log_type).info('dir fi_path = {}'.format(fi_path))
|
|
|
# 读取 info.txt，赋值给 data
|
|
# 读取 info.txt，赋值给 data
|
|
|
- if cls.local_file_path(crawler)["info_file"] in fi:
|
|
|
|
|
|
|
+ if cls.local_file_path()["info_file"] in fi:
|
|
|
f = open(fi_path, "r", encoding="UTF-8")
|
|
f = open(fi_path, "r", encoding="UTF-8")
|
|
|
# 读取数据 数据准确性写入的时候保证 读取暂不处理
|
|
# 读取数据 数据准确性写入的时候保证 读取暂不处理
|
|
|
for i in range(14):
|
|
for i in range(14):
|
|
@@ -363,7 +362,7 @@ class Publish:
|
|
|
fi_path = fi_d + '/' + fi
|
|
fi_path = fi_d + '/' + fi
|
|
|
# Common.logger(log_type).info('dir fi_path = {}'.format(fi_path))
|
|
# Common.logger(log_type).info('dir fi_path = {}'.format(fi_path))
|
|
|
# 上传oss
|
|
# 上传oss
|
|
|
- if cls.local_file_path(crawler)["video_file"] in fi:
|
|
|
|
|
|
|
+ if cls.local_file_path()["video_file"] in fi:
|
|
|
global oss_video_file
|
|
global oss_video_file
|
|
|
if env == "dev":
|
|
if env == "dev":
|
|
|
oss_video_file = cls.oss_file_path_video.format("dev", today, data['crawlerSrcId'])
|
|
oss_video_file = cls.oss_file_path_video.format("dev", today, data['crawlerSrcId'])
|
|
@@ -375,7 +374,7 @@ class Publish:
|
|
|
cls.put_file(log_type, oss_endpoint, oss_video_file, fi_path)
|
|
cls.put_file(log_type, oss_endpoint, oss_video_file, fi_path)
|
|
|
data['videoPath'] = oss_video_file
|
|
data['videoPath'] = oss_video_file
|
|
|
Common.logger(log_type).info("videoPath = {}".format(oss_video_file))
|
|
Common.logger(log_type).info("videoPath = {}".format(oss_video_file))
|
|
|
- elif cls.local_file_path(crawler)["image_file"] in fi:
|
|
|
|
|
|
|
+ elif cls.local_file_path()["image_file"] in fi:
|
|
|
global oss_image_file
|
|
global oss_image_file
|
|
|
if env == "dev":
|
|
if env == "dev":
|
|
|
oss_image_file = cls.oss_file_path_image.format("env", today, data['crawlerSrcId'])
|
|
oss_image_file = cls.oss_file_path_image.format("env", today, data['crawlerSrcId'])
|
|
@@ -406,6 +405,4 @@ class Publish:
|
|
|
else:
|
|
else:
|
|
|
Common.logger(log_type).error('file not a dir = {}'.format(fi_d))
|
|
Common.logger(log_type).error('file not a dir = {}'.format(fi_d))
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- # 删除视频文件夹
|
|
|
|
|
- shutil.rmtree(f"./{crawler}/videos/{fv}/")
|
|
|
|
|
Common.logger(log_type).exception('upload_and_publish error', e)
|
|
Common.logger(log_type).exception('upload_and_publish error', e)
|
