Merge branch 'master' into featrue-metirc_log-20230718

# Conflicts:
#	etl-core/src/main/java/com/tzld/crawler/etl/service/impl/EtlServiceImpl.java
#	etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/handler/TitleScoreHandler.java

etl-core/src/main/java/com/tzld/crawler/etl/common/CustomValidator.java

@@ -47,18 +47,44 @@ public class CustomValidator implements Validator {
         validator = factory.getValidator();
     }
 
-    public static String validate(Object param) {
+    public static Result validate(Object param) {
         Validator validator = new CustomValidator();
         BeanPropertyBindingResult errors = new BeanPropertyBindingResult(param, "param");
         validator.validate(param, errors);
         if (errors.hasErrors()) {
             StringJoiner errorMessage = new StringJoiner("; ", "[", "]");
             errors.getAllErrors().forEach(e -> errorMessage.add(e.getDefaultMessage()));
-            return errorMessage.toString();
+            return Result.error(errorMessage.toString());
 
         }
 
-        return "";
+        return Result.success();
+    }
+
+    public static class Result {
+        boolean success;
+        String message;
+
+        public Result(boolean success, String message) {
+            this.success = success;
+            this.message = message;
+        }
+
+        public static Result success() {
+            return new Result(true, "");
+        }
+
+        public static Result error(String message) {
+            return new Result(false, message);
+        }
+
+        public boolean notSuccess() {
+            return !success;
+        }
+
+        public String getMessage() {
+            return message;
+        }
     }
 
     @Override

etl-core/src/main/java/com/tzld/crawler/etl/common/enums/ExceptionEnum.java

@@ -12,7 +12,7 @@ public enum ExceptionEnum {
     DATA_ERROR(1001, "数据异常"),
     PARAM_ERROR(1002, "参数不对"),
     INVOKE_VIDEOAPI_ERROR(1003, "调用 longvideo api 接口服务失败"),
-    URL_FORBIDDEN(1004, "don't have permission to access the url on remote server."),
+    URL_FORBIDDEN(1004, "访问下载链接失败"),
 
     ;
 

etl-core/src/main/java/com/tzld/crawler/etl/service/impl/EtlServiceImpl.java

@@ -95,6 +95,7 @@ public class EtlServiceImpl implements EtlService {
     private final CrawlerVideoExtMapper crawlerVideoExtMapper;
     private final CrawlerUserV3Mapper crawlerUserV3Mapper;
 
+
     @Value("${download.file.path:/data/crawler/videos}")
     private String downloadPath;
     @Value("${aliyun.oss.video.bucket:art-pubbucket}")
@@ -145,9 +146,10 @@ public class EtlServiceImpl implements EtlService {
         long id = 0L;
         try {
             // 参数校验
-            String errorMessage = CustomValidator.validate(param);
-            if (!Strings.isNullOrEmpty(errorMessage)) {
-                throw new CommonException(ExceptionEnum.PARAM_ERROR, "param validate failed." + errorMessage);
+            CustomValidator.Result result = CustomValidator.validate(param);
+            if (result.notSuccess()) {
+                log.error("param validate failed. {}", result.getMessage());
+                return;
             }
 
             // 保存数据库（去重校验）
@@ -177,7 +179,7 @@ public class EtlServiceImpl implements EtlService {
                 crawlerVideo.setVideoId(videoId);
                 crawlerVideo.setTitleScore(data.getTitleScore());
                 return crawlerVideoMapper.updateByPrimaryKeySelective(crawlerVideo) > 0;
-            }, "synthesis", String.format("update video info [%s] to db", crawlerVideo));
+            }, "video2db", String.format("update video info [%s] to db", crawlerVideo));
 
             // 视频写入飞书
             async2Feishu(data, videoId);
@@ -187,9 +189,16 @@ public class EtlServiceImpl implements EtlService {
             log.error("etl server deal {} failed.", param, e);
             // 回滚数据
             if (id > 0) {
-                crawlerVideoMapper.deleteByPrimaryKey(id);
+                try {
+                    int i = crawlerVideoMapper.deleteByPrimaryKey(id);
+                    if (i <= 0) {
+                        log.error("delete data from db failed of {}", id);
+                    }
+                } catch (Exception e1) {
+                    log.error("delete data from db failed of {}", id, e1);
+                }
             }
-            throw new CommonException(ExceptionEnum.SYSTEM_ERROR, "etl server deal error: " + e.getMessage());
+//            throw new CommonException(ExceptionEnum.SYSTEM_ERROR, "etl server deal error: " + e.getMessage());
         } finally {
             metric(MetricTypeEnum.DURATION, sw.stop());
             tlParam.remove();
@@ -317,10 +326,10 @@ public class EtlServiceImpl implements EtlService {
         String strategy = data.getStrategy();
         String audioUrl = data.getAudioUrl();
 
-        String videoPath = urlDownload(data.getVideoUrl(), "longvideo/crawler_local/video", title);
+        String videoPath = urlDownload(data.getVideoUrl(), "longvideo/crawler_local/video", title, data.getOutVideoId());
         // 音、视频合成
         if (!Strings.isNullOrEmpty(audioUrl)) {
-            String audioPath = urlDownload(data.getAudioUrl(), "longvideo/crawler_local/audio", title);
+            String audioPath = urlDownload(data.getAudioUrl(), "longvideo/crawler_local/audio", title, data.getOutVideoId());
             retryFunc(t -> {
                 try {
                     VideoUtils.videoSynthesis(ffmpegPath, downloadPath + File.separator + t,
@@ -357,7 +366,7 @@ public class EtlServiceImpl implements EtlService {
         Files.deleteIfExists(Paths.get(new File(tempFilePath).getPath()));
 
         // 视频封面下载、上传 OSS
-        String coverPath = urlDownload(data.getCoverUrl(), "longvideo/crawler_local/image", title);
+        String coverPath = urlDownload(data.getCoverUrl(), "longvideo/crawler_local/image", title, data.getOutVideoId());
         file2oss(downloadPath + File.separator + coverPath, coverPath, platform, strategy);
         data.setCoverOssPath(coverPath);
         tempFilePath = downloadPath + File.separator + coverPath;
@@ -367,21 +376,24 @@ public class EtlServiceImpl implements EtlService {
         Files.deleteIfExists(Paths.get(new File(tempFilePath).getPath()));
     }
 
-    private String urlDownload(String fileUrl, String filePath, String title) {
-        String titleMmd5 = MD5Util.md5(title);
+    private String urlDownload(String fileUrl, String filePath, String title, String outVideoId) {
+        String fileName = MD5Util.md5(title) + outVideoId;
         String curDate = LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE);
         String videoFilePath = filePath + File.separator + env + File.separator + curDate;
         String localFilePath = downloadPath + File.separator + videoFilePath;
 
         File localFile = new File(localFilePath);
         if (!localFile.exists() && (!localFile.mkdirs())) {
-            log.warn("mkdir dir [{}] failed!", localFilePath);
+            log.error("mkdir dir [{}] failed!", localFilePath);
+
+            localFilePath = "/data";
         }
 
+        String finalLocalFilePath = localFilePath;
         retrySupplier(() -> {
             try {
                 // 下载文件
-                FileUtils.download(fileUrl, localFilePath + File.separator + titleMmd5);
+                FileUtils.download(fileUrl, finalLocalFilePath + File.separator + fileName);
                 return false;
             } catch (CommonException e) {
                 if (e.getCode() == ExceptionEnum.URL_FORBIDDEN.getCode()) {
@@ -394,7 +406,7 @@ public class EtlServiceImpl implements EtlService {
             }
         }, "download", String.format("download file from [%s] to [%s]", fileUrl, filePath));
 
-        return videoFilePath + File.separator + titleMmd5;
+        return videoFilePath + File.separator + fileName;
     }
 
     private void file2oss(String localFile, String ossBucketKey, String crawler, String mode) {

etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/StrategyHandlerService.java

@@ -73,6 +73,7 @@ public class StrategyHandlerService implements ApplicationContextAware {
         for (String strategy : strategies) {
             StrategyAbstractHandler handler = HANDLER_MAP.get(strategy);
             if (handler == null) {
+                log.error("can't find handler of {}", strategy);
                 return null;
             }
             result = handler.execute(param);

etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/handler/TitleScoreHandler.java

@@ -77,7 +77,7 @@ public class TitleScoreHandler extends StrategyAbstractHandler {
 
     @Override
     public StrategyDataDto execute(CrawlerEtlParam param) {
-        double score = 0;
+        double score = -1;
         StrategyDataDto result = new StrategyDataDto();
         BeanUtils.copyProperties(param, result);
         try {