metric log

etl-core/src/main/java/com/tzld/crawler/etl/common/CustomValidator.java

@@ -55,7 +55,6 @@ public class CustomValidator implements Validator {
             StringJoiner errorMessage = new StringJoiner("; ", "[", "]");
             errors.getAllErrors().forEach(e -> errorMessage.add(e.getDefaultMessage()));
             return Result.error(errorMessage.toString());
-
         }
 
         return Result.success();

etl-core/src/main/java/com/tzld/crawler/etl/enums/MetricTypeEnum.java

@@ -31,12 +31,12 @@ import com.fasterxml.jackson.annotation.JsonValue;
  * @since 2023-07-18 14:32.
  */
 public enum MetricTypeEnum {
-    SUCCESS("success", "成功任务"),
+    SUCCESS("success", "处理成功"),
     DURATION("duration", "处理时长"),
     DUPLICATED("duplicated", "重复视频数"),
-    FAIL("fail", "失败原因"),
+    FAIL("fail", "处理失败"),
     INVALID_URL("invalidUrl", "抓取链接无效"),
-    DOWNLOAD_INTERRUPT("downloadInterrupt", "文件下载中断"),
+    DOWNLOAD_FAILED("downloadFailed", "文件下载失败"),
     VIDEO_SYNTHESIS_FAILED("videoSynthesisFailed", "音视频合成失败"),
     UNKNOWN_EXCEPTION("unknownException", "系统处理异常"),
     VIDEO_SEND_FAILED("videoSendFailed", "发布视频失败");

etl-core/src/main/java/com/tzld/crawler/etl/service/impl/EtlServiceImpl.java

@@ -148,8 +148,7 @@ public class EtlServiceImpl implements EtlService {
             // 参数校验
             CustomValidator.Result result = CustomValidator.validate(param);
             if (result.notSuccess()) {
-                log.error("param validate failed. {}", result.getMessage());
-                return;
+                throw new CommonException(ExceptionEnum.PARAM_ERROR, "param validate failed." + result.getMessage());
             }
 
             // 保存数据库（去重校验）
@@ -185,6 +184,7 @@ public class EtlServiceImpl implements EtlService {
             async2Feishu(data, videoId);
 
             metric(MetricTypeEnum.SUCCESS);
+            metric(MetricTypeEnum.DURATION, sw.stop().elapsed().getSeconds());
         } catch (Exception e) {
             log.error("etl server deal {} failed.", param, e);
             // 回滚数据
@@ -198,9 +198,12 @@ public class EtlServiceImpl implements EtlService {
                     log.error("delete data from db failed of {}", id, e1);
                 }
             }
-//            throw new CommonException(ExceptionEnum.SYSTEM_ERROR, "etl server deal error: " + e.getMessage());
+
+            metric(MetricTypeEnum.FAIL);
+            if (!(e instanceof CommonException)) {
+                metric(MetricTypeEnum.UNKNOWN_EXCEPTION);
+            }
         } finally {
-            metric(MetricTypeEnum.DURATION, sw.stop());
             tlParam.remove();
         }
     }
@@ -249,6 +252,7 @@ public class EtlServiceImpl implements EtlService {
         CommonResponse<WxVideoVO> response = longVideoFeign.crawlerVideoSend(request);
         log.info("crawler send video request [{}] response [{}]", request, response);
         if (!response.isSuccess() || response.getData() == null || response.getData().getId() == null) {
+            metric(MetricTypeEnum.VIDEO_SEND_FAILED);
             throw new CommonException(ExceptionEnum.INVOKE_VIDEOAPI_ERROR,
                     "invoke crawler send video failed!" + response);
         }
@@ -330,15 +334,20 @@ public class EtlServiceImpl implements EtlService {
         // 音、视频合成
         if (!Strings.isNullOrEmpty(audioUrl)) {
             String audioPath = urlDownload(data.getAudioUrl(), "longvideo/crawler_local/audio", title, data.getOutVideoId());
-            retryFunc(t -> {
-                try {
-                    VideoUtils.videoSynthesis(ffmpegPath, downloadPath + File.separator + t,
-                            downloadPath + File.separator + audioPath, downloadPath + File.separator + t + "_comp.mp4");
-                    return false;
-                } catch (Exception e) {
-                    throw new RuntimeException(e);
-                }
-            }, videoPath, "synthesis", String.format("video [%s] audio [%s] synthesis", videoPath, audioPath));
+            try {
+                retryFunc(t -> {
+                    try {
+                        VideoUtils.videoSynthesis(ffmpegPath, downloadPath + File.separator + t,
+                                downloadPath + File.separator + audioPath, downloadPath + File.separator + t + "_comp.mp4");
+                        return false;
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                }, videoPath, "synthesis", String.format("video [%s] audio [%s] synthesis", videoPath, audioPath));
+            } catch (Exception e) {
+                metric(MetricTypeEnum.VIDEO_SYNTHESIS_FAILED);
+                throw new CommonException(e);
+            }
 
             // 清理合成音频之前的文件
             Files.deleteIfExists(Paths.get(new File(downloadPath + File.separator + videoPath).getPath()));
@@ -389,22 +398,27 @@ public class EtlServiceImpl implements EtlService {
             localFilePath = "/data";
         }
 
-        String finalLocalFilePath = localFilePath;
-        retrySupplier(() -> {
-            try {
-                // 下载文件
-                FileUtils.download(fileUrl, finalLocalFilePath + File.separator + fileName);
-                return false;
-            } catch (CommonException e) {
-                if (e.getCode() == ExceptionEnum.URL_FORBIDDEN.getCode()) {
-                    log.error("access to the url [{}] of remote server is prohibited.", fileUrl);
+        try {
+            retryFunc((t) -> {
+                try {
+                    // 下载文件
+                    FileUtils.download(fileUrl, t + File.separator + fileName);
                     return false;
+                } catch (CommonException e) {
+                    if (e.getCode() == ExceptionEnum.URL_FORBIDDEN.getCode()) {
+                        log.error("access to the url [{}] of remote server is prohibited.", fileUrl);
+                        metric(MetricTypeEnum.INVALID_URL);
+                        return false;
+                    }
+                    throw new RuntimeException(e);
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
                 }
-                throw new RuntimeException(e);
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        }, "download", String.format("download file from [%s] to [%s]", fileUrl, filePath));
+            }, localFilePath, "download", String.format("download file from [%s] to [%s]", fileUrl, filePath));
+        } catch (Exception e) {
+            metric(MetricTypeEnum.DOWNLOAD_FAILED);
+            throw new CommonException(e);
+        }
 
         return videoFilePath + File.separator + fileName;
     }