metric log

etl-core/src/main/java/com/tzld/crawler/etl/service/impl/EtlServiceImpl.java

@@ -382,24 +382,26 @@ public class EtlServiceImpl implements EtlService {
         Files.deleteIfExists(Paths.get(new File(tempFilePath).getPath()));
     }
 
-    private String urlDownload(String fileUrl, String filePath, String title, String outVideoId) {
+    private String urlDownload(String fileUrl, String typeDir, String title, String outVideoId) {
         String fileName = MD5Util.md5(title) + outVideoId;
-        String curDate = LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE);
-        String videoFilePath = filePath + File.separator + env + File.separator + curDate;
-        String localFilePath = downloadPath + File.separator + videoFilePath;
+        String relFileDir = typeDir + File.separator
+                + env + File.separator
+                + LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE);
+        String absFileDir = downloadPath + File.separator + relFileDir;
 
-        File localFile = new File(localFilePath);
-        if (!localFile.exists() && (!localFile.mkdirs())) {
-            log.error("mkdir dir [{}] failed!", localFilePath);
+        File absFile = new File(absFileDir);
+        if (!absFile.exists() && (!absFile.mkdirs())) {
+            log.error("mkdir typeDir [{}] failed!", absFileDir);
 
-            localFilePath = "/data";
+            absFileDir = "/data";
         }
 
+        String filePath = absFileDir + File.separator + fileName;
         try {
-            retryFunc((t) -> {
+            retryFunc((fpath) -> {
                 try {
                     // 下载文件
-                    FileUtils.download(fileUrl, t + File.separator + fileName);
+                    FileUtils.download(fileUrl, fpath);
                     return false;
                 } catch (CommonException e) {
                     if (e.getCode() == ExceptionEnum.URL_FORBIDDEN.getCode()) {
@@ -411,13 +413,21 @@ public class EtlServiceImpl implements EtlService {
                 } catch (Exception e) {
                     throw new RuntimeException(e);
                 }
-            }, localFilePath, "download", String.format("download file from [%s] to [%s]", fileUrl, filePath));
+            }, filePath, "download", String.format("download file from [%s] to [%s].", fileUrl, filePath));
         } catch (Exception e) {
-            metric(MetricTypeEnum.DOWNLOAD_FAILED, fileUrl);
+            metric(MetricTypeEnum.DOWNLOAD_FAILED, String.format("download file from [%s] error. 'cause of [%s]",
+                    fileUrl, CommonUtils.getRootCause(e).getMessage()));
             throw new CommonException(e);
         }
 
-        return videoFilePath + File.separator + fileName;
+        File file = new File(filePath);
+        if (!file.exists() || file.length() <= 0) {
+            String msg = String.format("download file from [%s] is empty.", fileUrl);
+            metric(MetricTypeEnum.DOWNLOAD_FAILED, msg);
+            throw new CommonException(ExceptionEnum.DATA_ERROR, msg);
+        }
+
+        return relFileDir + File.separator + fileName;
     }
 
     private void file2oss(String localFile, String ossBucketKey, String crawler, String mode) {

etl-core/src/main/java/com/tzld/crawler/etl/util/FileUtils.java

@@ -77,6 +77,5 @@ public class FileUtils {
         inputStream.close();
         outputStream.close();
         log.info("downloaded successfully [{}] to [{}]", fileUrl, filePath);
-
     }
 }

etl-core/src/main/java/com/tzld/crawler/etl/util/VideoUtils.java

@@ -90,7 +90,8 @@ public class VideoUtils {
         Process process = processBuilder.start();
         process.waitFor();
 
-        if (!new File(outPath).exists()) {
+        File file = new File(outPath);
+        if (!file.exists() || file.length() <= 0) {
             throw new CommonException(ExceptionEnum.DATA_ERROR, "video synthesis failed, No synthesized video file output.");
         }
         // } catch (Exception e) {