implementing retry strategies for core business logic

etl-core/src/main/java/com/tzld/crawler/etl/service/impl/EtlServiceImpl.java

@@ -171,7 +171,7 @@ public class EtlServiceImpl implements EtlService {
                 crawlerVideo.setVideoId(videoId);
                 crawlerVideo.setTitleScore(data.getTitleScore());
                 return crawlerVideoMapper.updateByPrimaryKeySelective(crawlerVideo) > 0;
-            }, "synthesis", String.format("update video info %s to db error", crawlerVideo));
+            }, "synthesis", String.format("update video info [%s] to db", crawlerVideo));
 
             // 视频写入飞书
             async2Feishu(data, videoId);
@@ -197,7 +197,7 @@ public class EtlServiceImpl implements EtlService {
                         crawlerVideo.getPlatform(), crawlerVideo.getStrategy());
                 return Pair.of(true, -1L);
             }
-        }, "video2db", String.format("save video info %s to db failed", crawlerVideo));
+        }, "video2db", String.format("save video info [%s] to db", crawlerVideo));
     }
 
     private long videoSend(StrategyDataDto data) {
@@ -224,7 +224,7 @@ public class EtlServiceImpl implements EtlService {
         }
         request.setCrawlerTaskTimestamp(System.currentTimeMillis());
         CommonResponse<WxVideoVO> response = longVideoFeign.crawlerVideoSend(request);
-        log.info("crawler send video request: {}, response: {}", request, response);
+        log.info("crawler send video request [{}] response [{}]", request, response);
         if (!response.isSuccess() || response.getData() == null || response.getData().getId() == null) {
             throw new CommonException(ExceptionEnum.INVOKE_VIDEOAPI_ERROR,
                     "invoke crawler send video failed!" + response);
@@ -239,20 +239,20 @@ public class EtlServiceImpl implements EtlService {
             try {
                 String sheetToken = feishuSheetTokenMap.get(platform);
                 if (Strings.isNullOrEmpty(sheetToken)) {
-                    log.error("feishu.sheet.token.map config of {} is null", platform);
+                    log.error("feishu.sheet.token.map config of [{}] is null", platform);
                     return;
                 }
                 String strategyType = Strings.nullToEmpty(data.getStrategyType());
                 String sheetId = feishuSheetIdMap.get(platform + strategyType);
                 if (Strings.isNullOrEmpty(sheetToken)) {
-                    log.error("feishu.sheet.id.map config of {} is null", platform);
+                    log.error("feishu.sheet.id.map config of [{}] is null", platform);
                     return;
                 }
                 String range = feishuRangeMap.get(platform + strategyType);
 
-                log.info("{} {} sheetToken {} sheetId {} range {}", platform, strategy, sheetToken, sheetId, range);
+                log.info("platform [{}] strategy [{}] sheetToken is [{}], sheetId is [{}], range is [{}]", platform, strategy, sheetToken, sheetId, range);
                 String fsResp = FeishuUtils.insertRows(feishuAppid, feishuAppsecret, sheetToken, sheetId, 1, 2);
-                log.debug("insert columns to feishu sheet response is {}", fsResp);
+                log.debug("insert columns to feishu sheet response is [{}]", fsResp);
 
                 List<List<Object>> values = new ArrayList<>();
                 List<Object> value = new ArrayList<>();
@@ -287,11 +287,11 @@ public class EtlServiceImpl implements EtlService {
                 values.add(value);
 
                 fsResp = FeishuUtils.updateValues(feishuAppid, feishuAppsecret, sheetToken, range, values);
-                log.debug("update feishu sheet value response is {}", fsResp);
+                log.debug("update feishu sheet value response is [{}]", fsResp);
 
                 slsService.log("message", "视频已保存至云文档", "crawler", platform, "mode", strategy);
             } catch (Exception e) {
-                log.error("save data to feishu sheet error. platform {}, strategy {}", platform, strategy, e);
+                log.error("save data to feishu sheet error. platform [{}], strategy [{}]", platform, strategy, e);
             }
         });
     }
@@ -314,7 +314,7 @@ public class EtlServiceImpl implements EtlService {
                 } catch (Exception e) {
                     throw new RuntimeException(e);
                 }
-            }, videoPath, "synthesis", String.format("video %s audio %s synthesis error", videoPath, audioPath));
+            }, videoPath, "synthesis", String.format("video [%s] audio [%s] synthesis", videoPath, audioPath));
 
             // 清理合成音频之前的文件
             Files.deleteIfExists(Paths.get(new File(downloadPath + File.separator + videoPath).getPath()));
@@ -338,17 +338,17 @@ public class EtlServiceImpl implements EtlService {
         }
 
         // 文件清理
-        log.info("begin delete {}", tempFilePath);
+        log.info("begin delete temp video file [{}]", tempFilePath);
         Files.deleteIfExists(Paths.get(new File(tempFilePath).getPath()));
 
         // 视频封面下载、上传 OSS
         String coverPath = urlDownload(data.getCoverUrl(), "longvideo/crawler_local/image", title);
         file2oss(downloadPath + File.separator + coverPath, coverPath, platform, strategy);
         data.setCoverOssPath(coverPath);
-        tempFilePath = downloadPath + File.separator + videoPath;
+        tempFilePath = downloadPath + File.separator + coverPath;
 
         // 文件清理
-        log.info("begin delete {}", tempFilePath);
+        log.info("begin delete temp cover file [{}]", tempFilePath);
         Files.deleteIfExists(Paths.get(new File(tempFilePath).getPath()));
     }
 
@@ -361,7 +361,7 @@ public class EtlServiceImpl implements EtlService {
         File localFile = new File(localFilePath);
         if (!localFile.exists()) {
             if (!localFile.mkdirs()) {
-                log.warn("mkdir {} failed", localFilePath);
+                log.warn("mkdir dir [{}] failed!", localFilePath);
             }
         }
 
@@ -374,13 +374,13 @@ public class EtlServiceImpl implements EtlService {
                 if (e instanceof CommonException) {
                     CommonException ce = (CommonException) e;
                     if (ce.getCode() == ExceptionEnum.URL_FORBIDDEN.getCode()) {
-                        log.error("access to the url {} of remote server is prohibited.", fileUrl);
+                        log.error("access to the url [{}] of remote server is prohibited.", fileUrl);
                         return true;
                     }
                 }
                 throw new RuntimeException(e);
             }
-        }, "download", String.format("download file from %s to %s", fileUrl, filePath));
+        }, "download", String.format("download file from [%s] to [%s]", fileUrl, filePath));
 
         return videoFilePath + File.separator + titleMmd5;
     }
@@ -397,7 +397,7 @@ public class EtlServiceImpl implements EtlService {
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
-        }, "", "2oss", String.format("upload file %s to oss buckekey %s", localFile, ossBucketKey));
+        }, "", "2oss", String.format("upload file [%s] to oss [%s]", localFile, ossBucketKey));
     }
 
     private void retrySupplier(Supplier<Boolean> supplier, String type, String errorMsg) {
@@ -421,9 +421,9 @@ public class EtlServiceImpl implements EtlService {
                     break;
                 }
             } catch (Exception e) {
-                log.error("the operation [{}] has failed on the {}th retry.", errorMsg, retry, e);
+                log.error("the operation '{}' has failed on the {}th retry.", errorMsg, retry, e);
                 if (retry >= retryTimes) {
-                    throw new CommonException(ExceptionEnum.SYSTEM_ERROR, "the operation [{}] has failed after " + retry + " times retry.");
+                    throw new CommonException(ExceptionEnum.SYSTEM_ERROR, "the operation '" + errorMsg + "' has failed after " + retry + " times retry.");
                 }
             }
         }

etl-core/src/main/java/com/tzld/crawler/etl/util/FileUtils.java

@@ -55,19 +55,19 @@ public class FileUtils {
 
     public static String formatFileSize(long sizeInBytes) {
         if (sizeInBytes < 1024) {
-            return sizeInBytes + " B";
+            return sizeInBytes + "B";
         } else if (sizeInBytes < 1024 * 1024) {
             double sizeInKB = sizeInBytes / 1024.0;
-            return String.format("%.2f KB", sizeInKB);
+            return String.format("%.2fKB", sizeInKB);
         } else {
             double sizeInMB = sizeInBytes / (1024.0 * 1024.0);
-            return String.format("%.2f MB", sizeInMB);
+            return String.format("%.2fMB", sizeInMB);
         }
     }
 
     public static void download(String fileUrl, String filePath) throws Exception {
         // try {
-        log.info("begin download {} to {}", fileUrl, filePath);
+        log.info("begin download [{}] to [{}]", fileUrl, filePath);
         URL url = new URL(fileUrl);
         HttpURLConnection conn = (HttpURLConnection) url.openConnection();
         if (conn.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN) {
@@ -75,7 +75,7 @@ public class FileUtils {
         }
         conn.setConnectTimeout(5000);
         conn.setReadTimeout(5000);
-        log.info("size is {} of url {}", formatFileSize(conn.getContentLength()), fileUrl);
+        log.info("download file size is {} of url [{}]", formatFileSize(conn.getContentLength()), fileUrl);
 
         InputStream inputStream = conn.getInputStream();
         FileOutputStream outputStream = new FileOutputStream(filePath);
@@ -86,7 +86,7 @@ public class FileUtils {
         }
         inputStream.close();
         outputStream.close();
-        log.info("downloaded successfully {} to {}", fileUrl, filePath);
+        log.info("downloaded successfully [{}] to [{}]", fileUrl, filePath);
         // return true;
         // } catch (Exception e) {
         //     log.error("downloaded error file url: {} local path: {} ", fileUrl, filePath, e);