fetch video info

etl-core/src/main/java/com/tzld/crawler/etl/model/dto/VideoInfoDto.java

@@ -0,0 +1,104 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.model.dto;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-26 19:24.
+ */
+public class VideoInfoDto {
+    private long duration;
+    private int width;
+    private int height;
+
+    public static VideoInfoDtoBuilder newBuilder() {
+        return new VideoInfoDtoBuilder();
+    }
+
+    @Override
+    public String toString() {
+        return "VideoInfoDto{" +
+                "duration=" + duration +
+                ", width=" + width +
+                ", height=" + height +
+                '}';
+    }
+
+    public long getDuration() {
+        return duration;
+    }
+
+    public void setDuration(long duration) {
+        this.duration = duration;
+    }
+
+    public int getWidth() {
+        return width;
+    }
+
+    public void setWidth(int width) {
+        this.width = width;
+    }
+
+    public int getHeight() {
+        return height;
+    }
+
+    public void setHeight(int height) {
+        this.height = height;
+    }
+
+    public static final class VideoInfoDtoBuilder {
+        private long duration;
+        private int width;
+        private int height;
+
+        private VideoInfoDtoBuilder() {
+        }
+
+        public VideoInfoDtoBuilder duration(long duration) {
+            this.duration = duration;
+            return this;
+        }
+
+        public VideoInfoDtoBuilder width(int width) {
+            this.width = width;
+            return this;
+        }
+
+        public VideoInfoDtoBuilder height(int height) {
+            this.height = height;
+            return this;
+        }
+
+        public VideoInfoDto build() {
+            VideoInfoDto videoInfoDto = new VideoInfoDto();
+            videoInfoDto.setDuration(duration);
+            videoInfoDto.setWidth(width);
+            videoInfoDto.setHeight(height);
+            return videoInfoDto;
+        }
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/model/vo/CrawlerVideoVO.java

@@ -104,32 +104,9 @@ public class CrawlerVideoVO {
 
     private String strategyType;
 
-    @Override
-    public String toString() {
-        return "CrawlerVideoVO{" +
-                "userId=" + userId +
-                ", outUserId='" + outUserId + '\'' +
-                ", userName='" + userName + '\'' +
-                ", avatarUrl='" + avatarUrl + '\'' +
-                ", platform='" + platform + '\'' +
-                ", strategy='" + strategy + '\'' +
-                ", outVideoId='" + outVideoId + '\'' +
-                ", videoTitle='" + videoTitle + '\'' +
-                ", coverUrl='" + coverUrl + '\'' +
-                ", videoUrl='" + videoUrl + '\'' +
-                ", duration=" + duration +
-                ", publishTime='" + publishTime + '\'' +
-                ", playCnt=" + playCnt +
-                ", likeCnt=" + likeCnt +
-                ", shareCnt=" + shareCnt +
-                ", collectionCnt=" + collectionCnt +
-                ", commentCnt=" + commentCnt +
-                ", crawlerRule='" + crawlerRule + '\'' +
-                ", width=" + width +
-                ", height=" + height +
-                ", strategyType='" + strategyType + '\'' +
-                '}';
-    }
+    private String coverOssPath;
+
+    private String videoOssPath;
 
     public String getStrategyType() {
         return strategyType;
@@ -179,6 +156,51 @@ public class CrawlerVideoVO {
         this.platform = platform;
     }
 
+    @Override
+    public String toString() {
+        return "CrawlerVideoVO{" +
+                "userId=" + userId +
+                ", outUserId='" + outUserId + '\'' +
+                ", userName='" + userName + '\'' +
+                ", avatarUrl='" + avatarUrl + '\'' +
+                ", platform='" + platform + '\'' +
+                ", strategy='" + strategy + '\'' +
+                ", outVideoId='" + outVideoId + '\'' +
+                ", videoTitle='" + videoTitle + '\'' +
+                ", coverUrl='" + coverUrl + '\'' +
+                ", videoUrl='" + videoUrl + '\'' +
+                ", duration=" + duration +
+                ", publishTime='" + publishTime + '\'' +
+                ", playCnt=" + playCnt +
+                ", likeCnt=" + likeCnt +
+                ", shareCnt=" + shareCnt +
+                ", collectionCnt=" + collectionCnt +
+                ", commentCnt=" + commentCnt +
+                ", crawlerRule='" + crawlerRule + '\'' +
+                ", width=" + width +
+                ", height=" + height +
+                ", strategyType='" + strategyType + '\'' +
+                ", coverOssPath='" + coverOssPath + '\'' +
+                ", videoOssPath='" + videoOssPath + '\'' +
+                '}';
+    }
+
+    public String getCoverOssPath() {
+        return coverOssPath;
+    }
+
+    public void setCoverOssPath(String coverOssPath) {
+        this.coverOssPath = coverOssPath;
+    }
+
+    public String getVideoOssPath() {
+        return videoOssPath;
+    }
+
+    public void setVideoOssPath(String videoOssPath) {
+        this.videoOssPath = videoOssPath;
+    }
+
     public String getStrategy() {
         return strategy;
     }

etl-core/src/main/java/com/tzld/crawler/etl/service/impl/EtlServiceImpl.java

@@ -34,6 +34,7 @@ import com.tzld.crawler.etl.common.enums.ExceptionEnum;
 import com.tzld.crawler.etl.common.exception.CommonException;
 import com.tzld.crawler.etl.dao.mapper.CrawlerVideoMapper;
 import com.tzld.crawler.etl.model.dto.StrategyDataDto;
+import com.tzld.crawler.etl.model.dto.VideoInfoDto;
 import com.tzld.crawler.etl.model.param.CrawlerVideoSendParam;
 import com.tzld.crawler.etl.model.po.CrawlerVideo;
 import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
@@ -45,6 +46,7 @@ import com.tzld.crawler.etl.service.strategy.StrategyHandlerService;
 import com.tzld.crawler.etl.util.FeishuUtils;
 import com.tzld.crawler.etl.util.FileUtils;
 import com.tzld.crawler.etl.util.MD5Util;
+import com.tzld.crawler.etl.util.VideoUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
@@ -95,6 +97,8 @@ public class EtlServiceImpl implements EtlService {
     private Map<String, String> feishuRangeMap;
     @Value("${admin.cms.url:https://testadmin.piaoquantv.com/cms/post-detail/%d/info}")
     private String adminCmsUrl;
+    @Value("${ffprobe.path:ffprobe}")
+    private String ffprobePath;
 
     private Executor pool;
 
@@ -110,8 +114,6 @@ public class EtlServiceImpl implements EtlService {
     @Override
     public void deal(CrawlerVideoVO param) {
         String title = param.getVideoTitle();
-        String videoUrl = param.getVideoUrl();
-        String coverUrl = param.getCoverUrl();
         String platform = param.getPlatform();
         String strategy = param.getStrategy();
         try {
@@ -125,18 +127,16 @@ public class EtlServiceImpl implements EtlService {
             }
 
             // 2.视频文件下载、上传 OSS、清理视频信息
-            String videoPath = url2oss(videoUrl, "longvideo/crawler_local/data", title, platform, strategy);
-
             // 3.视频封面下载、上传 OSS、清理视频信息
-            String coverPath = url2oss(coverUrl, "longvideo/crawler_local/image", title, platform, strategy);
+            processVideo(data);
 
             // 4.视频发布
             CrawlerVideoSendParam request = new CrawlerVideoSendParam();
             request.setLoginUid(data.getUserId());
             request.setAppType(888888);
             request.setTitle(title);
-            request.setVideoPath(videoPath);
-            request.setCoverImgPath(coverPath);
+            request.setVideoPath(data.getVideoOssPath());
+            request.setCoverImgPath(data.getCoverOssPath());
             request.setTotalTime(data.getDuration());
             request.setVersionCode(1);
             request.setViewStatus(1);
@@ -234,6 +234,40 @@ public class EtlServiceImpl implements EtlService {
         }
     }
 
+    private void processVideo(StrategyDataDto data) throws IOException {
+        String title = data.getVideoTitle();
+        String platform = data.getPlatform();
+        String strategy = data.getStrategy();
+
+        // 视频文件下载、上传 OSS
+        String videoPath = url2oss(data.getVideoUrl(), "longvideo/crawler_local/data", title, platform, strategy);
+        data.setVideoOssPath(videoPath);
+        String tempFilePath = downloadPath + File.separator + videoPath;
+        if (data.getDuration() == null || data.getDuration() <= 0
+                || data.getWidth() == null || data.getWidth() <= 0
+                || data.getHeight() == null || data.getHeight() <= 0) {
+            VideoInfoDto videoInfo = VideoUtils.getVideoInfo(tempFilePath, ffprobePath);
+            if (videoInfo != null) {
+                data.setDuration(videoInfo.getDuration());
+                data.setWidth(videoInfo.getWidth());
+                data.setHeight(videoInfo.getHeight());
+            }
+        }
+
+        // 文件清理
+        log.info("begin delete: {}", tempFilePath);
+        Files.deleteIfExists(Paths.get(new File(tempFilePath).getPath()));
+
+        // 视频封面下载、上传 OSS
+        String coverPath = url2oss(data.getCoverUrl(), "longvideo/crawler_local/image", title, platform, strategy);
+        data.setCoverOssPath(coverPath);
+        tempFilePath = downloadPath + File.separator + videoPath;
+
+        // 文件清理
+        log.info("begin delete: {}", tempFilePath);
+        Files.deleteIfExists(Paths.get(new File(tempFilePath).getPath()));
+    }
+
     private String url2oss(String fileUrl, String filePath, String title, String crawler, String mode) throws IOException {
         String titleMmd5 = MD5Util.md5(title);
         String curDate = LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE);
@@ -268,10 +302,6 @@ public class EtlServiceImpl implements EtlService {
         log.info("begin upload: {}/{} to oss key: {}/{}", localFilePath, titleMmd5, videoFilePath, titleMmd5);
         aliyunOssManager.putObject(ossBucket, videoFilePath + File.separator + titleMmd5, Files.newInputStream(Paths.get(localFilePath + File.separator + titleMmd5)));
 
-        // 文件清理
-        log.info("begin delete: {}/{}", localFilePath, titleMmd5);
-        Files.deleteIfExists(Paths.get(new File(localFilePath + File.separator + titleMmd5).getPath()));
-
         return videoFilePath + File.separator + titleMmd5;
     }
 

etl-core/src/main/java/com/tzld/crawler/etl/util/VideoUtils.java

@@ -0,0 +1,60 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.util;
+
+import com.tzld.crawler.etl.model.dto.VideoInfoDto;
+import net.bramp.ffmpeg.FFprobe;
+import net.bramp.ffmpeg.probe.FFmpegFormat;
+import net.bramp.ffmpeg.probe.FFmpegProbeResult;
+import net.bramp.ffmpeg.probe.FFmpegStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-26 18:01.
+ */
+public class VideoUtils {
+    private final static Logger log = LoggerFactory.getLogger(VideoUtils.class);
+
+    public static VideoInfoDto getVideoInfo(String videoPath, String ffprobePath) {
+        try {
+            FFprobe ffprobe = new FFprobe(ffprobePath);
+            FFmpegProbeResult probeResult = ffprobe.probe(videoPath);
+
+            FFmpegFormat format = probeResult.getFormat();
+
+            FFmpegStream stream = probeResult.getStreams().get(0);
+            return VideoInfoDto.newBuilder()
+                    .duration((long) format.duration)
+                    .height(stream.height)
+                    .width(stream.width)
+                    .build();
+        } catch (Exception e) {
+            log.error("get video {} info error.", videoPath, e);
+            return null;
+        }
+    }
+}

etl-server/src/test/java/com/tzld/crawler/etl/EtlServerApplicationTests.java

@@ -7,11 +7,16 @@ import com.google.common.collect.Lists;
 import com.huaban.analysis.jieba.JiebaSegmenter;
 import com.huaban.analysis.jieba.SegToken;
 import com.tzld.crawler.etl.mq.EtlMQConsumer;
+import net.bramp.ffmpeg.FFprobe;
+import net.bramp.ffmpeg.probe.FFmpegFormat;
+import net.bramp.ffmpeg.probe.FFmpegProbeResult;
+import net.bramp.ffmpeg.probe.FFmpegStream;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.IntStream;
@@ -67,4 +72,25 @@ class EtlServerApplicationTests {
         }
     }
 
+
+    public static void main(String[] args) throws IOException {
+        // getVideoSize("/Users/ehlxr/logs/crawler/videos/longvideo/crawler_local/video/dev/20230614/9778d5d219c5080b9a6a17bef029331c.mp4");
+        getVideoSize("http://cdn-xalbum-baishan.xiaoniangao.cn/4607211824?Expires=1704038400&OSSAccessKeyId=LTAI5tB7cRkYiqHcTdkVprwb&Signature=oFnKr5ObYT2xqNTadYEPHfzMdIM%3D");
+    }
+
+
+    public static void getVideoSize(String videoPath) throws IOException {
+        FFprobe ffprobe = new FFprobe("ffprobe");
+        FFmpegProbeResult probeResult = ffprobe.probe(videoPath);
+
+        FFmpegFormat format = probeResult.getFormat();
+
+        FFmpegStream stream = probeResult.getStreams().get(0);
+        System.out.format("Duration: %f Width: %d ; Height: %d", format.duration,
+                stream.width,
+                stream.height
+        );
+    }
+
+
 }

pom.xml

@@ -148,6 +148,12 @@
             <version>1.0.2</version>
         </dependency>
 
+        <dependency>
+            <groupId>net.bramp.ffmpeg</groupId>
+            <artifactId>ffmpeg</artifactId>
+            <version>0.7.0</version>
+        </dependency>
+
     </dependencies>
 
 </project>