下载403重新获取下载链接

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/ContentServiceImpl.java

@@ -76,7 +76,7 @@ public class ContentServiceImpl implements ContentService {
         publishMatchContentMapper.insertSelective(publishMatchContent);
     }
 
-    public MatchVideo getContent(String contentId, String ghId, Integer publishFlag) {
+    public MatchVideo  getContent(String contentId, String ghId, Integer publishFlag) {
         MatchVideoExample matchVideoExample = new MatchVideoExample();
         matchVideoExample.createCriteria().andGhIdEqualTo(ghId).andContentIdEqualTo(contentId)
                 .andPublishFlagEqualTo(publishFlag);

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/CoreServiceImpl.java

@@ -977,7 +977,8 @@ public class CoreServiceImpl implements CoreService {
                                 aigcService.updateMatchMiniprogramStatus(statusParam);
                                 continue;
                             }
-                            MatchVideo matchVideo = contentService.getContent(matchContent.getSourceId(), matchContent.getGhId(), 2);
+                            MatchVideo matchVideo = contentService.getContent(matchContent.getSourceId(),
+                                    matchContent.getGhId(), PublicFlagEnum.NOT_PUBLIC.getStatusCode());
                             if (matchVideo != null) {
                                 Integer contentStatus = matchVideo.getContentStatus();
                                 if (ContentStatusEnum.isSuccess(contentStatus)) {

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/CrawlerVideoServiceImpl.java

@@ -297,13 +297,15 @@ public class CrawlerVideoServiceImpl {
             String outVideoId = crawlerVideo.getOutVideoId();
             String videoPath = VideoDownloader.downloadVideo(outVideoId, platform, crawlerVideo.getVideoUrl());
             String coverPath = VideoDownloader.downloadCover(outVideoId, platform, crawlerVideo.getCoverUrl());
-            if (StringUtils.isNotEmpty(videoPath) && StringUtils.isNotEmpty(coverPath)) {
+            if (StringUtils.isNotEmpty(videoPath)) {
                 String videoOssPath = OSSUploader.uploadToOSS(videoPath, "video");
-                String coverOssPath = OSSUploader.uploadToOSS(coverPath, "image");
-                if (StringUtils.isNotEmpty(videoOssPath) && StringUtils.isNotEmpty(coverOssPath)) {
+                if (StringUtils.isNotEmpty(videoOssPath)) {
                     CrawlerVideo udpateCrawlerVideo = new CrawlerVideo();
                     udpateCrawlerVideo.setVideoOssPath(videoOssPath);
-                    udpateCrawlerVideo.setCoverOssPath(coverOssPath);
+                    if(StringUtils.isNotEmpty(coverPath)){
+                        String coverOssPath = OSSUploader.uploadToOSS(coverPath, "image");
+                        udpateCrawlerVideo.setCoverOssPath(coverOssPath);
+                    }
                     udpateCrawlerVideo.setId(crawlerVideo.getId());
                     udpateCrawlerVideo.setDownloadStatus(2);
                     crawlerVideoMapper.updateByPrimaryKeySelective(udpateCrawlerVideo);

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/MatchVideoServiceImpl.java

@@ -75,25 +75,26 @@ public class MatchVideoServiceImpl {
 
     // 定义一个阻塞队列
     private static final ArrayBlockingQueue<MatchVideo> matchKimiVideoQueue = new ArrayBlockingQueue<>(100000);
-    private static final int size = 5;
     // 定义一个线程池，设置消费线程的数量
-    private static final ThreadPoolExecutor matchKimiVideoPoolExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(size);
+    private static final ThreadPoolExecutor matchKimiVideoPoolExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(5);
 
 
     // 定义一个阻塞队列
     private static final ArrayBlockingQueue<MatchVideo> matchCrawlerVideoQueue = new ArrayBlockingQueue<>(100000);
     // 定义一个线程池，设置消费线程的数量
-    private static final ThreadPoolExecutor matchCrawlerVideoPoolExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(size);
+    private static final ThreadPoolExecutor matchCrawlerVideoPoolExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
 
     // 定义一个阻塞队列
     private static final ArrayBlockingQueue<MatchVideo> uploadCrawlerVideoQueue = new ArrayBlockingQueue<>(100000);
     // 定义一个线程池，设置消费线程的数量
-    private static final ThreadPoolExecutor uploadCrawlerVideoPoolExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(size);
+    private static final ThreadPoolExecutor uploadCrawlerVideoPoolExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(20);
 
     @Transactional
     public void addMatchVideo(MatchContent matchContent) {
         MatchVideoExample example = new MatchVideoExample();
-        example.createCriteria().andContentIdEqualTo(matchContent.getSourceId()).andGhIdEqualTo(matchContent.getGhId());
+        example.createCriteria().andContentIdEqualTo(matchContent.getSourceId())
+                .andGhIdEqualTo(matchContent.getGhId())
+                .andPublishFlagEqualTo(matchContent.getPublishFlag());
         long l = matchVideoMapper.countByExample(example);
         if (l > 0) {
             return;
@@ -145,7 +146,7 @@ public class MatchVideoServiceImpl {
                             throw new RuntimeException(e);
                         }
                     }
-                    log.info("启动匹配小程序线程结束");
+                    log.info("启动Kimi搜索线程结束");
                     countDownLatch.countDown();
                 }));
             }
@@ -166,6 +167,9 @@ public class MatchVideoServiceImpl {
                 page.setPageSize(1000);
                 example.setPage(page);
                 matchVideos = matchVideoMapper.selectByExample(example);
+                if (CollectionUtils.isEmpty(matchVideos)) {
+                    break;
+                }
                 boolean flag = true;
                 for (MatchVideo matchVideo : matchVideos) {
                     if (targetStatus.contains(matchVideo.getContentStatus())) {
@@ -174,6 +178,7 @@ public class MatchVideoServiceImpl {
                     }
                     //需要发布的状态需要到4才能结束
                     if (matchVideo.getContentStatus() == ContentStatusEnum.SUCCESS_3.getStatusCode() && matchVideo.getPublishFlag() == 1) {
+                        flag = false;
                         break;
                     }
                 }
@@ -359,7 +364,7 @@ public class MatchVideoServiceImpl {
         return null;
     }
 
-    public void uploadCrawlerVideo() {
+    public void uploadCrawlerVideo() throws InterruptedException {
         if (uploadCrawlerVideoPoolExecutor.getCorePoolSize() - uploadCrawlerVideoPoolExecutor.getActiveCount() > 0) {
             int threadSize = uploadCrawlerVideoPoolExecutor.getCorePoolSize() - uploadCrawlerVideoPoolExecutor.getActiveCount();
             log.info("threadNum={}", threadSize);
@@ -406,7 +411,9 @@ public class MatchVideoServiceImpl {
                     uploadCrawlerVideoQueue.addAll(matchVideos);
                 } while (!CollectionUtils.isEmpty(matchVideos));
             }
+            countDownLatch.await();
         }
+
     }
 
     public void processUploadCrawlerVideo(MatchVideo matchVideo) {

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/utils/other/DouyinSearch.java

@@ -84,7 +84,6 @@ public class DouyinSearch {
     }
 
     public static JSONObject douyinDetail(String videoId) {
-        System.out.println(videoId);
         String url = "http://crawler-cn.aiddit.com/crawler/dou_yin/detail";
         JSONObject payload = new JSONObject();
         payload.put("content_id", videoId);

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/utils/other/VideoDownloader.java

@@ -1,6 +1,8 @@
 package com.tzld.piaoquan.longarticle.utils.other;
 
 import cn.hutool.http.HttpUtil;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import com.tzld.piaoquan.longarticle.utils.ConfigUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
@@ -9,10 +11,14 @@ import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.message.BasicHeader;
+import org.springframework.util.CollectionUtils;
 
 import java.io.*;
+import java.net.Authenticator;
 import java.net.HttpURLConnection;
+import java.net.PasswordAuthentication;
 import java.net.URL;
+import java.nio.file.Files;
 import java.util.Objects;
 import java.util.UUID;
 
@@ -27,82 +33,95 @@ public class VideoDownloader {
 
     public static String downloadCover(String outVideoId, String platform, String coverUrl) {
         String path = generateCoverPath(platform, outVideoId);
-        if (download(path, coverUrl, platform)) {
+        if (download(path, coverUrl, platform) == 0) {
             return path;
         }
         return "";
     }
 
     public static String downloadVideo(String outVideoId, String platform, String videoUrl) {
-        String path = generateVideoPath(platform, outVideoId);
-        if (download(path, videoUrl, platform)) {
-            return path;
-        }
-        return "";
-    }
-
-    public static boolean download(String path, String videoUrl, String platform) {
-        try {
-            int retries = 0;
-            long fileSize = 0;
+        int retries = 0;
+        while (retries < MAX_RETRIES) {
+            String path = generateVideoPath(platform, outVideoId);
+            int download = download(path, videoUrl, platform);
+            if (download == 0) {
+                return path;
+            }
 
-            while (retries < MAX_RETRIES) {
-                File file = new File(path);
-                if (file.exists()) {
-                    file.delete();
-                }
-                HttpURLConnection connection = (HttpURLConnection) new URL(videoUrl).openConnection();
-                connection.setRequestMethod("GET");
-                connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0");
-                connection.setRequestProperty("Accept", "*/*");
-                connection.setRequestProperty("Range", "bytes=" + fileSize + "-");
-                connection.setRequestProperty("accept-language", "en,zh;q=0.9,zh-CN;q=0.8");
-                if (Objects.equals(platform, "dy_search")) {
-                    connection.setRequestProperty("referer", "https://v11-coldf.douyinvod.com/");
+            //百度重新获取链接
+            if (download == 403 && Objects.equals(platform, "baidu_search")) {
+                JSONObject videoDetail = HkspSearch.getVideoDetail(outVideoId);
+                if (videoDetail != null) {
+                    String newVideoUrl = videoDetail.getString("playurl");
+                    if (newVideoUrl != null) {
+                        videoUrl = newVideoUrl;
+                    }
                 }
+            }
 
-
-                // 设置代理
-                System.setProperty("http.proxyHost", PROXY_HOST);
-                System.setProperty("http.proxyPort", String.valueOf(PROXY_PORT));
-                System.setProperty("http.proxyUser", USERNAME);
-                System.setProperty("http.proxyPassword", PASSWORD);
-
-                // 连接并获取响应
-                connection.connect();
-
-                int responseCode = connection.getResponseCode();
-                if (responseCode == HttpURLConnection.HTTP_OK || responseCode == HttpURLConnection.HTTP_PARTIAL) {
-                    // 下载文件
-                    try (InputStream inputStream = connection.getInputStream();
-                         OutputStream outputStream = new FileOutputStream(file, true)) { // 追加模式
-                        byte[] buffer = new byte[1024 * 1024]; // 1MB buffer
-                        int bytesRead;
-                        while ((bytesRead = inputStream.read(buffer)) != -1) {
-                            outputStream.write(buffer, 0, bytesRead);
+            //抖音重新获取链接
+            if (download == 403 && Objects.equals(platform, "dy_search")) {
+                JSONObject videoDetail = DouyinSearch.douyinDetail(outVideoId);
+                if (videoDetail != null) {
+                    JSONArray videoUrlList = videoDetail.getJSONArray("video_url_list");
+                    if (!CollectionUtils.isEmpty(videoUrlList)) {
+                        String newVideoUrl = videoUrlList.getJSONObject(0).getString("video_url");
+                        if (newVideoUrl != null) {
+                            videoUrl = newVideoUrl;
                         }
                     }
+                }
+            }
+            retries++;
+        }
+        return "";
+    }
 
-                    // 检查文件是否为空
-                    if (file.length() == 0) {
-                        System.out.println("下载的文件为空，重试...");
-                        retries++;
-                        continue;
-                    } else {
-                        return true;
+    public static int download(String path, String videoUrl, String platform) {
+        try {
+            File file = new File(path);
+            if (file.exists()) {
+                file.delete();
+            }
+            // 设置代理
+            System.setProperty("http.proxyHost", PROXY_HOST);
+            System.setProperty("http.proxyPort", String.valueOf(PROXY_PORT));
+            System.setProperty("http.proxyUser", USERNAME);
+            System.setProperty("http.proxyPassword", PASSWORD);
+
+            HttpURLConnection connection = (HttpURLConnection) new URL(videoUrl).openConnection();
+            connection.setRequestMethod("GET");
+            connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0");
+            connection.setRequestProperty("Accept", "*/*");
+            connection.setRequestProperty("accept-language", "en,zh;q=0.9,zh-CN;q=0.8");
+            if (Objects.equals(platform, "dy_search")) {
+                connection.setRequestProperty("referer", "https://v11-coldf.douyinvod.com/");
+            }
+            // 连接并获取响应
+            connection.connect();
+
+            int responseCode = connection.getResponseCode();
+            if (responseCode == HttpURLConnection.HTTP_OK || responseCode == HttpURLConnection.HTTP_PARTIAL) {
+                // 下载文件
+                try (InputStream inputStream = connection.getInputStream();
+                     OutputStream outputStream = Files.newOutputStream(file.toPath())) { // 追加模式
+                    byte[] buffer = new byte[1024 * 1024]; // 1MB buffer
+                    int bytesRead;
+                    while ((bytesRead = inputStream.read(buffer)) != -1) {
+                        outputStream.write(buffer, 0, bytesRead);
                     }
-                } else {
-                    System.out.println("下载失败，HTTP 状态码：" + responseCode);
-                    retries++;
+                }
+                // 检查文件是否为空
+                if (file.length() != 0) {
+                    return 0;
                 }
             }
 
-            log.error("下载失败，已达到最大重试次数：" + MAX_RETRIES);
-            return false;
+            return responseCode;
         } catch (Exception e) {
             log.error("download error", e);
         }
-        return false;
+        return -1;
     }
 
     private static String generateVideoPath(String platform, String videoId) {
@@ -119,4 +138,17 @@ public class VideoDownloader {
         return String.join(File.separator, ConfigUtil.getConfig("download.path"), coverName); // 返回文件路径和封面路径
     }
 
+    public static void main(String[] args) {
+        System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
+        Authenticator.setDefault(
+                new Authenticator() {
+                    public PasswordAuthentication getPasswordAuthentication() {
+                        return new PasswordAuthentication(
+                                "t11983523373311", "mtuhdr2z".toCharArray());
+                    }
+                }
+        );
+        downloadVideo("2562802497357206175", "baidu_search", "https://vdept3.bdstatic.com/mda-qkt120240g2y8uka/cae_h264/1732792181901415854/mda-qkt120240g2y8uka.mp4?v_from_s=hkapp-haokan-suzhou&auth_key=1738759058-0-0-115429d241bdab4208d998b5f3f95507&bcevod_channel=searchbox_feed&cr=0&cd=0&pd=1&pt=3&logid=2258462520&vid=2562802497357206175&abtest=132219_1");
+    }
+
 }