下载公众号视频优化

etl-core/src/main/java/com/tzld/crawler/etl/util/FileUtils.java

@@ -276,60 +276,76 @@ public class FileUtils {
         SSLUtils.ignoreSsl();
         URL url = new URL(fileUrl);
         HttpURLConnection conn;
-        if (useProxy) {
-            String proxyUrl = proxyInfo.getOrDefault("url", "");
-            int port = Integer.parseInt(proxyInfo.getOrDefault("port", "0"));
-            String username = proxyInfo.getOrDefault("username", "");
-            String password = proxyInfo.getOrDefault("password", "");
-            // 创建代理服务的地址和端口
-            Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyUrl, port));
-            Authenticator authenticator = new Authenticator() {
-                @Override
-                public PasswordAuthentication getPasswordAuthentication() {
-                    return (new PasswordAuthentication(username, password.toCharArray()));
+        HttpHost proxy = new HttpHost("q796.kdltps.com", 15818);
+        CredentialsProvider credsProvider = new BasicCredentialsProvider();
+        credsProvider.setCredentials(
+                new AuthScope(proxy),
+                new UsernamePasswordCredentials("t17772369458618", "5zqcjkmy")
+        );
+        try (CloseableHttpClient client = HttpClients.custom().setDefaultCredentialsProvider(credsProvider).setProxy(proxy).build()) {
+            HttpGet request = new HttpGet(String.valueOf(url));
+            request.setHeader("User-Agent", FakeUserAgent.getRandomUserAgent());
+            request.setHeader("Accept-Encoding", "identity;q=1, *;q=0");
+            request.setHeader("Accept", "*/*");
+            request.setHeader("Host", "mpvideo.qpic.cn");
+            request.setHeader("origin", "https://mp.weixin.qq.com");
+            request.setHeader("Referer", "https://mp.weixin.qq.com/");
+            request.setHeader("Accept-Language", "en-US,en;q=0.9");
+            HttpResponse response = client.execute(request);
+            try (
+                    InputStream is = response.getEntity().getContent();
+                    OutputStream os = Files.newOutputStream(Paths.get(filePath)))
+                {
+                    byte[] buffer = new byte[1024];
+                    int bytesRead;
+                    while ((bytesRead = is.read(buffer)) != -1) {
+                        os.write(buffer, 0, bytesRead);
+                    }
+                    System.out.println("Video download complete.");
                 }
-            };
-            Authenticator.setDefault(authenticator);
-            conn = (HttpURLConnection) url.openConnection(proxy);
-        } else {
-            conn = (HttpURLConnection) url.openConnection();
+            catch (Exception e){
+                System.out.println("Download failed with exception: " + e.getMessage());
+            }
         }
-
-        if (useUa) {
-            conn.setRequestProperty("User-Agent", FakeUserAgent.getRandomUserAgent());
+        catch (Exception e){
+            System.out.println("Download failed with exception: " + e.getMessage());
         }
 
-        conn.setRequestProperty("Accept", "*/*");
-        conn.setRequestProperty("Accept-Encoding", "identity;q=1, *;q=0");
-        conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9");
-        conn.setRequestProperty("Host", "mpvideo.qpic.cn");
-        conn.setRequestProperty("Origin", "https://mp.weixin.qq.com");
-        conn.setRequestProperty("Referer", "https://mp.weixin.qq.com");
-
-        if (conn.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN) {
-            throw new CommonException(ExceptionEnum.URL_FORBIDDEN);
-        }
-        conn.setConnectTimeout(5000);
-        conn.setReadTimeout(5000);
-        log.info("download file size is {} of url [{}]", formatFileSize(conn.getContentLength()), fileUrl);
 
-        InputStream inputStream = conn.getInputStream();
-        FileOutputStream outputStream = new FileOutputStream(filePath);
-        byte[] buffer = new byte[4096];
-        int len;
-        while ((len = inputStream.read(buffer)) != -1) {
-            outputStream.write(buffer, 0, len);
-        }
-        inputStream.close();
-        outputStream.close();
-        log.info("downloaded successfully [{}] to [{}]", fileUrl, filePath);
+//        conn.setRequestProperty("User-Agent", );
+//        conn.setRequestProperty("Accept", "*/*");
+//        conn.setRequestProperty("Accept-Encoding", "identity;q=1, *;q=0");
+//        conn.setRequestProperty("Connection", "keep-alive");
+//        conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9");
+//        conn.setRequestProperty("Host", "mpvideo.qpic.cn");
+//        conn.setRequestProperty("Origin", "https://mp.weixin.qq.com");
+//        conn.setRequestProperty("Referer", "https://mp.weixin.qq.com/");
+//
+//
+//        if (conn.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN) {
+//            throw new CommonException(ExceptionEnum.URL_FORBIDDEN);
+//        }
+//        conn.setConnectTimeout(5000);
+//        conn.setReadTimeout(5000);
+//        log.info("download file size is {} of url [{}]", formatFileSize(conn.getContentLength()), fileUrl);
+//
+//        InputStream inputStream = conn.getInputStream();
+//        FileOutputStream outputStream = new FileOutputStream(filePath);
+//        byte[] buffer = new byte[4096];
+//        int len;
+//        while ((len = inputStream.read(buffer)) != -1) {
+//            outputStream.write(buffer, 0, len);
+//        }
+//        inputStream.close();
+//        outputStream.close();
+//        log.info("downloaded successfully [{}] to [{}]", fileUrl, filePath);
     }
 
     public static void main(String[] args) throws Exception {
         // try {
 //        System.out.println("https://api-hl.huoshan.com/hotsoon/item/video/_source/?video_id=v0300fg10000ckmbrbbc77uc3nq19840&line=0&app_id=0&vquality=normal&watermark=0&long_video=0&sf=4&ts=1697528496&item_id=7290410334844718376".replace("https://api-hl.huoshan.com", "http://api-hl.huoshan.com"));
-        downloadForFQW(
-                "https://znl-video-bos.cdn.bcebos.com/e368801a814c548e443835086d37caaf/65e93632/video/20240306/820ee1498e3ed2a59d37aed54d39ae95_1.mp4",
+        downloadForGZH(
+                "https://mpvideo.qpic.cn/0bc3siadqaaa4uamj42j4nsvbewdhcjaaoaa.f10002.mp4?dis_k=119b904f4b94954dd215d29d963d323d&dis_t=1710471237&play_scene=10120&auth_info=D7qugVRFbFm0+6STCiFrOW5pTjZkRTFMSnhkWFNIJGA2W2pVKhgDO2QzR0wuNlEz&auth_key=ccc54f3579c8abf94fd7ebd601f697d1&vid=wxv_3183565730357952514&format_id=10002&support_redirect=0&mmversion=false",
                 "/Users/luojunhui/Downloads/" + System.currentTimeMillis() + ".mp4",
                 true,
                 false,