trust all cert

etl-core/src/main/java/com/tzld/crawler/etl/util/FileUtils.java

@@ -140,6 +140,7 @@ public class FileUtils {
 
     public static void downloadForGZH(String fileUrl, String filePath, boolean useUa, boolean useProxy, Map<String, String> proxyInfo) throws Exception {
         log.info("begin download [{}] to [{}] useUa [{}] useProxy [{}] proxyInfo[{}]", fileUrl, filePath, useUa, useProxy, proxyInfo);
+        SSLUtils.ignoreSsl();
         URL url = new URL(fileUrl);
         HttpURLConnection conn;
         if (useProxy) {
@@ -193,7 +194,7 @@ public class FileUtils {
 
     public static void main(String[] args) throws Exception {
         // try {
-        download("https://mpvideo.qpic.cn/0b2eamabmaaaiaakmkq3jfsvaa6dcybqafqa.f10002.mp4?dis_k=af0809430b6148733c161cb94368bd30&dis_t=1696833857&play_scene=10120&auth_info=LdSvh8hZbHlJiPWF6A9dSW1HMR9TdjtFUl17LX1YNxkLblZiChITAzhoFzJxNFFq&auth_key=5fca6f1024c9f40e9f6ad50aad0e437e&vid=wxv_3132660909191184387&format_id=10002&support_redirect=0&mmversion=false", "/Users/dingyunpeng/Downloads/" + System.currentTimeMillis(), true);
+        download("https://mpvideo.qpic.cn/0bc3tuacuaaataaeceuifbrvbhodfkoqakqa.f10004.mp4?dis_k=dd10b558d25ce61609d74e0bbce16511&dis_t=1696852869&play_scene=10120&auth_info=SoLXgsJxHz1WiLfcsFknajhsMkhsN0djSkl6ZA1TQSRsM1doU31CUjRgMRMReDBQMg==&auth_key=c898b13e9980e164e21174ab60c10e0e&vid=wxv_2689260325199855618&format_id=10004&support_redirect=0&mmversion=false", "/Users/dingyunpeng/Downloads/" + System.currentTimeMillis(), true);
         //
         // } catch (Exception e) {
         //     e.printStackTrace();

etl-core/src/main/java/com/tzld/crawler/etl/util/SSLUtils.java

@@ -0,0 +1,61 @@
+package com.tzld.crawler.etl.util;
+
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+public class SSLUtils {
+
+    public static void trustAllHttpsCertificates() throws Exception {
+        TrustManager[] trustAllCerts = new TrustManager[1];
+        TrustManager tm = new miTM();
+        trustAllCerts[0] = tm;
+        SSLContext sc = SSLContext.getInstance("SSL");
+        sc.init(null, trustAllCerts, null);
+        HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+    }
+
+    static class miTM implements TrustManager,X509TrustManager {
+        public X509Certificate[] getAcceptedIssuers() {
+            return null;
+        }
+
+        public boolean isServerTrusted(X509Certificate[] certs) {
+            return true;
+        }
+
+        public boolean isClientTrusted(X509Certificate[] certs) {
+            return true;
+        }
+
+        public void checkServerTrusted(X509Certificate[] certs, String authType)
+                throws CertificateException {
+            return;
+        }
+
+        public void checkClientTrusted(X509Certificate[] certs, String authType)
+                throws CertificateException {
+            return;
+        }
+    }
+
+    /**
+     * 忽略HTTPS请求的SSL证书，必须在openConnection之前调用
+     * @throws Exception
+     */
+    public static void ignoreSsl() throws Exception{
+        HostnameVerifier hv = new HostnameVerifier() {
+            public boolean verify(String urlHostName, SSLSession session) {
+                return true;
+            }
+        };
+        trustAllHttpsCertificates();
+        HttpsURLConnection.setDefaultHostnameVerifier(hv);
+    }
+}