|
|
@@ -1,8 +1,6 @@
|
|
|
package com.tzld.piaoquan.longarticle.utils.other;
|
|
|
|
|
|
import cn.hutool.crypto.digest.DigestUtil;
|
|
|
-import cn.hutool.http.HttpRequest;
|
|
|
-import cn.hutool.http.HttpResponse;
|
|
|
import com.alibaba.fastjson.JSONArray;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
import okhttp3.*;
|
|
|
@@ -13,15 +11,46 @@ import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
|
import java.net.*;
|
|
|
import okhttp3.Authenticator;
|
|
|
+
|
|
|
import java.security.MessageDigest;
|
|
|
import java.security.NoSuchAlgorithmException;
|
|
|
import java.util.*;
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
|
|
|
import static com.tzld.piaoquan.longarticle.common.constants.ProxyConstant.*;
|
|
|
|
|
|
@Slf4j
|
|
|
public class HkspSearch {
|
|
|
|
|
|
+ private static OkHttpClient okHttpClient;
|
|
|
+
|
|
|
+ static {
|
|
|
+ createClient();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static synchronized void createClient() {
|
|
|
+ try {
|
|
|
+ Proxy proxy = new Proxy(
|
|
|
+ Proxy.Type.HTTP,
|
|
|
+ new InetSocketAddress(PROXY_HOST, PROXY_PORT)
|
|
|
+ );
|
|
|
+ Authenticator authenticator = (route, response) -> {
|
|
|
+ String credential = Credentials.basic(USERNAME, PASSWORD);
|
|
|
+ return response.request().newBuilder()
|
|
|
+ .header("Proxy-Authorization", credential)
|
|
|
+ .build();
|
|
|
+ };
|
|
|
+ okHttpClient = new OkHttpClient.Builder()
|
|
|
+ .proxy(proxy)
|
|
|
+ .proxyAuthenticator(authenticator)
|
|
|
+ .connectTimeout(15, TimeUnit.MINUTES)
|
|
|
+ .readTimeout(15, TimeUnit.MINUTES)
|
|
|
+ .writeTimeout(15, TimeUnit.MINUTES)
|
|
|
+ .build();
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("Failed to create HkspSearch client", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
public static CrawlerVideo baiduVideoProduce(JSONObject jsonObject) {
|
|
|
CrawlerVideo crawlerVideo = new CrawlerVideo();
|
|
|
@@ -55,20 +84,6 @@ public class HkspSearch {
|
|
|
|
|
|
public static List<JSONObject> hkspSearch(String key, List<String> sensitiveWords, String traceId) {
|
|
|
try {
|
|
|
- Proxy proxy = new Proxy(
|
|
|
- Proxy.Type.HTTP,
|
|
|
- new InetSocketAddress(PROXY_HOST, PROXY_PORT)
|
|
|
- );
|
|
|
- okhttp3.Authenticator authenticator = (route, response) -> {
|
|
|
- String credential = Credentials.basic(USERNAME, PASSWORD);
|
|
|
- return response.request().newBuilder()
|
|
|
- .header("Proxy-Authorization", credential)
|
|
|
- .build();
|
|
|
- };
|
|
|
- OkHttpClient client = new OkHttpClient.Builder()
|
|
|
- .proxy(proxy)
|
|
|
- .proxyAuthenticator(authenticator)
|
|
|
- .build();
|
|
|
|
|
|
String encodedKey = URLEncoder.encode(key, "UTF-8");
|
|
|
long timestampMilliseconds = System.currentTimeMillis();
|
|
|
@@ -97,7 +112,7 @@ public class HkspSearch {
|
|
|
.get()
|
|
|
.build();
|
|
|
|
|
|
- Response response = client.newCall(request).execute();
|
|
|
+ Response response = okHttpClient.newCall(request).execute();
|
|
|
String result = response.body().string();
|
|
|
|
|
|
List<JSONObject> resultList = new ArrayList<>();
|
|
|
@@ -158,26 +173,44 @@ public class HkspSearch {
|
|
|
|
|
|
public static JSONObject getVideoDetail(String videoId) {
|
|
|
try {
|
|
|
- String url = "https://haokan.baidu.com/v";
|
|
|
- JSONObject params = new JSONObject();
|
|
|
- params.put("vid", videoId);
|
|
|
- params.put("_format", "json");
|
|
|
+ // 构建URL并添加查询参数
|
|
|
+ HttpUrl url = HttpUrl.parse("https://haokan.baidu.com/v")
|
|
|
+ .newBuilder()
|
|
|
+ .addQueryParameter("vid", videoId)
|
|
|
+ .addQueryParameter("_format", "json")
|
|
|
+ .build();
|
|
|
|
|
|
+ // 生成BIDUPSID
|
|
|
String base64String = Base64.getEncoder().encodeToString(UUID.randomUUID().toString().getBytes());
|
|
|
- HttpResponse response = HttpRequest.get(url)
|
|
|
- .header("Accept", "*/*")
|
|
|
- .header("cookie", "BIDUPSID=" + base64String)
|
|
|
- .header("Accept-Language", "en,zh;q=0.9,zh-CN;q=0.8")
|
|
|
- .header("Cache-Control", "no-cache")
|
|
|
- .header("Connection", "keep-alive")
|
|
|
- .header("Content-Type", "application/x-www-form-urlencoded")
|
|
|
- .header("Referer", "https://haokan.baidu.com")
|
|
|
- .header("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
|
|
|
- .timeout(120000) // 设置超时时间
|
|
|
- .form(params)
|
|
|
- .setProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(PROXY_HOST, PROXY_PORT)))
|
|
|
- .execute();
|
|
|
- return JSONObject.parseObject(response.body()).getJSONObject("data").getJSONObject("apiData").getJSONObject("curVideoMeta");
|
|
|
+
|
|
|
+ // 构建请求头
|
|
|
+ Headers headers = Headers.of(
|
|
|
+ "Accept", "*/*",
|
|
|
+ "cookie", "BIDUPSID=" + base64String,
|
|
|
+ "Accept-Language", "en,zh;q=0.9,zh-CN;q=0.8",
|
|
|
+ "Cache-Control", "no-cache",
|
|
|
+ "Connection", "keep-alive",
|
|
|
+ "Content-Type", "application/x-www-form-urlencoded",
|
|
|
+ "Referer", "https://haokan.baidu.com",
|
|
|
+ "User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
|
|
+ );
|
|
|
+
|
|
|
+ // 构建请求
|
|
|
+ Request request = new Request.Builder()
|
|
|
+ .url(url)
|
|
|
+ .headers(headers)
|
|
|
+ .get()
|
|
|
+ .build();
|
|
|
+
|
|
|
+ // 发送请求并获取响应
|
|
|
+ Response response = okHttpClient.newCall(request).execute();
|
|
|
+ String responseBody = response.body().string();
|
|
|
+
|
|
|
+ // 解析响应
|
|
|
+ return JSONObject.parseObject(responseBody)
|
|
|
+ .getJSONObject("data")
|
|
|
+ .getJSONObject("apiData")
|
|
|
+ .getJSONObject("curVideoMeta");
|
|
|
} catch (Exception e) {
|
|
|
log.error("getVideoDetail error", e);
|
|
|
}
|