spider

core/src/main/java/com/tzld/supply/annotation/NoRequestLog.java

@@ -0,0 +1,15 @@
+package com.tzld.supply.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * 请求日志排除注解
+ *
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface NoRequestLog {
+}

core/src/main/java/com/tzld/supply/api/SpiderApiService.java

@@ -31,7 +31,7 @@ public class SpiderApiService {
      * 查询内容热榜请求
      * @param sortType 排序类型
      * @param category 分类
-     * @param cursor 分页游标
+     * @param cursor   分页游标
      * @return ContentRankResponse 响应对象
      * @throws IOException 如果请求失败
      */
@@ -66,28 +66,42 @@ public class SpiderApiService {
      * @return ContentSearchResponse.DataItem 响应对象
      */
     public List<ContentSearchResponse.DataItem> searchContentDetail(String title) {
-        // todo 搜索内容详情
-        String url = "http://crawapi.piaoquantv.com/crawler/jin_ri_re_bang/content_rank";
+        // 搜索内容详情
+        String url = "http://crawapi.piaoquantv.com/crawler/bai_du/keyword";
         JSONObject param = new JSONObject();
-        param.put("title", title);
+        param.put("keyword", title);
         RequestBody body = RequestBody.create(MediaType.parse("application/json; charset=utf-8"), param.toJSONString());
         Request request = new Request.Builder()
                 .url(url)
                 .post(body)
                 .build();
-        try {
-            Response response = client.newCall(request).execute();
-            if (response.isSuccessful()) {
-                String responseBody = response.body().string();
-                // 将 JSON 响应映射到 ContentSearchResponse 对象
-                JSONObject jsonObject = JSONObject.parseObject(responseBody);
-                ContentSearchResponse contentSearchResponse = jsonObject.getObject("data", ContentSearchResponse.class);
-                if (contentSearchResponse != null && contentSearchResponse.getData() != null) {
-                    return contentSearchResponse.getData();
+        int maxRetries = 3; // 最大重试次数
+        int retryCount = 0; // 当前重试计数
+        while (retryCount < maxRetries) {
+            try {
+                Response response = client.newCall(request).execute();
+                if (response.isSuccessful()) {
+                    // 请求成功，解析响应并返回
+                    String responseBody = response.body().string();
+                    // 将 JSON 响应映射到 ContentSearchResponse 对象
+                    JSONObject jsonObject = JSONObject.parseObject(responseBody);
+                    ContentSearchResponse contentSearchResponse = jsonObject.getObject("data", ContentSearchResponse.class);
+                    if (contentSearchResponse != null && contentSearchResponse.getData() != null) {
+                        return contentSearchResponse.getData();
+                    }
+                }
+            } catch (IOException e) {
+                log.error("搜索内容详情失败: {}", e.getMessage(), e);
+            }
+            retryCount++;
+            // 如果还有重试次数，添加短暂延迟后重试
+            if (retryCount < maxRetries) {
+                try {
+                    TimeUnit.MILLISECONDS.sleep(2000); // 2秒延迟
+                } catch (InterruptedException ie) {
+                    log.error("重试被中断: {}", ie.getMessage(), ie);
                 }
             }
-        } catch (IOException e) {
-            log.error("搜索内容详情失败: {}", e.getMessage(), e);
         }
         return null;
     }

core/src/main/java/com/tzld/supply/job/ContentScreenJob.java

@@ -135,8 +135,8 @@ public class ContentScreenJob {
                 Integer aiPrecisionStatus = checkPrecisionScreenStatus(obj);
                 content.setAiPrecisionStatus(aiPrecisionStatus);
                 if (aiPrecisionStatus == 1
-                        && obj.getPropagationScore() > 0.8
-                        && obj.getSuitabilityScore() > 0.8) {
+                        && obj.getPropagationScore() >= 0.8
+                        && obj.getSuitabilityScore() >= 0.8) {
                     content.setStatus(2);
                     content.setAiPrecisionResult(JSONObject.toJSONString(obj));
                 } else {

core/src/main/java/com/tzld/supply/util/IpUtil.java

@@ -0,0 +1,69 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2021 xrv <xrg@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.supply.util;
+
+import javax.servlet.http.HttpServletRequest;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * @author ehlxr
+ * @since 2021-12-14 15:27.
+ */
+public class IpUtil {
+    public static String getIpAddr(HttpServletRequest request) {
+        String ipAddress;
+        ipAddress = request.getHeader("x-forwarded-for");
+        if (ipAddress == null || ipAddress.length() == 0
+                || "unknown".equalsIgnoreCase(ipAddress)) {
+            ipAddress = request.getHeader("Proxy-Client-IP");
+        }
+        if (ipAddress == null || ipAddress.length() == 0
+                || "unknown".equalsIgnoreCase(ipAddress)) {
+            ipAddress = request.getHeader("WL-Proxy-Client-IP");
+        }
+        if (ipAddress == null || ipAddress.length() == 0
+                || "unknown".equalsIgnoreCase(ipAddress)) {
+            ipAddress = request.getRemoteAddr();
+            if ("127.0.0.1".equals(ipAddress)) {
+                // 根据网卡取本机配置的IP
+                InetAddress inet;
+                try {
+                    inet = InetAddress.getLocalHost();
+                    ipAddress = inet.getHostAddress();
+                } catch (UnknownHostException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+        // 对于通过多个代理的情况，第一个IP为客户端真实IP,多个IP按照','分割
+        if (ipAddress != null && ipAddress.length() > 15) { // "***.***.***.***".length()
+            if (ipAddress.indexOf(",") > 0) {
+                ipAddress = ipAddress.substring(0, ipAddress.indexOf(","));
+            }
+        }
+        return ipAddress;
+    }
+}

core/src/main/java/com/tzld/supply/util/Md5Util.java

@@ -0,0 +1,72 @@
+package com.tzld.supply.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+
+public class Md5Util {
+	static char hexDigits[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };  
+	
+	public static final String encoderByMd5(String msg) {
+		try {
+			byte[] strTemp = msg.getBytes();
+            //如果输入“SHA”，就是实现SHA加密。
+			MessageDigest mdTemp = MessageDigest.getInstance("MD5"); 
+			mdTemp.update(strTemp);
+			byte[] md = mdTemp.digest();
+			int j = md.length;
+			char str[] = new char[j * 2];
+			int k = 0;
+			for (int i = 0; i < j; i++) {
+				byte byte0 = md[i];
+				str[k++] = hexDigits[byte0 >>> 4 & 0xf];
+				str[k++] = hexDigits[byte0 & 0xf];
+			}
+			return new String(str);
+		} catch (Exception e) {
+			return null;
+		}
+	}
+
+	public static final String encoderFileByMd5(InputStream is){
+        try {  
+            MessageDigest md = MessageDigest.getInstance("MD5");  
+            byte[] buffer = new byte[2048];  
+            int length = -1;  
+            while ((length = is.read(buffer)) != -1) {  
+                md.update(buffer, 0, length);  
+            }  
+            byte[] b = md.digest();  
+            return byteToHexString(b);  
+        } catch (Exception e) {  
+            e.printStackTrace();  
+            return null;  
+        } finally {  
+            try {  
+                is.close();  
+            } catch (IOException e) {  
+                e.printStackTrace();  
+            }  
+        }  
+	}
+	
+	private static String byteToHexString(byte[] tmp) {  
+        String s;  
+        // 用字节表示就是 16 个字节  
+        // 每个字节用 16 进制表示的话，使用两个字符，所以表示成 16 进制需要 32 个字符  
+        // 比如一个字节为01011011，用十六进制字符来表示就是“5b”  
+        char str[] = new char[16 * 2];  
+        int k = 0; // 表示转换结果中对应的字符位置  
+        for (int i = 0; i < 16; i++) { // 从第一个字节开始，对 MD5 的每一个字节转换成 16 进制字符的转换  
+            byte byte0 = tmp[i]; // 取第 i 个字节  
+            str[k++] = hexDigits[byte0 >>> 4 & 0xf]; // 取字节中高 4 位的数字转换, >>> 为逻辑右移，将符号位一起右移  
+            str[k++] = hexDigits[byte0 & 0xf]; // 取字节中低 4 位的数字转换  
+        }  
+        s = new String(str); // 换后的结果转换为字符串  
+        return s;  
+    }  
+	
+	public static void main(String[] args) {
+		System.out.println(encoderByMd5("test1002"));
+	}
+}

server/src/main/java/com/tzld/supply/aop/LogRequestAop.java

@@ -0,0 +1,159 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2021 xrv <xrg@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.supply.aop;
+
+import com.alibaba.fastjson.JSON;
+import com.google.common.base.Strings;
+import com.tzld.commons.aliyun.log.AliyunLogManager;
+import com.tzld.supply.util.IpUtil;
+import org.apache.commons.lang3.StringUtils;
+import org.aspectj.lang.JoinPoint;
+import org.aspectj.lang.annotation.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.request.RequestContextHolder;
+import org.springframework.web.context.request.ServletRequestAttributes;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * 请求日志记录
+ *
+ * @author ehlxr
+ * @since 2021-12-14 14:03.
+ */
+@Aspect
+@Component
+public class LogRequestAop {
+    private static final Logger log = LoggerFactory.getLogger(LogRequestAop.class);
+
+    private final AliyunLogManager aliyunLogManager;
+    /**
+     * aliyun log 配置
+     */
+    @Value("${aliyun.log.project:}")
+    private String projcet;
+    @Value("${aliyun.log.logstore.request:request-log}")
+    private String logStore;
+    @Value("${aliyun.log.topic:}")
+    private String topic;
+
+    public static String LOG_TRACE_ID = "logTraceId";
+    private String REQUEST_START_TIME = "request_start_time";
+
+    @Autowired
+    public LogRequestAop(AliyunLogManager aliyunLogManager) {
+        this.aliyunLogManager = aliyunLogManager;
+    }
+
+    /**
+     * 切入点
+     */
+    @Pointcut("execution(public * com.tzld.supply.controller..*(..)) " +
+            "&& !@annotation(com.tzld.supply.annotation.NoRequestLog)")
+    public void requestLog() {
+    }
+
+    /**
+     * 前置操作
+     */
+    @Before("requestLog()")
+    public void beforeLog() {
+        ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
+        HttpServletRequest request = Objects.requireNonNull(attributes).getRequest();
+
+        request.setAttribute(REQUEST_START_TIME, System.currentTimeMillis());
+    }
+
+    /**
+     * 后置操作
+     */
+    @AfterReturning(pointcut = "requestLog()", returning = "returnValue")
+    public void afterReturning(JoinPoint point, Object returnValue) {
+        logRecord(point, JSON.toJSONString(returnValue));
+    }
+
+    @AfterThrowing(pointcut = "requestLog()", throwing = "ex")
+    public void afterThrowing(JoinPoint point, Exception ex) {
+        logRecord(point, ex.toString());
+    }
+
+    private void logRecord(JoinPoint point, String message) {
+        try {
+            ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
+            HttpServletRequest request = Objects.requireNonNull(attributes).getRequest();
+
+            Long startTime = (Long) request.getAttribute(REQUEST_START_TIME);
+            if (Objects.isNull(startTime)) {
+                return;
+            }
+
+            String query = request.getQueryString();
+            String uri = request.getRequestURI();
+
+            Map<String, Object> logMap = new HashMap<>(8);
+            String url = Strings.isNullOrEmpty(query) ? uri : uri + "?" + query;
+            if (StringUtils.isNotBlank(url) && url.contains("/test")) {
+                return;
+            }
+            logMap.put("url", url);
+            logMap.put("method", request.getMethod());
+            logMap.put("header", getHeaders(request));
+            logMap.put("elapsedTime", String.valueOf(System.currentTimeMillis() - startTime));
+            logMap.put("clientIp", IpUtil.getIpAddr(request));
+            logMap.put("requestBody", getRequestBody(point.getArgs()));
+            logMap.put(LOG_TRACE_ID, Strings.nullToEmpty(MDC.get(LOG_TRACE_ID)));
+            logMap.put("responseBody", message);
+            aliyunLogManager.sendLog(projcet, logStore, topic, logMap);
+        } catch (Exception e) {
+            log.error("log report request error", e);
+        }
+    }
+
+    private Object getRequestBody(Object[] args) {
+        try {
+            return JSON.toJSONString(args[0]);
+        } catch (Exception e) {
+            return args;
+        }
+    }
+
+    private String getHeaders(HttpServletRequest request) {
+        return Collections.list(request.getHeaderNames())
+                .stream()
+                .collect(Collectors.toMap(
+                        name -> name,
+                        request::getHeader)).toString();
+    }
+}

server/src/main/java/com/tzld/supply/controller/HealthController.java

@@ -0,0 +1,21 @@
+package com.tzld.supply.controller;
+
+import com.tzld.supply.annotation.NoRequestLog;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/")
+public class HealthController {
+
+    /**
+     * 探活
+     * @return
+     */
+    @GetMapping("/healthcheck")
+    @NoRequestLog
+    public String healthcheck() {
+        return "ok";
+    }
+}

server/src/main/java/com/tzld/supply/controller/SpiderTaskController.java

@@ -1,26 +0,0 @@
-package com.tzld.supply.controller;
-
-import com.tzld.supply.common.base.CommonResponse;
-import com.tzld.supply.model.po.supply.spider.SpiderTask;
-import com.tzld.supply.service.SpiderTaskService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-import java.util.List;
-
-@RestController
-@RequestMapping("/api/spider-tasks")
-public class SpiderTaskController {
-
-    @Autowired
-    private SpiderTaskService spiderTaskService;
-
-
-    @GetMapping("/list")
-    public CommonResponse<List<SpiderTask>> getSpiderTaskList() {
-        List<SpiderTask> taskList = spiderTaskService.getSpiderTaskList();
-        return CommonResponse.success(taskList);
-    }
-}