Merge branch 'wyp/0113-sensitiveCache' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/SensitiveStrategy.java

@@ -1,7 +1,8 @@
 package com.tzld.longarticle.recommend.server.service.recommend.filter.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleSensitive;
@@ -18,10 +19,9 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 @Component
 @Slf4j
@@ -37,7 +37,8 @@ public class SensitiveStrategy implements FilterStrategy {
     @ApolloJsonValue("${UnSafeTitles:[]}")
     private static List<String> UnSafeTitles;
 
-    private final ExecutorService pool = ThreadPoolFactory.deDuplicatePool();
+    Cache<String, Boolean> similarityCache = CacheBuilder.newBuilder()
+            .expireAfterWrite(60 * 15, TimeUnit.SECONDS).build();
 
     @Override
     public FilterResult filter(FilterParam param) {
@@ -65,10 +66,19 @@ public class SensitiveStrategy implements FilterStrategy {
             }
         }
         List<Set<Character>> unsafeTitleCache = TitleSimilarCheckUtil.makeCache(allUnSafeTitles);
+        List<String> allTitles = param.getContents().stream().map(Content::getTitle).collect(Collectors.toList());
+        Map<String, Boolean> similarityMap = new HashMap<>(similarityCache.getAllPresent(allTitles));
         for (Content content : param.getContents()) {
             try {
-                if (TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), unsafeTitleCache,
-                        TitleSimilarCheckUtil.SIMILARITY_THRESHOLD)) {
+                Boolean isSimilar;
+                if (similarityMap.containsKey(content.getTitle())) {
+                    isSimilar = similarityMap.get(content.getTitle());
+                } else {
+                    isSimilar = TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), unsafeTitleCache,
+                            TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
+                    similarityCache.put(content.getTitle(), isSimilar);
+                }
+                if (isSimilar) {
                     content.setFilterReason("安全违规");
                     filterContents.add(content);
                 } else {