|
|
@@ -1,7 +1,8 @@
|
|
|
package com.tzld.longarticle.recommend.server.service.recommend.filter.strategy;
|
|
|
|
|
|
import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
-import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
|
|
|
+import com.google.common.cache.Cache;
|
|
|
+import com.google.common.cache.CacheBuilder;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
|
|
|
import com.tzld.longarticle.recommend.server.model.dto.Content;
|
|
|
import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleSensitive;
|
|
|
@@ -18,10 +19,9 @@ import org.apache.commons.collections4.CollectionUtils;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
-import java.util.ArrayList;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Set;
|
|
|
-import java.util.concurrent.ExecutorService;
|
|
|
+import java.util.*;
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
+import java.util.stream.Collectors;
|
|
|
|
|
|
@Component
|
|
|
@Slf4j
|
|
|
@@ -37,7 +37,8 @@ public class SensitiveStrategy implements FilterStrategy {
|
|
|
@ApolloJsonValue("${UnSafeTitles:[]}")
|
|
|
private static List<String> UnSafeTitles;
|
|
|
|
|
|
- private final ExecutorService pool = ThreadPoolFactory.deDuplicatePool();
|
|
|
+ Cache<String, Boolean> similarityCache = CacheBuilder.newBuilder()
|
|
|
+ .expireAfterWrite(60 * 15, TimeUnit.SECONDS).build();
|
|
|
|
|
|
@Override
|
|
|
public FilterResult filter(FilterParam param) {
|
|
|
@@ -65,10 +66,19 @@ public class SensitiveStrategy implements FilterStrategy {
|
|
|
}
|
|
|
}
|
|
|
List<Set<Character>> unsafeTitleCache = TitleSimilarCheckUtil.makeCache(allUnSafeTitles);
|
|
|
+ List<String> allTitles = param.getContents().stream().map(Content::getTitle).collect(Collectors.toList());
|
|
|
+ Map<String, Boolean> similarityMap = new HashMap<>(similarityCache.getAllPresent(allTitles));
|
|
|
for (Content content : param.getContents()) {
|
|
|
try {
|
|
|
- if (TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), unsafeTitleCache,
|
|
|
- TitleSimilarCheckUtil.SIMILARITY_THRESHOLD)) {
|
|
|
+ Boolean isSimilar;
|
|
|
+ if (similarityMap.containsKey(content.getTitle())) {
|
|
|
+ isSimilar = similarityMap.get(content.getTitle());
|
|
|
+ } else {
|
|
|
+ isSimilar = TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), unsafeTitleCache,
|
|
|
+ TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
|
|
|
+ similarityCache.put(content.getTitle(), isSimilar);
|
|
|
+ }
|
|
|
+ if (isSimilar) {
|
|
|
content.setFilterReason("安全违规");
|
|
|
filterContents.add(content);
|
|
|
} else {
|
