标题删除文章，标题加入黑名单

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleUnsafeTitle.java

@@ -0,0 +1,29 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "article_unsafe_title")
+public class ArticleUnsafeTitle {
+    @Id
+    @Column(name = "title_md5")
+    private String titleMd5;
+    @Column(name = "title")
+    private String title;
+    @Column(name = "status")
+    private Integer status;
+    @Column(name = "create_timestamp")
+    private Long createTimestamp;
+    @Column(name = "finish_timestamp")
+    private Long finishTimestamp;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/ArticleUnsafeTitleRepository.java

@@ -0,0 +1,15 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleUnsafeTitle;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+@Repository
+public interface ArticleUnsafeTitleRepository extends JpaRepository<ArticleUnsafeTitle, String> {
+
+    ArticleUnsafeTitle getByTitleMd5(String titleMd5);
+
+    List<ArticleUnsafeTitle> getByStatus(Integer status);
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleAuditService.java

@@ -62,6 +62,8 @@ public class ArticleAuditService {
     private PublishAccountRepository publishAccountRepository;
     @Autowired
     private ArticleRepository articleRepository;
+    @Autowired
+    private ArticleUnsafeTitleRepository articleUnsafeTitleRepository;
 
 
     @XxlJob("articleVideoAudit")
@@ -251,5 +253,16 @@ public class ArticleAuditService {
         // 根据已发布文章查找推送id 并删除
         List<String> publishContentIds = publishContents.stream().map(PublishContentDTO::getId).collect(Collectors.toList());
         buildArticleAuditDelete(publishContentIds);
+        buildArticleUnsafeTitle(titleMd5, param.getTitle());
+    }
+
+    private void buildArticleUnsafeTitle(String titleMd5, String title) {
+        ArticleUnsafeTitle unsafeTitle = new ArticleUnsafeTitle();
+        unsafeTitle.setTitle(title);
+        unsafeTitle.setTitleMd5(titleMd5);
+        unsafeTitle.setStatus(StatusEnum.ONE.getCode());
+        unsafeTitle.setCreateTimestamp(System.currentTimeMillis());
+        articleUnsafeTitleRepository.save(unsafeTitle);
     }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/SensitiveStrategy.java

@@ -1,26 +1,27 @@
 package com.tzld.longarticle.recommend.server.service.recommend.filter.strategy;
 
-import cn.hutool.core.collection.CollectionUtil;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
+import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
+import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleSensitive;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleUnsafeTitle;
 import com.tzld.longarticle.recommend.server.remote.ArticleSensitiveRemoteService;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleSensitiveRepository;
-import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleSensitive;
+import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleUnsafeTitleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterStrategy;
-import com.tzld.longarticle.recommend.server.util.Md5Util;
 import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import java.util.*;
-import java.util.concurrent.CountDownLatch;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.stream.Collectors;
 
 @Component
 @Slf4j
@@ -30,6 +31,8 @@ public class SensitiveStrategy implements FilterStrategy {
     private ArticleSensitiveRemoteService articleSensitiveRemoteService;
     @Autowired
     private ArticleSensitiveRepository articleSensitiveRepository;
+    @Autowired
+    private ArticleUnsafeTitleRepository articleUnsafeTitleRepository;
 
     @ApolloJsonValue("${UnSafeTitles:[]}")
     private static List<String> UnSafeTitles;
@@ -54,8 +57,14 @@ public class SensitiveStrategy implements FilterStrategy {
 //        if (CollectionUtil.isNotEmpty(articleSensitiveList)) {
 //            articleSensitiveMap = articleSensitiveList.stream().collect(Collectors.toMap(ArticleSensitive::getMd5, o -> o));
 //        }
-
-        List<Set<Character>> unsafeTitleCache = TitleSimilarCheckUtil.makeCache(UnSafeTitles);
+        List<String> allUnSafeTitles = UnSafeTitles;
+        List<ArticleUnsafeTitle> articleUnsafeTitleList = articleUnsafeTitleRepository.getByStatus(StatusEnum.ONE.getCode());
+        if (CollectionUtils.isNotEmpty(articleUnsafeTitleList)) {
+            for (ArticleUnsafeTitle articleUnsafeTitle : articleUnsafeTitleList) {
+                allUnSafeTitles.add(articleUnsafeTitle.getTitle());
+            }
+        }
+        List<Set<Character>> unsafeTitleCache = TitleSimilarCheckUtil.makeCache(allUnSafeTitles);
         for (Content content : param.getContents()) {
             try {
                 if (TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), unsafeTitleCache,