TodayPublishStrategy title filter

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -99,6 +99,8 @@ public interface AigcBaseMapper {
 
     List<PublishContent> getTodayPublishFailContentList(String accountId, Long todayStart);
 
+    List<String> getTodayPublishContentTitles(List<String> contentIds);
+
     List<String> getAllAuditProducePlan();
 
     void topContentDeprecate(String topProducePlanId);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/TodayPublishStrategy.java

@@ -9,13 +9,13 @@ import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterPara
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterStrategy;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
+import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 import java.util.concurrent.*;
 import java.util.stream.Collectors;
 
@@ -50,6 +50,35 @@ public class TodayPublishStrategy implements FilterStrategy {
             List<String> todayPublishFailIdList = todayPublishFailList.stream().map(PublishContent::getId).collect(Collectors.toList());
             result.removeAll(todayPublishFailIdList);
         }
+        // 标题相似度过滤
+        if (CollectionUtils.isNotEmpty(result)) {
+            List<String> todayPublishContentIds = new ArrayList<>();
+            if (CollectionUtils.isNotEmpty(todayPublishContentList)) {
+                todayPublishContentList.stream().map(PublishContent::getId).forEach(todayPublishContentIds::add);
+            }
+            if (CollectionUtils.isNotEmpty(todayPublishFailList)) {
+                todayPublishFailList.stream().map(PublishContent::getId).forEach(todayPublishContentIds::add);
+            }
+            List<String> todayPublishTitles = aigcBaseMapper.getTodayPublishContentTitles(todayPublishContentIds);
+            if (CollectionUtils.isNotEmpty(todayPublishTitles)) {
+                Map<String, Content> contentMap = param.getContents().stream()
+                        .collect(Collectors.toMap(Content::getId, c -> c, (a, b) -> a));
+                List<Set<Character>> todayTitleCache = TitleSimilarCheckUtil.makeCache(todayPublishTitles);
+                List<String> titleFilterResult = new ArrayList<>();
+                List<Content> filteredContents = new ArrayList<>();
+                for (String contentId : result) {
+                    Content content = contentMap.get(contentId);
+                    if (content != null && content.getTitle() != null
+                            && TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), todayTitleCache, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD)) {
+                        filteredContents.add(content);
+                    } else {
+                        titleFilterResult.add(contentId);
+                    }
+                }
+                result = titleFilterResult;
+                filterResult.setFilterContent(filteredContents);
+            }
+        }
         filterResult.setContentIds(result);
         log.info("TodayPublishStrategy filter cost time:{}", System.currentTimeMillis() - start);
         return filterResult;

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -471,6 +471,17 @@
         select id from publish_content where publish_account_id = #{accountId} and status = 5 and update_timestamp > #{todayStart}
     </select>
 
+    <select id="getTodayPublishContentTitles" resultType="java.lang.String">
+        select distinct output.output
+        from publish_content pc
+        join produce_plan_exe_record pper on pc.source_id = pper.plan_exe_id
+        join produce_plan_module_output output on pper.plan_exe_id = output.plan_exe_id and output.produce_module_type = 3
+        where pc.id in
+        <foreach collection="contentIds" item="id" open="(" close=")" separator=",">
+            #{id}
+        </foreach>
+    </select>
+
     <select id="getAllAuditProducePlan" resultType="java.lang.String">
         select id from produce_plan where plan_status = 1 and produce_modal = 3
     </select>