去重过滤

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/entity/crawler/PublishSortLog.java

@@ -34,6 +34,10 @@ public class PublishSortLog {
     private String category;
     @Column(name = "strategy")
     private String strategy;
+    @Column(name = "score")
+    private String score;
+    @Column(name = "score_map")
+    private String scoreMap;
     @Column(name = "create_timestamp")
     private Long createTimestamp;
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/DataDashboardService.java

@@ -162,7 +162,10 @@ public class DataDashboardService {
                 }
             }
             Map<String, AccountAvgInfo> accountAvgInfoMap = accountAvgInfoIndexMap.get(article.getGhId());
-            AccountAvgInfo avgInfo = accountAvgInfoMap.get(article.getItemIndex().toString());
+            AccountAvgInfo avgInfo = null;
+            if (Objects.nonNull(accountAvgInfoMap)) {
+                avgInfo = accountAvgInfoMap.get(article.getItemIndex().toString());
+            }
             String date = DateUtils.timestampToYMDStr(article.getUpdateTime(), "yyyyMMdd");
             NewSortStrategyExport obj = new NewSortStrategyExport();
             obj.setGhId(article.getGhId());

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/RecommendService.java

@@ -259,6 +259,8 @@ public class RecommendService {
             sortLog.setIndexAvgCount(accountIndexAvgViewCountService.getAvgReadCountByDB(avgInfoList, param.getGhId(), i));
             sortLog.setCategory(JSONObject.toJSONString(content.getCategory()));
             sortLog.setStrategy(param.getStrategy());
+            sortLog.setScore(String.valueOf(content.getScore()));
+            sortLog.setScoreMap(JSONObject.toJSONString(content.getScoreMap()));
             sortLog.setCreateTimestamp(System.currentTimeMillis());
             publishSortLogSaveList.add(sortLog);
         }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/FilterService.java

@@ -32,6 +32,7 @@ public class FilterService {
     }
 
     private FilterResult contentFilter(FilterParam param) {
+        long start = System.currentTimeMillis();
         FilterResult result = new FilterResult();
 
         List<FilterStrategy> strategies = getStrategies(param);
@@ -84,21 +85,21 @@ public class FilterService {
         }
         result.setContentIds(contentIds);
         result.setFilterContent(filterContents);
+        log.info("filter cost:{}", System.currentTimeMillis() - start);
         return result;
     }
 
     private List<FilterStrategy> getStrategies(FilterParam param) {
         List<FilterStrategy> strategies = new ArrayList<>();
+        strategies.add(ServiceBeanFactory.getBean(BadStrategy.class));
+        strategies.add(ServiceBeanFactory.getBean(SensitiveStrategy.class));
+        strategies.add(ServiceBeanFactory.getBean(DeDuplicationStrategy.class));
         switch (param.getScene()) {
             case FWH_COLD_START:
                 strategies.add(ServiceBeanFactory.getBean(HistoryTitleForFwhColdStartStrategy.class));
-                strategies.add(ServiceBeanFactory.getBean(BadStrategy.class));
-                strategies.add(ServiceBeanFactory.getBean(SensitiveStrategy.class));
                 break;
             default:
                 strategies.add(ServiceBeanFactory.getBean(HistoryTitleStrategy.class));
-                strategies.add(ServiceBeanFactory.getBean(BadStrategy.class));
-                strategies.add(ServiceBeanFactory.getBean(SensitiveStrategy.class));
                 strategies.add(ServiceBeanFactory.getBean(LowScoreStrategy.class));
                 break;
         }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/strategy/DeDuplicationStrategy.java

@@ -0,0 +1,133 @@
+package com.tzld.longarticle.recommend.server.service.filter.strategy;
+
+import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
+import com.tzld.longarticle.recommend.server.common.enums.ContentPoolEnum;
+import com.tzld.longarticle.recommend.server.model.Content;
+import com.tzld.longarticle.recommend.server.service.filter.FilterParam;
+import com.tzld.longarticle.recommend.server.service.filter.FilterResult;
+import com.tzld.longarticle.recommend.server.service.filter.FilterStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class DeDuplicationStrategy implements FilterStrategy {
+
+    private final ExecutorService pool = ThreadPoolFactory.deDuplicatePool();
+
+    @Override
+    public FilterResult filter(FilterParam param) {
+        long start = System.currentTimeMillis();
+        FilterResult filterResult = new FilterResult();
+        List<String> result;
+        List<Content> filterContents = new CopyOnWriteArrayList<>();
+        // 先对内容池内部去重
+        List<Content> middleContent = innerDeduplication(param.getContents(), filterContents);
+        // 内容池间进行去重
+        result = groupDeduplication(middleContent, filterContents);
+
+        filterResult.setContentIds(result);
+        filterResult.setFilterContent(filterContents);
+        log.info("DeDuplicationStrategy cost:{}", System.currentTimeMillis() - start);
+        return filterResult;
+    }
+
+    private List<Content> innerDeduplication(List<Content> contentList, List<Content> filterContents) {
+        List<Content> result = new ArrayList<>();
+        Map<String, List<Content>> contentMap = contentList.stream().collect(Collectors.groupingBy(Content::getContentPoolType));
+        List<Future<List<Content>>> futures = new ArrayList<>();
+        List<String> contentPoolList = ContentPoolEnum.getOrderContentPool();
+        CountDownLatch cdl = new CountDownLatch(contentPoolList.size());
+        for (String contentPool : contentPoolList) {
+            List<Content> contents = contentMap.get(contentPool);
+            Future<List<Content>> future = pool.submit(() -> {
+                try {
+                    List<Content> res = new ArrayList<>();
+                    List<String> titles = new ArrayList<>();
+                    if (CollectionUtils.isEmpty(contents)) {
+                        return new ArrayList<>();
+                    }
+                    for (Content c : contents) {
+                        if (titles.contains(c.getTitle())) {
+                            c.setFilterReason("重复文章");
+                            filterContents.add(c);
+                        } else {
+                            res.add(c);
+                            titles.add(c.getTitle());
+                        }
+                    }
+                    return res;
+                } finally {
+                    cdl.countDown();
+                }
+            });
+            futures.add(future);
+        }
+        try {
+            cdl.await();
+        } catch (InterruptedException e) {
+            log.error("filter error", e);
+            return null;
+        }
+        for (Future<List<Content>> f : futures) {
+            try {
+                List<Content> res = f.get();
+                result.addAll(res);
+            } catch (Exception e) {
+                log.error("future get error ", e);
+            }
+        }
+        return result;
+    }
+
+    private List<String> groupDeduplication(List<Content> contentList, List<Content> filterContents) {
+        List<String> result = new CopyOnWriteArrayList<>();
+        List<String> titles = new CopyOnWriteArrayList<>();
+        Map<String, List<Content>> contentMap = contentList.stream().collect(Collectors.groupingBy(Content::getContentPoolType));
+
+        List<String> contentPoolList = ContentPoolEnum.getOrderContentPool();
+        for (String contentPool : contentPoolList) {
+            List<Content> contents = contentMap.get(contentPool);
+            if (CollectionUtils.isEmpty(titles) && CollectionUtils.isNotEmpty(contents)) {
+                result.addAll(contents.stream().map(Content::getId).collect(Collectors.toList()));
+                titles.addAll(contents.stream().map(Content::getTitle).collect(Collectors.toList()));
+            }
+            if (CollectionUtils.isEmpty(contents)) {
+                continue;
+            }
+            CountDownLatch cdl = new CountDownLatch(contents.size());
+            for (Content content : contents) {
+                pool.submit(() -> {
+                    try {
+                        if (titles.contains(content.getTitle())) {
+                            content.setFilterReason("重复文章");
+                            filterContents.add(content);
+                        } else {
+                            result.add(content.getId());
+                            titles.add(content.getTitle());
+                        }
+                    } finally {
+                        cdl.countDown();
+                    }
+                });
+            }
+            try {
+                cdl.await();
+            } catch (InterruptedException e) {
+                log.error("filter error", e);
+                return null;
+            }
+        }
+        return result;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/strategy/HistoryTitleStrategy.java

@@ -22,7 +22,6 @@ import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 /**
@@ -56,16 +55,15 @@ public class HistoryTitleStrategy implements FilterStrategy {
         List<String> firstSecondContentPool = new ArrayList<>();
         String[] contentPoolConfig = accountContentPoolConfigService.getContentPools(param.getAccountName());
         if (Objects.nonNull(contentPoolConfig)) {
-            firstSecondContentPool = Arrays.asList(contentPoolConfig[0], contentPoolConfig[1]);
+            firstSecondContentPool.addAll(Arrays.asList(contentPoolConfig[0], contentPoolConfig[1]));
         }
         List<Future<Content>> futures = new ArrayList<>();
         CountDownLatch cdl = new CountDownLatch(param.getContents().size());
         for (Content content : param.getContents()) {
-            List<String> finalFirstSecondContentPool = firstSecondContentPool;
             Future<Content> future = pool.submit(() -> {
                 try {
                     boolean isDuplicate;
-                    if (CollectionUtils.isNotEmpty(finalFirstSecondContentPool) && finalFirstSecondContentPool.contains(content.getContentPoolType())) {
+                    if (CollectionUtils.isNotEmpty(firstSecondContentPool) && firstSecondContentPool.contains(content.getContentPoolType())) {
                         // 四个内容池 配置 判断头条，次头条
                         isDuplicate = TitleSimilarCheckUtil.isDuplicateContent(content.getTitle(), firstSecondTitleList);
                     } else {
@@ -82,7 +80,7 @@ public class HistoryTitleStrategy implements FilterStrategy {
             futures.add(future);
         }
         try {
-            cdl.await(5000, TimeUnit.MILLISECONDS);
+            cdl.await();
         } catch (InterruptedException e) {
             log.error("filter error", e);
             return null;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/RankStrategy.java

@@ -1,6 +1,13 @@
 package com.tzld.longarticle.recommend.server.service.rank;
 
 
+import com.tzld.longarticle.recommend.server.model.Content;
+import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
 /**
  * @author dyp
  */
@@ -8,4 +15,39 @@ public interface RankStrategy {
 
     RankResult rank(RankParam param);
 
+    public static void deduplication(List<Content> result, Map<String, List<Content>> contentMap, String[] publishPool) {
+        List<String> titles = new ArrayList<>();
+        for (int i = 0; i < result.size(); i++) {
+            if (TitleSimilarCheckUtil.isDuplicateContent(result.get(i).getTitle(), titles)) {
+                if (i == 0) {
+                    for (Content content : contentMap.get(publishPool[0])) {
+                        if (!TitleSimilarCheckUtil.isDuplicateContent(content.getTitle(), titles)) {
+                            result.set(i, content);
+                            titles.add(content.getTitle());
+                            break;
+                        }
+                    }
+                } else if (i == 1) {
+                    for (Content content : contentMap.get(publishPool[1])) {
+                        if (!TitleSimilarCheckUtil.isDuplicateContent(content.getTitle(), titles)) {
+                            result.set(i, content);
+                            titles.add(content.getTitle());
+                            break;
+                        }
+                    }
+                } else {
+                    for (Content content : contentMap.get(publishPool[2])) {
+                        if (!TitleSimilarCheckUtil.isDuplicateContent(content.getTitle(), titles)) {
+                            result.set(i, content);
+                            titles.add(content.getTitle());
+                            break;
+                        }
+                    }
+                }
+            } else {
+                titles.add(result.get(i).getTitle());
+            }
+        }
+    }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/strategy/DefaultRankStrategy.java

@@ -86,8 +86,9 @@ public class DefaultRankStrategy implements RankStrategy {
         List<Content> contents = CommonCollectionUtils.toList(sortedItems, RankItem::getContent);
 
         // 3 相似去重
-        contents = deduplication(contents);
+//        contents = deduplication(contents);
         log.info("Deduplication {}", JSONUtils.toJson(contents));
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
 
         // 4 文章按照内容池分组
         Map<String, List<Content>> contentMap = new HashMap<>();
@@ -110,6 +111,7 @@ public class DefaultRankStrategy implements RankStrategy {
         if (CollectionUtils.isNotEmpty(pool)) {
             result.add(pool.get(0));
             if (result.size() == 1 && pool.size() > 1) {
+                publishPool[0] = contentPools[1];
                 result.add(pool.get(1));
             }
         }
@@ -120,6 +122,8 @@ public class DefaultRankStrategy implements RankStrategy {
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
         }
 
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
         return new RankResult(result);
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/strategy/FwhColdStartRankStrategy.java

@@ -69,7 +69,7 @@ public class FwhColdStartRankStrategy implements RankStrategy {
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
         // 2 相似去重
         List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
-        contents = deduplication(contents);
+//        contents = deduplication(contents);
 
         // 3 选文章
         List<Content> result = new ArrayList<>();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/strategy/RankV10Strategy.java

@@ -75,7 +75,7 @@ public class RankV10Strategy implements RankStrategy {
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
         // 2 相似去重
         List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
-        contents = deduplication(contents);
+//        contents = deduplication(contents);
 
         // 3 文章按照内容池分组
         Map<String, List<Content>> contentMap = new HashMap<>();
@@ -85,6 +85,7 @@ public class RankV10Strategy implements RankStrategy {
         }
         // 4 选文章
         List<Content> result = new ArrayList<>();
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
@@ -96,6 +97,7 @@ public class RankV10Strategy implements RankStrategy {
             if (Objects.nonNull(replacePoolConfig)) {
                 List<Content> pool1Replace = contentMap.get(replacePoolConfig.getContentPool());
                 if (CollectionUtils.isNotEmpty(pool1Replace)) {
+                    publishPool[0] = replacePoolConfig.getContentPool();
                     result.add(pool1Replace.get(0));
                 }
             }
@@ -111,6 +113,7 @@ public class RankV10Strategy implements RankStrategy {
                 while (i == j && pool2.size() > 1) {
                     j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
                     if (i != j) {
+                        publishPool[0] = contentPools[1];
                         result.add(pool2.get(1));
                         break;
                     }
@@ -122,6 +125,7 @@ public class RankV10Strategy implements RankStrategy {
             if (Objects.nonNull(replacePoolConfig)) {
                 List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
                 if (CollectionUtils.isNotEmpty(pool2Replace)) {
+                    publishPool[1] = replacePoolConfig.getContentPool();
                     result.add(pool2Replace.get(0));
                 }
             }
@@ -129,10 +133,12 @@ public class RankV10Strategy implements RankStrategy {
 
         // 3-8
         List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool)) {
+        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
         }
 
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
         return new RankResult(result);
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/strategy/RankV3Strategy.java

@@ -75,7 +75,7 @@ public class RankV3Strategy implements RankStrategy {
         log.info("SortResult {}", JSONUtils.toJson(items));
         // 2 相似去重
         List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
-        contents = deduplication(contents);
+//        contents = deduplication(contents);
         log.info("Deduplication {}", JSONUtils.toJson(contents));
 
         // 3 文章按照内容池分组
@@ -88,7 +88,7 @@ public class RankV3Strategy implements RankStrategy {
         // 4 选文章
         List<Content> result = new ArrayList<>();
         String[] contentPools = accountContentPoolConfigService.getContentPools(param.getAccountName());
-
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
 
         // 头、次
         if (StringUtils.equals(contentPools[0], contentPools[1])) {
@@ -134,6 +134,7 @@ public class RankV3Strategy implements RankStrategy {
                 }
             } else if (CollectionUtils.isNotEmpty(pool2)) {
                 result.add(pool2.get(0));
+                publishPool[0] = contentPools[1];
                 if (pool2.size() > 1) {
                     result.add(pool2.get(1));
                 }
@@ -146,6 +147,8 @@ public class RankV3Strategy implements RankStrategy {
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
         }
 
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
         return new RankResult(result);
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/strategy/RankV4Strategy.java

@@ -72,7 +72,7 @@ public class RankV4Strategy implements RankStrategy {
         log.info("SortResult {}", JSONUtils.toJson(items));
         // 2 相似去重
         List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
-        contents = deduplication(contents);
+//        contents = deduplication(contents);
         log.info("Deduplication {}", JSONUtils.toJson(contents));
 
         // 3 文章按照内容池分组
@@ -85,7 +85,7 @@ public class RankV4Strategy implements RankStrategy {
         // 4 选文章
         List<Content> result = new ArrayList<>();
         String[] contentPools = accountContentPoolConfigService.getContentPools(param.getAccountName());
-
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
 
         // 头、次
         if (StringUtils.equals(contentPools[0], contentPools[1])) {
@@ -131,6 +131,7 @@ public class RankV4Strategy implements RankStrategy {
                 }
             } else if (CollectionUtils.isNotEmpty(pool2)) {
                 result.add(pool2.get(0));
+                publishPool[0] = contentPools[1];
                 if (pool2.size() > 1) {
                     result.add(pool2.get(1));
                 }
@@ -143,6 +144,8 @@ public class RankV4Strategy implements RankStrategy {
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
         }
 
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
         return new RankResult(result);
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/strategy/RankV5Strategy.java

@@ -74,7 +74,7 @@ public class RankV5Strategy implements RankStrategy {
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
         // 2 相似去重
         List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
-        contents = deduplication(contents);
+//        contents = deduplication(contents);
 
         // 3 文章按照内容池分组
         Map<String, List<Content>> contentMap = new HashMap<>();
@@ -84,6 +84,7 @@ public class RankV5Strategy implements RankStrategy {
         }
         // 4 选文章
         List<Content> result = new ArrayList<>();
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
@@ -95,6 +96,7 @@ public class RankV5Strategy implements RankStrategy {
             if (Objects.nonNull(replacePoolConfig)) {
                 List<Content> pool1Replace = contentMap.get(replacePoolConfig.getContentPool());
                 if (CollectionUtils.isNotEmpty(pool1Replace)) {
+                    publishPool[0] = replacePoolConfig.getContentPool();
                     result.add(pool1Replace.get(0));
                 }
             }
@@ -110,6 +112,7 @@ public class RankV5Strategy implements RankStrategy {
                 while (i == j && pool2.size() > 1) {
                     j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
                     if (i != j) {
+                        publishPool[0] = contentPools[1];
                         result.add(pool2.get(1));
                         break;
                     }
@@ -121,6 +124,7 @@ public class RankV5Strategy implements RankStrategy {
             if (Objects.nonNull(replacePoolConfig)) {
                 List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
                 if (CollectionUtils.isNotEmpty(pool2Replace)) {
+                    publishPool[1] = replacePoolConfig.getContentPool();
                     result.add(pool2Replace.get(0));
                 }
             }
@@ -132,6 +136,8 @@ public class RankV5Strategy implements RankStrategy {
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
         }
 
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
         return new RankResult(result);
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/strategy/RankV7Strategy.java

@@ -75,7 +75,7 @@ public class RankV7Strategy implements RankStrategy {
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
         // 2 相似去重
         List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
-        contents = deduplication(contents);
+//        contents = deduplication(contents);
 
         // 3 文章按照内容池分组
         Map<String, List<Content>> contentMap = new HashMap<>();
@@ -85,6 +85,7 @@ public class RankV7Strategy implements RankStrategy {
         }
         // 4 选文章
         List<Content> result = new ArrayList<>();
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
@@ -96,6 +97,7 @@ public class RankV7Strategy implements RankStrategy {
             if (Objects.nonNull(replacePoolConfig)) {
                 List<Content> pool1Replace = contentMap.get(replacePoolConfig.getContentPool());
                 if (CollectionUtils.isNotEmpty(pool1Replace)) {
+                    publishPool[0] = replacePoolConfig.getContentPool();
                     result.add(pool1Replace.get(0));
                 }
             }
@@ -111,6 +113,7 @@ public class RankV7Strategy implements RankStrategy {
                 while (i == j && pool2.size() > 1) {
                     j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
                     if (i != j) {
+                        publishPool[0] = contentPools[1];
                         result.add(pool2.get(1));
                         break;
                     }
@@ -122,6 +125,7 @@ public class RankV7Strategy implements RankStrategy {
             if (Objects.nonNull(replacePoolConfig)) {
                 List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
                 if (CollectionUtils.isNotEmpty(pool2Replace)) {
+                    publishPool[1] = replacePoolConfig.getContentPool();
                     result.add(pool2Replace.get(0));
                 }
             }
@@ -129,10 +133,12 @@ public class RankV7Strategy implements RankStrategy {
 
         // 3-8
         List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool)) {
+        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
         }
 
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
         return new RankResult(result);
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/strategy/RankV8Strategy.java

@@ -75,7 +75,7 @@ public class RankV8Strategy implements RankStrategy {
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
         // 2 相似去重
         List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
-        contents = deduplication(contents);
+//        contents = deduplication(contents);
 
         // 3 文章按照内容池分组
         Map<String, List<Content>> contentMap = new HashMap<>();
@@ -85,6 +85,7 @@ public class RankV8Strategy implements RankStrategy {
         }
         // 4 选文章
         List<Content> result = new ArrayList<>();
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
@@ -96,6 +97,7 @@ public class RankV8Strategy implements RankStrategy {
             if (Objects.nonNull(replacePoolConfig)) {
                 List<Content> pool1Replace = contentMap.get(replacePoolConfig.getContentPool());
                 if (CollectionUtils.isNotEmpty(pool1Replace)) {
+                    publishPool[0] = replacePoolConfig.getContentPool();
                     result.add(pool1Replace.get(0));
                 }
             }
@@ -111,6 +113,7 @@ public class RankV8Strategy implements RankStrategy {
                 while (i == j && pool2.size() > 1) {
                     j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
                     if (i != j) {
+                        publishPool[0] = contentPools[1];
                         result.add(pool2.get(1));
                         break;
                     }
@@ -122,6 +125,7 @@ public class RankV8Strategy implements RankStrategy {
             if (Objects.nonNull(replacePoolConfig)) {
                 List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
                 if (CollectionUtils.isNotEmpty(pool2Replace)) {
+                    publishPool[1] = replacePoolConfig.getContentPool();
                     result.add(pool2Replace.get(0));
                 }
             }
@@ -129,10 +133,12 @@ public class RankV8Strategy implements RankStrategy {
 
         // 3-8
         List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool)) {
+        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
         }
 
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
         return new RankResult(result);
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/strategy/RankV9Strategy.java

@@ -75,7 +75,7 @@ public class RankV9Strategy implements RankStrategy {
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
         // 2 相似去重
         List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
-        contents = deduplication(contents);
+//        contents = deduplication(contents);
 
         // 3 文章按照内容池分组
         Map<String, List<Content>> contentMap = new HashMap<>();
@@ -85,6 +85,7 @@ public class RankV9Strategy implements RankStrategy {
         }
         // 4 选文章
         List<Content> result = new ArrayList<>();
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
@@ -96,6 +97,7 @@ public class RankV9Strategy implements RankStrategy {
             if (Objects.nonNull(replacePoolConfig)) {
                 List<Content> pool1Replace = contentMap.get(replacePoolConfig.getContentPool());
                 if (CollectionUtils.isNotEmpty(pool1Replace)) {
+                    publishPool[0] = replacePoolConfig.getContentPool();
                     result.add(pool1Replace.get(0));
                 }
             }
@@ -111,6 +113,7 @@ public class RankV9Strategy implements RankStrategy {
                 while (i == j && pool2.size() > 1) {
                     j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
                     if (i != j) {
+                        publishPool[0] = contentPools[1];
                         result.add(pool2.get(1));
                         break;
                     }
@@ -122,6 +125,7 @@ public class RankV9Strategy implements RankStrategy {
             if (Objects.nonNull(replacePoolConfig)) {
                 List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
                 if (CollectionUtils.isNotEmpty(pool2Replace)) {
+                    publishPool[1] = replacePoolConfig.getContentPool();
                     result.add(pool2Replace.get(0));
                 }
             }
@@ -129,10 +133,12 @@ public class RankV9Strategy implements RankStrategy {
 
         // 3-8
         List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool)) {
+        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
         }
 
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
         return new RankResult(result);
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recall/RecallService.java

@@ -296,7 +296,7 @@ public class RecallService implements ApplicationContextAware {
         Double t0FissionByFansSum = 0.0;
         Double t0FissionByReadAvgSum = 0.0;
         for (ContentHisPublishArticle article : content.getHisPublishArticleList()) {
-            if (article.getItemIndex() != 1
+            if (article.getItemIndex() != 1 || !article.isInnerAccount()
                     || CollectionUtils.isEmpty(article.getArticleDetailInfoList())) {
                 continue;
             }