需求视频内容池 按渠道去重保留分数高的一条

api-module/src/main/java/com/tzld/piaoquan/api/job/contentplatform/ContentPlatformDemandVideoJob.java

@@ -143,6 +143,16 @@ public class ContentPlatformDemandVideoJob {
                     .filter(v -> StringUtils.hasText(v.getTitle()))
                     .collect(Collectors.toList());
 
+            // 按crowd_segment粒度去重，相同videoId保留分数最高的一条
+            saveList = saveList.stream()
+                    .collect(Collectors.groupingBy(v -> v.getCrowdSegment() + "_" + v.getVideoId()))
+                    .values().stream()
+                    .map(group -> group.stream()
+                            .max(Comparator.comparingDouble(v -> v.getScore() != null ? v.getScore() : 0.0))
+                            .orElse(null))
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toList());
+
             // 先删除当天数据，再批量插入
             demandVideoMapperExt.deleteByDt(dt);
             for (List<ContentPlatformDemandVideo> partition : Lists.partition(saveList, 500)) {