syncContentPlatformDemandVideoJob 去重逻辑去除

api-module/src/main/java/com/tzld/piaoquan/api/job/contentplatform/ContentPlatformDemandVideoJob.java

@@ -218,19 +218,20 @@ public class ContentPlatformDemandVideoJob {
                 .filter(v -> StringUtils.hasText(v.getTitle()))
                 .collect(Collectors.toList());
 
-        // 按crowd_segment粒度去重，相同videoId保留分数最高的一条
-        saveList = saveList.stream()
-                .collect(Collectors.groupingBy(v -> v.getCrowdSegment() + "_" + v.getVideoId()))
-                .values().stream()
-                .map(group -> group.stream()
-                        .max(Comparator.comparingDouble(v -> v.getScore() != null ? v.getScore() : 0.0))
-                        .orElse(null))
-                .filter(Objects::nonNull)
-                .collect(Collectors.toList());
+//        // 按crowd_segment粒度去重，相同videoId保留分数最高的一条
+//        saveList = saveList.stream()
+//                .collect(Collectors.groupingBy(v -> v.getCrowdSegment() + "_"
+//                        + v.getChannelLevel3() + "_" + v.getDimension() + "_" + v.getVideoId()))
+//                .values().stream()
+//                .map(group -> group.stream()
+//                        .max(Comparator.comparingDouble(v -> v.getScore() != null ? v.getScore() : 0.0))
+//                        .orElse(null))
+//                .filter(Objects::nonNull)
+//                .collect(Collectors.toList());
 
         // 先删除当天该渠道数据，再批量插入
         demandVideoMapperExt.deleteByDtAndChannelName(dt, syncChannelName);
-        for (List<ContentPlatformDemandVideo> partition : Lists.partition(saveList, 500)) {
+        for (List<ContentPlatformDemandVideo> partition : Lists.partition(saveList, 1000)) {
             demandVideoMapperExt.batchInsert(partition);
         }