同批次相同内容过滤

core/src/main/java/com/tzld/supply/job/SpiderJob.java

@@ -62,6 +62,7 @@ public class SpiderJob {
         }
         List<SpiderContent> saveList = new ArrayList<>();
         Long now = System.currentTimeMillis();
+        List<String> titles = new ArrayList<>();
         for (ContentRankResponse.DataItem dataItem : dataItems) {
             if (dataItem.getRankList() == null || dataItem.getRankList().isEmpty()) {
                 continue;
@@ -74,9 +75,10 @@ public class SpiderJob {
                     continue;
                 }
                 // Filter 过滤已存在
-                if (checkContentExist(rankListItem)) {
+                if (checkContentExist(rankListItem) || titles.contains(rankListItem.getTitle())) {
                     continue;
                 }
+                titles.add(rankListItem.getTitle());
                 SpiderContent content = new SpiderContent();
                 content.setTaskId(task.getId());
                 content.setSource(dataItem.getSource());