Merge branch 'wyp/1017-rootSrouceIdJob' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/LongArticlesRootSourceIdRepository.java

@@ -4,6 +4,10 @@ import com.tzld.longarticle.recommend.server.model.entity.crawler.LongArticlesRo
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
+import java.util.List;
+
 @Repository
 public interface LongArticlesRootSourceIdRepository extends JpaRepository<LongArticlesRootSourceId, String> {
+
+    List<LongArticlesRootSourceId> getByRootSourceIdIn(List<String> rootSourceIds);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/XxlJobService.java

@@ -152,7 +152,14 @@ public class XxlJobService {
                 }
             }
             if (CollectionUtil.isNotEmpty(saveList)) {
-                crawlerBaseMapper.batchInsertLongArticlesRootSourceId(saveList);
+                List<String> rootSourceIds = saveList.stream().map(LongArticlesRootSourceId::getRootSourceId).distinct().collect(Collectors.toList());
+                List<LongArticlesRootSourceId> existList = longArticlesRootSourceIdRepository.getByRootSourceIdIn(rootSourceIds);
+                for (LongArticlesRootSourceId existItem : existList) {
+                    saveList.removeIf(item -> item.getRootSourceId().equals(existItem.getRootSourceId()));
+                }
+                if (CollectionUtil.isNotEmpty(saveList)) {
+                    crawlerBaseMapper.batchInsertLongArticlesRootSourceId(saveList);
+                }
             }
         } catch (Exception e) {
             log.error("Error processCrawlerEachData: {}", JSONObject.toJSONString(longArticlesVideo), e);
@@ -203,7 +210,14 @@ public class XxlJobService {
                 saveList.add(saveItem);
             }
             if (CollectionUtil.isNotEmpty(saveList)) {
-                crawlerBaseMapper.batchInsertLongArticlesRootSourceId(saveList);
+                List<String> rootSourceIds = saveList.stream().map(LongArticlesRootSourceId::getRootSourceId).distinct().collect(Collectors.toList());
+                List<LongArticlesRootSourceId> existList = longArticlesRootSourceIdRepository.getByRootSourceIdIn(rootSourceIds);
+                for (LongArticlesRootSourceId existItem : existList) {
+                    saveList.removeIf(item -> item.getRootSourceId().equals(existItem.getRootSourceId()));
+                }
+                if (CollectionUtil.isNotEmpty(saveList)) {
+                    crawlerBaseMapper.batchInsertLongArticlesRootSourceId(saveList);
+                }
             }
         } catch (Exception e) {
             log.error("Error processArticleEachData: {}", JSONObject.toJSONString(longArticlesMatchVideo), e);