code

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/crawler/CrawlerBaseMapper.java

@@ -23,7 +23,7 @@ public interface CrawlerBaseMapper {
 
     Integer countLongArticlesVideos();
 
-    List<LongArticlesVideo> pageLongArticlesVideos(int offset, int pageSize);
+    List<LongArticlesVideo> pageLongArticlesVideos(long id, int pageSize);
 
     List<LongArticlesText> getLongArticlesText();
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/DataFlushService.java

@@ -134,11 +134,8 @@ public class DataFlushService {
         log.info("flushLongArticlesCrawlerVideos flushNum:{}", flushNum);
     }
 
-    public void flushLongArticlesVideos(Integer pageNum) {
+    public void flushLongArticlesVideos(Integer pageNum, Long id) {
         int pageSize = 1000;
-        if (pageNum == null) {
-            pageNum = 1;
-        }
         List<LongArticlesText> kimiTitleList = crawlerBaseMapper.getLongArticlesText();
         Map<String, LongArticlesText> kimiTitleMap = kimiTitleList.stream().collect(
                 Collectors.toMap(LongArticlesText::getContentId, o -> o, (existing, replacement) -> replacement));
@@ -146,9 +143,12 @@ public class DataFlushService {
         int totalPage = count / pageSize + 1;
         int longArticlesTextFlushNum = 0;
         int longArticlesMatchVideosFlushNum = 0;
-        while (pageNum <= totalPage) {
-            int offset = (pageNum - 1) * pageSize;
-            List<LongArticlesVideo> list = crawlerBaseMapper.pageLongArticlesVideos(offset, pageSize);
+        while (true) {
+            List<LongArticlesVideo> list = crawlerBaseMapper.pageLongArticlesVideos(id, pageSize);
+            if (CollectionUtils.isEmpty(list)) {
+                break;
+            }
+            id = list.stream().mapToLong(LongArticlesVideo::getId).max().getAsLong();
             List<LongArticlesText> batchSaveLongArticlesTextList = new ArrayList<>();
             List<LongArticlesMatchVideos> batchSaveLongArticlesMatchVideosList = new ArrayList<>();
             Set<String> existsIdSet = new HashSet<>();
@@ -184,7 +184,7 @@ public class DataFlushService {
                 longArticlesMatchVideos.setTraceId(longArticlesVideo.getTraceId());
                 longArticlesMatchVideos.setGhId(longArticlesVideo.getGhId());
                 longArticlesMatchVideos.setAccountName(longArticlesVideo.getAccountName());
-                longArticlesMatchVideos.setContentStatus(longArticlesVideo.getContentStatus());
+                longArticlesMatchVideos.setContentStatus(4);
                 longArticlesMatchVideos.setSuccessStatus(longArticlesVideo.getSuccess());
                 longArticlesMatchVideos.setRequestTimestamp(longArticlesVideo.getRequestTimeStamp());
                 longArticlesMatchVideos.setUpdateTime(longArticlesVideo.getUpdateTime());
@@ -217,7 +217,8 @@ public class DataFlushService {
                     longArticlesMatchVideosFlushNum += batchSaveLongArticlesMatchVideosList.size();
                 }
             }
-            log.info("flushLongArticlesVideos pageNum:{} totalPage:{}", pageNum, totalPage);
+            log.info("flushLongArticlesVideos pageNum:{} totalPage:{} id:{} longArticlesMatchVideosFlushNum:{}",
+                    pageNum, totalPage, id, longArticlesMatchVideosFlushNum);
             pageNum++;
         }
         log.info("flushLongArticlesVideos longArticlesTextFlushNum:{} longArticlesMatchVideosFlushNum:{}",

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/DataFlushController.java

@@ -34,9 +34,9 @@ public class DataFlushController {
         }).start();
     }
     @GetMapping("/flush/long_articles_video")
-    public void flushLongArticlesVideos(Integer pageNum) {
+    public void flushLongArticlesVideos(Integer pageNum, Long id) {
         new Thread(() -> {
-            service.flushLongArticlesVideos(pageNum);
+            service.flushLongArticlesVideos(pageNum, id);
         }).start();
     }
     @GetMapping("/flush/long_articles_text")

long-article-recommend-service/src/main/resources/mapper/crawler/CrawlerBaseMapper.xml

@@ -29,11 +29,11 @@
         select * from article_match_videos order by video_id
     </select>
     <select id="countLongArticlesVideos" resultType="java.lang.Integer">
-        select count(1) from long_articles_video where success = 1 and content_status = 2
+        select count(1) from long_articles_video where content_status = 2 and success = 1
     </select>
     <select id="pageLongArticlesVideos"
             resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideo">
-        select * from long_articles_video where success = 1 and content_status = 2 order by id limit #{offset}, #{pageSize}
+        select * from long_articles_video where content_status = 2 and success = 1 and id > #{id} order by id limit #{pageSize}
     </select>
     <select id="getLongArticlesText" resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesText">
         select content_id, kimi_title, kimi_summary, kimi_keys from long_articles_video where kimi_summary is not null group by content_id