数据迁移

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/DataFlushService.java

@@ -4,6 +4,7 @@ import cn.hutool.core.collection.CollectionUtil;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
+import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.model.dto.*;
@@ -15,6 +16,7 @@ import org.springframework.util.StringUtils;
 
 import java.net.URLDecoder;
 import java.util.*;
+import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 
 @Service
@@ -26,6 +28,8 @@ public class DataFlushService {
     @Autowired
     private CrawlerBaseMapper crawlerBaseMapper;
 
+    private final ExecutorService pool = ThreadPoolFactory.deDuplicatePool();
+
 
     public void flushGetOffVideos(Integer pageNum) {
         int pageSize = 1000;
@@ -239,10 +243,12 @@ public class DataFlushService {
         for (LongArticlesText update : updateList) {
             LongArticlesText kimi = kimiTitleMap.get(update.getContentId());
             if (Objects.nonNull(kimi)) {
-                update.setKimiTitle(kimi.getKimiTitle());
-                update.setKimiSummary(kimi.getKimiSummary());
-                update.setKimiKeys(kimi.getKimiKeys());
-                longArticleBaseMapper.updateLongArticlesText(update);
+                pool.submit(() -> {
+                    update.setKimiTitle(kimi.getKimiTitle());
+                    update.setKimiSummary(kimi.getKimiSummary());
+                    update.setKimiKeys(kimi.getKimiKeys());
+                    longArticleBaseMapper.updateLongArticlesText(update);
+                });
             }
         }
     }

long-article-recommend-service/src/main/resources/mapper/crawler/CrawlerBaseMapper.xml

@@ -16,10 +16,10 @@
         select * from get_off_videos where publish_time > 1727239202 order by video_id limit #{offset}, #{pageSize}
     </select>
     <select id="countLongArticlesRootSourceId" resultType="java.lang.Integer">
-        select count(1) from long_articles_root_source_id where rootSourceId > 'WeCom_ffddf52cc30880856d2408fa2b06ab64'
+        select count(1) from long_articles_root_source_id where requestTime > 1727192229
     </select>
     <select id="pageLongArticlesRootSourceId" resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesRootSourceId">
-        select * from long_articles_root_source_id where rootSourceId > 'WeCom_ffddf52cc30880856d2408fa2b06ab64' order by rootSourceId limit #{offset}, #{pageSize}
+        select * from long_articles_root_source_id where requestTime > 1727192229 order by rootSourceId limit #{offset}, #{pageSize}
     </select>
     <select id="countArticleMatchVideos" resultType="java.lang.Integer">
         select count(1) from article_match_videos where DATE(update_time) &gt;= '2024-09-20'
@@ -36,7 +36,7 @@
         select * from long_articles_video where DATE(update_time) &gt;= '2024-09-20' and success = 1 and content_status = 2 order by id limit #{offset}, #{pageSize}
     </select>
     <select id="getLongArticlesText" resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesText">
-        select content_id, kimi_summary, kimi_keys from long_articles_video where kimi_summary is not null group by content_id
+        select content_id, kimi_title, kimi_summary, kimi_keys from long_articles_video where kimi_summary is not null group by content_id
     </select>
 
 </mapper>

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -85,7 +85,7 @@
     </select>
     <select id="getNeedUpdateRecords"
             resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesText">
-        select * from long_articles_text_copy1 where kimi_summary is null
+        select * from long_articles_text_copy1 where kimi_title is null
     </select>
     <update id="updateLongArticlesText">
         update long_articles_text_copy1