更新long_articles_match_videos response

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/crawler/CrawlerBaseMapper.java

@@ -26,4 +26,6 @@ public interface CrawlerBaseMapper {
     List<LongArticlesVideo> pageLongArticlesVideos(long id, int pageSize);
 
     List<LongArticlesText> getLongArticlesText();
+
+    List<LongArticlesVideo> getLongArticlesVideo(List<String> traceIds);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -36,4 +36,10 @@ public interface LongArticleBaseMapper {
     List<LongArticlesMatchVideos> getLongArticlesMatchVideos(List<String> traceIds);
 
     List<LongArticlesCrawlerVideos> getLongArticlesCrawlerVideos(List<String> contentIds);
+
+    List<LongArticlesMatchVideos> getNeedMatchVideos(Long id, Integer pageSize);
+
+    void updateLongArticleMatchVideosResponse(LongArticlesMatchVideos longArticlesMatchVideos);
+
+    int countNeedMatchVideos(Long id);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/DataFlushService.java

@@ -4,6 +4,8 @@ import cn.hutool.core.collection.CollectionUtil;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.tzld.longarticle.recommend.server.common.CommonThreadPoolExecutor;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
@@ -16,7 +18,7 @@ import org.springframework.util.StringUtils;
 
 import java.net.URLDecoder;
 import java.util.*;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.*;
 import java.util.stream.Collectors;
 
 @Service
@@ -261,10 +263,14 @@ public class DataFlushService {
         String productionPath = fromJSON.getString("productionPath");
         String uid = getParamFromPath(productionPath, "su");
         String videoId = getParamFromPath(productionPath, "id");
+        String rootSourceId = getParamFromPath(productionPath, "rootSourceId");
         jsonObject.put("uid", uid);
         if (StringUtils.hasText(videoId)) {
             jsonObject.put("videoId", Long.valueOf(videoId));
         }
+        if (StringUtils.hasText(rootSourceId)) {
+            jsonObject.put("rootSourceId", rootSourceId);
+        }
         return jsonObject;
     }
 
@@ -303,4 +309,66 @@ public class DataFlushService {
         }
         log.info("flushLongArticlesText updateNum:{}", updateNum);
     }
+
+    private final static ExecutorService batchPool = new CommonThreadPoolExecutor(
+            5,
+            5,
+            0L, TimeUnit.SECONDS,
+            new LinkedBlockingQueue<>(10000),
+            new ThreadFactoryBuilder().setNameFormat("batch-%d").build(),
+            new ThreadPoolExecutor.AbortPolicy());
+
+    public void updateLongArticleMatchVideosResponse(Long id) {
+        int pageSize = 1000;
+        if (Objects.isNull(id)) {
+            id = 0L;
+        }
+        int count = longArticleBaseMapper.countNeedMatchVideos(id);
+        CountDownLatch cdl = new CountDownLatch((count / 1000) + 1);
+        while (true) {
+            List<LongArticlesMatchVideos> matchVideosList = longArticleBaseMapper.getNeedMatchVideos(id, pageSize);
+            if (CollectionUtil.isEmpty(matchVideosList)) {
+                break;
+            }
+            id = matchVideosList.stream().mapToLong(LongArticlesMatchVideos::getId).max().getAsLong();
+            Long finalId = id;
+            batchPool.submit(() -> {
+                try {
+                    long start = System.currentTimeMillis();
+                    List<String> traceIds = matchVideosList.stream().map(LongArticlesMatchVideos::getTraceId)
+                            .distinct().collect(Collectors.toList());
+                    List<LongArticlesVideo> longArticlesVideoList = crawlerBaseMapper.getLongArticlesVideo(traceIds);
+                    Map<String, LongArticlesVideo> longArticlesVideoMap = longArticlesVideoList.stream().collect(
+                            Collectors.toMap(LongArticlesVideo::getTraceId, o -> o, (existing, replacement) -> replacement));
+                    CountDownLatch countDownLatch = new CountDownLatch(matchVideosList.size());
+                    for (LongArticlesMatchVideos longArticlesMatchVideos : matchVideosList) {
+                        pool.submit(() -> {
+                            try {
+                                LongArticlesVideo longArticlesVideo = longArticlesVideoMap.get(longArticlesMatchVideos.getTraceId());
+                                if (Objects.nonNull(longArticlesVideo)) {
+                                    longArticlesMatchVideos.setResponse(getLongArticleVideoResponse(longArticlesVideo));
+                                    longArticleBaseMapper.updateLongArticleMatchVideosResponse(longArticlesMatchVideos);
+                                }
+                            } finally {
+                                countDownLatch.countDown();
+                            }
+                        });
+                    }
+                    try {
+                        countDownLatch.await();
+                    } catch (InterruptedException e) {
+                        log.error("updateLongArticleMatchVideosResponse InterruptedException", e);
+                    }
+                    log.info("updateLongArticleMatchVideosResponse end id:{}, cost:{}", finalId, System.currentTimeMillis() - start);
+                } finally {
+                    cdl.countDown();
+                }
+            });
+        }
+        try {
+            cdl.await();
+        } catch (InterruptedException e) {
+            log.error("updateLongArticleMatchVideosResponse InterruptedException", e);
+        }
+    }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/DataFlushController.java

@@ -45,6 +45,12 @@ public class DataFlushController {
             service.flushLongArticlesText();
         }).start();
     }
+    @GetMapping("/update/long_article_match_videos/response")
+    public void updateLongArticleMatchVideosResponse(Long id) {
+        new Thread(() -> {
+            service.updateLongArticleMatchVideosResponse(id);
+        }).start();
+    }
 
 
 }

long-article-recommend-service/src/main/resources/mapper/crawler/CrawlerBaseMapper.xml

@@ -39,4 +39,12 @@
         select content_id, kimi_title, kimi_summary, kimi_keys from long_articles_video where kimi_summary is not null group by content_id
     </select>
 
+    <select id="getLongArticlesVideo"
+            resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideo">
+        select * from long_articles_video where trace_id in
+        <foreach collection="traceIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+    </select>
+
 </mapper>

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -116,6 +116,12 @@
         where content_id = #{contentId}
     </update>
 
+    <update id="updateLongArticleMatchVideosResponse">
+        update long_articles_match_videos
+        set response = #{response}
+        where trace_id = #{traceId}
+    </update>
+
     <select id="getLongArticlesMatchVideos"
             resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesMatchVideos">
         select * from long_articles_match_videos where trace_id in
@@ -132,6 +138,14 @@
         and download_status = 2
     </select>
 
+    <select id="getNeedMatchVideos"
+            resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesMatchVideos">
+        select * from long_articles_match_videos where flow_pool_level is null and id > #{id} order by id limit #{pageSize}
+    </select>
+    <select id="countNeedMatchVideos" resultType="java.lang.Integer">
+        select count(1) from long_articles_match_videos where id > 3962393 and flow_pool_level is null
+    </select>
+
     <insert id="batchInsertLongArticlesMatchVideos" parameterType="list">
         INSERT INTO long_articles_match_videos (trace_id, content_id, flow_pool_level, gh_id, account_name,
                                                       content_status, success_status, request_timestamp, response,