merge

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/crawler/CrawlerBaseMapper.java

@@ -3,7 +3,7 @@ package com.tzld.longarticle.recommend.server.mapper.crawler;
 import com.tzld.longarticle.recommend.server.model.dto.ArticleMatchVideos;
 import com.tzld.longarticle.recommend.server.model.dto.GetOffVideos;
 import com.tzld.longarticle.recommend.server.model.dto.LongArticlesText;
-import com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideo;
+import com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideoDTO;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountCorrelation;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
@@ -41,9 +41,9 @@ public interface CrawlerBaseMapper {
 
     Integer countLongArticlesVideos();
 
-    List<LongArticlesVideo> pageLongArticlesVideos(long id, int pageSize);
+    List<LongArticlesVideoDTO> pageLongArticlesVideos(long id, int pageSize);
 
     List<LongArticlesText> getLongArticlesText();
 
-    List<LongArticlesVideo> getLongArticlesVideo(List<String> traceIds);
+    List<LongArticlesVideoDTO> getLongArticlesVideo(List<String> traceIds);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/LongArticlesVideo.java → long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/LongArticlesVideoDTO.java

@@ -5,7 +5,7 @@ import lombok.Data;
 import java.util.Date;
 
 @Data
-public class LongArticlesVideo {
+public class LongArticlesVideoDTO {
     private Long id;
     private String traceId;
     private String contentId;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/DataFlushService.java

@@ -147,52 +147,52 @@ public class DataFlushService {
         int longArticlesTextFlushNum = 0;
         int longArticlesMatchVideosFlushNum = 0;
         while (true) {
-            List<LongArticlesVideo> list = crawlerBaseMapper.pageLongArticlesVideos(id, pageSize);
+            List<LongArticlesVideoDTO> list = crawlerBaseMapper.pageLongArticlesVideos(id, pageSize);
             if (CollectionUtils.isEmpty(list)) {
                 break;
             }
-            id = list.stream().mapToLong(LongArticlesVideo::getId).max().getAsLong();
+            id = list.stream().mapToLong(LongArticlesVideoDTO::getId).max().getAsLong();
             List<LongArticlesText> batchSaveLongArticlesTextList = new ArrayList<>();
             List<LongArticlesMatchVideos> batchSaveLongArticlesMatchVideosList = new ArrayList<>();
             Set<String> existsIdSet = new HashSet<>();
-            for (LongArticlesVideo longArticlesVideo : list) {
-                if (longArticlesVideo.getContentId().endsWith("lehuo")) {
+            for (LongArticlesVideoDTO longArticlesVideoDTO : list) {
+                if (longArticlesVideoDTO.getContentId().endsWith("lehuo")) {
                     continue;
                 }
-                if (!existsIdSet.contains(longArticlesVideo.getContentId())) {
+                if (!existsIdSet.contains(longArticlesVideoDTO.getContentId())) {
                     LongArticlesText longArticlesText = new LongArticlesText();
-                    longArticlesText.setContentId(longArticlesVideo.getContentId());
-                    longArticlesText.setArticleTitle(longArticlesVideo.getArticleTitle());
-                    longArticlesText.setArticleText(longArticlesVideo.getArticleText());
-                    if (StringUtils.hasText(longArticlesVideo.getArticleText())) {
-                        longArticlesText.setKimiTitle(longArticlesVideo.getKimiTitle().replace("\"", ""));
+                    longArticlesText.setContentId(longArticlesVideoDTO.getContentId());
+                    longArticlesText.setArticleTitle(longArticlesVideoDTO.getArticleTitle());
+                    longArticlesText.setArticleText(longArticlesVideoDTO.getArticleText());
+                    if (StringUtils.hasText(longArticlesVideoDTO.getArticleText())) {
+                        longArticlesText.setKimiTitle(longArticlesVideoDTO.getKimiTitle().replace("\"", ""));
                     }
-                    if (StringUtils.hasText(longArticlesVideo.getKimiSummary())) {
-                        longArticlesText.setKimiSummary(longArticlesVideo.getKimiSummary());
-                        longArticlesText.setKimiKeys(longArticlesVideo.getKimiKeys());
+                    if (StringUtils.hasText(longArticlesVideoDTO.getKimiSummary())) {
+                        longArticlesText.setKimiSummary(longArticlesVideoDTO.getKimiSummary());
+                        longArticlesText.setKimiKeys(longArticlesVideoDTO.getKimiKeys());
                     } else {
-                        LongArticlesText text = kimiTitleMap.get(longArticlesVideo.getContentId());
+                        LongArticlesText text = kimiTitleMap.get(longArticlesVideoDTO.getContentId());
                         longArticlesText.setKimiSummary(text.getKimiSummary());
                         longArticlesText.setKimiKeys(text.getKimiKeys());
                     }
                     longArticlesText.setKimiStatus(1);
                     batchSaveLongArticlesTextList.add(longArticlesText);
-                    existsIdSet.add(longArticlesVideo.getContentId());
+                    existsIdSet.add(longArticlesVideoDTO.getContentId());
                 }
-                if (Objects.isNull(longArticlesVideo.getRequestTimeStamp())) {
+                if (Objects.isNull(longArticlesVideoDTO.getRequestTimeStamp())) {
                     continue;
                 }
                 LongArticlesMatchVideos longArticlesMatchVideos = new LongArticlesMatchVideos();
-                longArticlesMatchVideos.setContentId(longArticlesVideo.getContentId());
-                longArticlesMatchVideos.setTraceId(longArticlesVideo.getTraceId());
-                longArticlesMatchVideos.setGhId(longArticlesVideo.getGhId());
-                longArticlesMatchVideos.setAccountName(longArticlesVideo.getAccountName());
+                longArticlesMatchVideos.setContentId(longArticlesVideoDTO.getContentId());
+                longArticlesMatchVideos.setTraceId(longArticlesVideoDTO.getTraceId());
+                longArticlesMatchVideos.setGhId(longArticlesVideoDTO.getGhId());
+                longArticlesMatchVideos.setAccountName(longArticlesVideoDTO.getAccountName());
                 longArticlesMatchVideos.setContentStatus(4);
-                longArticlesMatchVideos.setSuccessStatus(longArticlesVideo.getSuccess());
-                longArticlesMatchVideos.setRequestTimestamp(longArticlesVideo.getRequestTimeStamp());
-                longArticlesMatchVideos.setUpdateTime(longArticlesVideo.getUpdateTime());
-                longArticlesMatchVideos.setProcessTimes(longArticlesVideo.getProcessTimes());
-                longArticlesMatchVideos.setResponse(getLongArticleVideoResponse(longArticlesVideo));
+                longArticlesMatchVideos.setSuccessStatus(longArticlesVideoDTO.getSuccess());
+                longArticlesMatchVideos.setRequestTimestamp(longArticlesVideoDTO.getRequestTimeStamp());
+                longArticlesMatchVideos.setUpdateTime(longArticlesVideoDTO.getUpdateTime());
+                longArticlesMatchVideos.setProcessTimes(longArticlesVideoDTO.getProcessTimes());
+                longArticlesMatchVideos.setResponse(getLongArticleVideoResponse(longArticlesVideoDTO));
                 batchSaveLongArticlesMatchVideosList.add(longArticlesMatchVideos);
             }
             if (CollectionUtil.isNotEmpty(batchSaveLongArticlesTextList)) {
@@ -228,28 +228,28 @@ public class DataFlushService {
                 longArticlesTextFlushNum, longArticlesMatchVideosFlushNum);
     }
 
-    private String getLongArticleVideoResponse(LongArticlesVideo longArticlesVideo) {
+    private String getLongArticleVideoResponse(LongArticlesVideoDTO longArticlesVideoDTO) {
         JSONArray jsonArray = new JSONArray();
-        if (StringUtils.hasText(longArticlesVideo.getResult1())) {
-            if (StringUtils.hasText(longArticlesVideo.getKimiTitle()) && longArticlesVideo.getKimiTitle().contains("\"")) {
-                longArticlesVideo.setResult1(longArticlesVideo.getResult1().replace(longArticlesVideo.getKimiTitle(),
-                        longArticlesVideo.getKimiTitle().replace("\"", "")));
+        if (StringUtils.hasText(longArticlesVideoDTO.getResult1())) {
+            if (StringUtils.hasText(longArticlesVideoDTO.getKimiTitle()) && longArticlesVideoDTO.getKimiTitle().contains("\"")) {
+                longArticlesVideoDTO.setResult1(longArticlesVideoDTO.getResult1().replace(longArticlesVideoDTO.getKimiTitle(),
+                        longArticlesVideoDTO.getKimiTitle().replace("\"", "")));
             }
-            jsonArray.add(resultToResponse(longArticlesVideo.getResult1()));
+            jsonArray.add(resultToResponse(longArticlesVideoDTO.getResult1()));
         }
-        if (StringUtils.hasText(longArticlesVideo.getResult2())) {
-            if (StringUtils.hasText(longArticlesVideo.getKimiTitle()) && longArticlesVideo.getKimiTitle().contains("\"")) {
-                longArticlesVideo.setResult2(longArticlesVideo.getResult2().replace(longArticlesVideo.getKimiTitle(),
-                        longArticlesVideo.getKimiTitle().replace("\"", "")));
+        if (StringUtils.hasText(longArticlesVideoDTO.getResult2())) {
+            if (StringUtils.hasText(longArticlesVideoDTO.getKimiTitle()) && longArticlesVideoDTO.getKimiTitle().contains("\"")) {
+                longArticlesVideoDTO.setResult2(longArticlesVideoDTO.getResult2().replace(longArticlesVideoDTO.getKimiTitle(),
+                        longArticlesVideoDTO.getKimiTitle().replace("\"", "")));
             }
-            jsonArray.add(resultToResponse(longArticlesVideo.getResult2()));
+            jsonArray.add(resultToResponse(longArticlesVideoDTO.getResult2()));
         }
-        if (StringUtils.hasText(longArticlesVideo.getResult3())) {
-            if (StringUtils.hasText(longArticlesVideo.getKimiTitle()) && longArticlesVideo.getKimiTitle().contains("\"")) {
-                longArticlesVideo.setResult3(longArticlesVideo.getResult3().replace(longArticlesVideo.getKimiTitle(),
-                        longArticlesVideo.getKimiTitle().replace("\"", "")));
+        if (StringUtils.hasText(longArticlesVideoDTO.getResult3())) {
+            if (StringUtils.hasText(longArticlesVideoDTO.getKimiTitle()) && longArticlesVideoDTO.getKimiTitle().contains("\"")) {
+                longArticlesVideoDTO.setResult3(longArticlesVideoDTO.getResult3().replace(longArticlesVideoDTO.getKimiTitle(),
+                        longArticlesVideoDTO.getKimiTitle().replace("\"", "")));
             }
-            jsonArray.add(resultToResponse(longArticlesVideo.getResult3()));
+            jsonArray.add(resultToResponse(longArticlesVideoDTO.getResult3()));
         }
         return JSONObject.toJSONString(jsonArray);
     }
@@ -338,16 +338,16 @@ public class DataFlushService {
                     long start = System.currentTimeMillis();
                     List<String> traceIds = matchVideosList.stream().map(LongArticlesMatchVideos::getTraceId)
                             .distinct().collect(Collectors.toList());
-                    List<LongArticlesVideo> longArticlesVideoList = crawlerBaseMapper.getLongArticlesVideo(traceIds);
-                    Map<String, LongArticlesVideo> longArticlesVideoMap = longArticlesVideoList.stream().collect(
-                            Collectors.toMap(LongArticlesVideo::getTraceId, o -> o, (existing, replacement) -> replacement));
+                    List<LongArticlesVideoDTO> longArticlesVideoDTOList = crawlerBaseMapper.getLongArticlesVideo(traceIds);
+                    Map<String, LongArticlesVideoDTO> longArticlesVideoMap = longArticlesVideoDTOList.stream().collect(
+                            Collectors.toMap(LongArticlesVideoDTO::getTraceId, o -> o, (existing, replacement) -> replacement));
                     CountDownLatch countDownLatch = new CountDownLatch(matchVideosList.size());
                     for (LongArticlesMatchVideos longArticlesMatchVideos : matchVideosList) {
                         pool.submit(() -> {
                             try {
-                                LongArticlesVideo longArticlesVideo = longArticlesVideoMap.get(longArticlesMatchVideos.getTraceId());
-                                if (Objects.nonNull(longArticlesVideo)) {
-                                    longArticlesMatchVideos.setResponse(getLongArticleVideoResponse(longArticlesVideo));
+                                LongArticlesVideoDTO longArticlesVideoDTO = longArticlesVideoMap.get(longArticlesMatchVideos.getTraceId());
+                                if (Objects.nonNull(longArticlesVideoDTO)) {
+                                    longArticlesMatchVideos.setResponse(getLongArticleVideoResponse(longArticlesVideoDTO));
                                     longArticleBaseMapper.updateLongArticleMatchVideosResponse(longArticlesMatchVideos);
                                 }
                             } finally {

long-article-recommend-service/src/main/resources/mapper/crawler/CrawlerBaseMapper.xml

@@ -32,7 +32,7 @@
         select count(1) from long_articles_video where content_status = 2 and success = 0
     </select>
     <select id="pageLongArticlesVideos"
-            resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideo">
+            resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideoDTO">
         select * from long_articles_video where content_status = 2 and success = 0 and id > #{id} order by id limit #{pageSize}
     </select>
     <select id="getLongArticlesText" resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesText">
@@ -40,7 +40,7 @@
     </select>
 
     <select id="getLongArticlesVideo"
-            resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideo">
+            resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideoDTO">
         select * from long_articles_video where trace_id in
         <foreach collection="traceIds" item="item" open="(" close=")" separator=",">
             #{item}