长文小程序匹配优化

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/config/AigcDataSourceConfig.java

@@ -31,13 +31,11 @@ public class AigcDataSourceConfig {
         return new DruidDataSource();
     }
 
-    @Primary
     @Bean(name = "aigcTransactionManager")
     public PlatformTransactionManager aigcTransactionManager(@Qualifier("aigcDataSource") DataSource dataSource) {
         return new DataSourceTransactionManager(dataSource);
     }
 
-    @Primary
     @Bean(name = "aigcSqlSessionFactory")
     public SqlSessionFactory aigcSqlSessionFactory(@Qualifier("aigcDataSource") DataSource dataSource) throws Exception {
         final SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean();

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/model/bo/MatchContent.java

@@ -24,4 +24,6 @@ public class MatchContent {
     private Integer sourceType;
 
     private String channelContentId;
+
+    private Integer publishFlag;
 }

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/CrawlerVideoServiceImpl.java

@@ -10,6 +10,7 @@ import com.tzld.piaoquan.longarticle.utils.other.*;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
@@ -33,6 +34,8 @@ public class CrawlerVideoServiceImpl {
 
     private static final int MAX_NUM = 3;
 
+    private static final int MIN_NUM = 1;
+
     private static final List<String> sensitiveWords = new ArrayList<String>() {{
         add("人民");
         add("必胜");
@@ -46,20 +49,22 @@ public class CrawlerVideoServiceImpl {
         add("习");
     }};
 
-    public void addCrawlerVideo(String contentId, String rootContentId, LongArticlesText kimiText) {
+    public boolean addCrawlerVideo(String contentId, String rootContentId, LongArticlesText kimiText) {
         boolean b = existCrawlerVideo(contentId);
         if (b) {
-            return;
+            return true;
         }
         if (rootContentId != null) {
             CrawlerVideoExample example = new CrawlerVideoExample();
             example.createCriteria().andContentIdEqualTo(rootContentId);
             List<CrawlerVideo> crawlerVideos = crawlerVideoMapper.selectByExample(example);
-            for (CrawlerVideo crawlerVideo : crawlerVideos) {
-                crawlerVideo.setContentId(contentId);
-                crawlerVideoMapper.insertSelective(crawlerVideo);
+            if (!CollectionUtils.isEmpty(crawlerVideos)) {
+                for (CrawlerVideo crawlerVideo : crawlerVideos) {
+                    crawlerVideo.setContentId(contentId);
+                    crawlerVideoMapper.insertSelective(crawlerVideo);
+                    return true;
+                }
             }
-            return;
         }
         List<CrawlerVideo> crawlerVideoList = searchVideosFromWeb(kimiText);
         if (!CollectionUtils.isEmpty(crawlerVideoList)) {
@@ -81,14 +86,14 @@ public class CrawlerVideoServiceImpl {
                 crawlerVideoMapper.insertSelective(crawlerVideo);
             }
         }
-        pushOss(contentId);
+        return pushOss(contentId);
     }
 
     public boolean existCrawlerVideo(String contentId) {
         CrawlerVideoExample example = new CrawlerVideoExample();
         example.createCriteria().andContentIdEqualTo(contentId).andDownloadStatusEqualTo(2);
         long l = crawlerVideoMapper.countByExample(example);
-        return l >= MAX_NUM;
+        return l >= MIN_NUM;
     }
 
 
@@ -218,13 +223,13 @@ public class CrawlerVideoServiceImpl {
         return crawlerVideo;
     }
 
-    public void pushOss(String contentId) {
+    public boolean pushOss(String contentId) {
         CrawlerVideoExample example = new CrawlerVideoExample();
         example.createCriteria().andContentIdEqualTo(contentId).andDownloadStatusEqualTo(0);
         example.setOrderByClause("score desc");
         List<CrawlerVideo> crawlerVideoList = crawlerVideoMapper.selectByExampleWithBLOBs(example);
         if (CollectionUtils.isEmpty(crawlerVideoList)) {
-            return;
+            return false;
         }
         int count = 0;
         for (CrawlerVideo crawlerVideo : crawlerVideoList) {
@@ -258,9 +263,10 @@ public class CrawlerVideoServiceImpl {
                 }
             }
             if (count >= MAX_NUM) {
-                break;
+                return true;
             }
         }
+        return false;
     }
 
 }

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/MatchVideoServiceImpl.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.longarticle.service.local.impl;
 
+import com.tzld.piaoquan.longarticle.common.enums.ContentStatusEnum;
 import com.tzld.piaoquan.longarticle.component.RedisLock;
 import com.tzld.piaoquan.longarticle.dao.mapper.aigc.ProducePlanExeRecordMapper;
 import com.tzld.piaoquan.longarticle.dao.mapper.longarticle.LongArticlesTextMapper;
@@ -32,6 +33,7 @@ public class MatchVideoServiceImpl {
 
     private static final String CRAWLER_LOCK_KEY = "crawler_lock_key_%s";
 
+    private static final String CRAWLER_COUNT_KEY = "crawler_count_key_%s";
 
     @Autowired
     KimiService kimiService;
@@ -82,7 +84,7 @@ public class MatchVideoServiceImpl {
         matchVideo.setGhId(matchContent.getGhId());
         matchVideo.setFlowPoolLevel(matchContent.getFlowPoolLevelTag());
         matchVideo.setContentStatus(0);
-        matchVideo.setPublishFlag(2);
+        matchVideo.setPublishFlag(matchContent.getPublishFlag());
         long timestamp = System.currentTimeMillis() / 1000;
         matchVideo.setContentStatusUpdateTime(Long.valueOf(timestamp).intValue());
         matchVideo.setRequestTimestamp(Long.valueOf(timestamp).intValue());
@@ -100,7 +102,7 @@ public class MatchVideoServiceImpl {
         }
     }
 
-    public void getMatchVideo() {
+    public void getMatchVideo() throws InterruptedException {
         if (matchVideoPoolExecutor.getCorePoolSize() - matchVideoPoolExecutor.getActiveCount() > 0) {
             int threadSize = matchVideoPoolExecutor.getCorePoolSize() - matchVideoPoolExecutor.getActiveCount();
             log.info("threadNum={}", threadSize);
@@ -125,62 +127,61 @@ public class MatchVideoServiceImpl {
                     countDownLatch.countDown();
                 }));
             }
-        }
-
+            List<Integer> targetStatus = new ArrayList<Integer>() {{
+                add(0);
+                add(1);
+                add(2);
+            }};
+            List<MatchVideo> matchVideos;
+            //循环增加游标位置，通过主键索引过滤已经处理的数据
+            do {
+                Integer id = (Integer) redisTemplate.opsForValue().get("last_match_video_id");
+                if (id == null) {
+                    id = 0;
+                }
+                MatchVideoExample example = new MatchVideoExample();
+                example.createCriteria().andIdGreaterThan(id);
+                example.setOrderByClause("id asc");
+                Page page = new Page<>();
+                page.setCurrentPage(1);
+                page.setPageSize(5000);
+                example.setPage(page);
+                matchVideos = matchVideoMapper.selectByExample(example);
+                boolean flag = true;
+                for (MatchVideo matchVideo : matchVideos) {
+                    if (targetStatus.contains(matchVideo.getContentStatus())) {
+                        flag = false;
+                        break;
+                    }
+                }
+                if (flag) {
+                    Integer lastId = matchVideos.get(matchVideos.size() - 1).getId();
+                    redisTemplate.opsForValue().set("last_match_video_id", lastId);
+                }
+            } while (CollectionUtils.isEmpty(matchVideos));
 
-        List<Integer> targetStatus = new ArrayList<Integer>() {{
-            add(0);
-            add(1);
-            add(2);
-        }};
-        List<MatchVideo> matchVideos;
-        //循环增加游标位置，通过主键索引过滤已经处理的数据
-        do {
-            Integer id = (Integer) redisTemplate.opsForValue().get("last_match_video_id");
-            if (id == null) {
-                id = 0;
-            }
-            MatchVideoExample example = new MatchVideoExample();
-            example.createCriteria().andIdGreaterThan(id);
-            example.setOrderByClause("id asc");
-            Page page = new Page<>();
-            page.setCurrentPage(1);
-            page.setPageSize(5000);
-            example.setPage(page);
-            matchVideos = matchVideoMapper.selectByExample(example);
-            boolean flag = true;
-            for (MatchVideo matchVideo : matchVideos) {
-                if (targetStatus.contains(matchVideo.getContentStatus())) {
-                    flag = false;
+            int pageNum = 1;
+            do {
+                Integer id = (Integer) redisTemplate.opsForValue().get("last_match_video_id");
+                if (id == null) {
                     break;
                 }
-            }
-            if (flag) {
-                Integer lastId = matchVideos.get(matchVideos.size() - 1).getId();
-                redisTemplate.opsForValue().set("last_match_video_id", lastId);
-            }
-        } while (CollectionUtils.isEmpty(matchVideos));
 
-        int pageNum = 1;
-        do {
-            Integer id = (Integer) redisTemplate.opsForValue().get("last_match_video_id");
-            if (id == null) {
-                break;
-            }
-
-            int pageSize = 5000;
-            MatchVideoExample example = new MatchVideoExample();
-            example.createCriteria().andIdGreaterThan(id).andContentStatusIn(targetStatus);
-            example.setOrderByClause("id asc");
-            Page page = new Page<>();
-            page.setCurrentPage(pageNum);
-            page.setPageSize(pageSize);
-            example.setPage(page);
-            matchVideos = matchVideoMapper.selectByExample(example);
-            matchVideoQueue.addAll(matchVideos);
-            pageNum++;
-        } while (CollectionUtils.isEmpty(matchVideos));
+                int pageSize = 5000;
+                MatchVideoExample example = new MatchVideoExample();
+                example.createCriteria().andIdGreaterThan(id).andContentStatusIn(targetStatus);
+                example.setOrderByClause("id asc");
+                Page page = new Page<>();
+                page.setCurrentPage(pageNum);
+                page.setPageSize(pageSize);
+                example.setPage(page);
+                matchVideos = matchVideoMapper.selectByExample(example);
+                matchVideoQueue.addAll(matchVideos);
+                pageNum++;
+            } while (CollectionUtils.isEmpty(matchVideos));
+            countDownLatch.await();
 
+        }
     }
 
 
@@ -213,38 +214,69 @@ public class MatchVideoServiceImpl {
         }
         boolean existCrawlerVideo = crawlerVideoService.existCrawlerVideo(matchVideo.getContentId());
         if (!existCrawlerVideo) {
+            MatchVideoExample example = new MatchVideoExample();
+            example.createCriteria().andContentIdEqualTo(matchVideo.getContentId());
+            List<MatchVideo> matchVideos = matchVideoMapper.selectByExample(example);
+            if (!CollectionUtils.isEmpty(matchVideos)) {
+                for (MatchVideo matchVideo1 : matchVideos) {
+                    if (matchVideo1.getId().equals(matchVideo.getId())) {
+                        continue;
+                    }
+                    if (ContentStatusEnum.isFail(matchVideo1.getContentStatus())) {
+                        updateStatus(matchVideo.getId(), matchVideo1.getContentStatus());
+                        return;
+                    }
+                }
+            }
             String lockKey = String.format(CRAWLER_LOCK_KEY, matchVideo.getContentId());
             String lockValue = UUID.randomUUID().toString();
             boolean lock = redisLock.tryLock(lockKey, lockValue, 600, TimeUnit.SECONDS);
             if (lock) {
                 LongArticlesText kimiText = kimiService.getKimiText(matchVideo.getContentId());
                 String rootContentId = getRootContentId(matchVideo.getContentId());
-                crawlerVideoService.addCrawlerVideo(matchVideo.getContentId(), rootContentId, kimiText);
+                boolean res = crawlerVideoService.addCrawlerVideo(matchVideo.getContentId(), rootContentId, kimiText);
+                if (res) {
+                    updateStatus(matchVideo.getId(), 3);
+                } else {
+                    //匹配失败记录
+                    String countKey = String.format(CRAWLER_COUNT_KEY, matchVideo.getContentId());
+                    Integer count = (Integer) redisTemplate.opsForValue().get(countKey);
+                    if (count != null && count >= 3) {
+                        //更新状态为失败
+                        updateStatus(matchVideo.getId(), ContentStatusEnum.ERROR_99.getStatusCode());
+                    } else {
+                        if (count == null) {
+                            redisTemplate.opsForValue().set(countKey, 1, 3, TimeUnit.DAYS);
+                        } else {
+                            redisTemplate.opsForValue().set(countKey, count + 1, 3, TimeUnit.DAYS);
+                        }
+                    }
+                }
                 redisLock.unlock(lockKey, lockValue);
             }
         } else {
             //更新状态为etl执行完成
-            updateStatus(matchVideo.getId(), 3);
+            updateStatus(matchVideo.getId(), ContentStatusEnum.SUCCESS_3.getStatusCode());
         }
     }
 
     private String getRootContentId(String contentId) {
-        ProducePlanExeRecordExample example = new ProducePlanExeRecordExample();
-        example.createCriteria().andPlanExeIdEqualTo(contentId);
-        List<ProducePlanExeRecord> producePlanExeRecords = producePlanExeRecordMapper.selectByExample(example);
-        if (!CollectionUtils.isEmpty(producePlanExeRecords)) {
-            String channelContentId = producePlanExeRecords.get(0).getChannelContentId();
-            if (channelContentId == null) {
-                return null;
-            }
-            PromotionSourceExample promotionSourceExample = new PromotionSourceExample();
-            promotionSourceExample.createCriteria().andChannelContentIdEqualTo(channelContentId);
-            List<PromotionSource> promotionSources = promotionSourceMapper.selectByExample(promotionSourceExample);
-            if (CollectionUtils.isEmpty(promotionSources)) {
-                return null;
-            }
-            return promotionSources.get(0).getRootProduceContentId();
-        }
+//        ProducePlanExeRecordExample example = new ProducePlanExeRecordExample();
+//        example.createCriteria().andPlanExeIdEqualTo(contentId);
+//        List<ProducePlanExeRecord> producePlanExeRecords = producePlanExeRecordMapper.selectByExample(example);
+//        if (!CollectionUtils.isEmpty(producePlanExeRecords)) {
+//            String channelContentId = producePlanExeRecords.get(0).getChannelContentId();
+//            if (channelContentId == null) {
+//                return null;
+//            }
+//            PromotionSourceExample promotionSourceExample = new PromotionSourceExample();
+//            promotionSourceExample.createCriteria().andChannelContentIdEqualTo(channelContentId);
+//            List<PromotionSource> promotionSources = promotionSourceMapper.selectByExample(promotionSourceExample);
+//            if (CollectionUtils.isEmpty(promotionSources)) {
+//                return null;
+//            }
+//            return promotionSources.get(0).getRootProduceContentId();
+//        }
         return null;
     }