Merge branch 'wyp/0311-categoryVersion' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/ArticleCategoryMapper.java

@@ -20,5 +20,5 @@ public interface ArticleCategoryMapper {
 
     void updateDatastatScoreCategory(String title, String category);
 
-    List<PublishSingleVideoSource> getVideoPoolArticleCategoryDealList();
+    List<PublishSingleVideoSource> getVideoPoolArticleCategoryDealList(Integer version);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleCategory.java

@@ -17,6 +17,9 @@ import javax.persistence.Table;
 public class ArticleCategory {
 
     @Id
+    @Column(name = "id")
+    private Long id;
+
     @Column(name = "produce_content_id")
     private String produceContentId;
 
@@ -47,6 +50,9 @@ public class ArticleCategory {
     @Column(name = "retry_times")
     private Integer retryTimes;
 
+    @Column(name = "version")
+    private Integer version;
+
     @Column(name = "create_timestamp")
     private Long createTimestamp;
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/ArticleCategoryRepository.java

@@ -9,15 +9,15 @@ import org.springframework.stereotype.Repository;
 import java.util.List;
 
 @Repository
-public interface ArticleCategoryRepository extends JpaRepository<ArticleCategory, String> {
+public interface ArticleCategoryRepository extends JpaRepository<ArticleCategory, Long> {
 
-    List<ArticleCategory> getByProduceContentIdIn(List<String> produceContentIds);
+    List<ArticleCategory> getByProduceContentIdInAndVersion(List<String> produceContentIds, Integer version);
 
-    List<ArticleCategory> getByStatus(Integer status);
+    List<ArticleCategory> getByStatusAndVersion(Integer status, Integer version);
 
-    Page<ArticleCategory> getByStatus(Integer status, PageRequest pageRequest);
+    Page<ArticleCategory> getByStatusAndVersion(Integer status, Integer version, PageRequest pageRequest);
 
-    List<ArticleCategory> getAllByChannelContentIdIn(List<String> channelContentIds);
+    List<ArticleCategory> getAllByChannelContentIdInAndVersion(List<String> channelContentIds, Integer version);
 
-    List<ArticleCategory> getByStatusAndRetryTimesLessThan(Integer status, Integer retryTimes);
+    List<ArticleCategory> getByStatusAndRetryTimesLessThanAndVersion(Integer status, Integer retryTimes, Integer version);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleCategoryService.java

@@ -68,6 +68,9 @@ public class ArticleCategoryService {
     @Value("${kimiCategoryPrompt:}")
     private String kimiCategoryPrompt;
 
+    @Value("${category.active.version:1}")
+    private Integer activeVersion;
+
     @ApolloJsonValue("${articlePromotionProduceConfig:{}}")
     private Map<String, Map<String, Map<String, String>>> produceConfig;
 
@@ -101,7 +104,7 @@ public class ArticleCategoryService {
     }
 
     private void dealArticleCategory() {
-        List<ArticleCategory> dealList = articleCategoryRepository.getByStatus(ArticleCategoryStatusEnum.WAITING.getCode());
+        List<ArticleCategory> dealList = articleCategoryRepository.getByStatusAndVersion(ArticleCategoryStatusEnum.WAITING.getCode(), activeVersion);
         List<List<ArticleCategory>> partitionList = Lists.partition(dealList, 20);
         for (List<ArticleCategory> partition : partitionList) {
             List<String> partitionTitles = partition.stream().map(ArticleCategory::getTitle).distinct().collect(Collectors.toList());
@@ -148,7 +151,7 @@ public class ArticleCategoryService {
         for (ArticleCrawlerPlan crawlerPlan : articleCrawlerPlanList) {
             List<ProduceContentCrawlerVO> list = crawlerContentByPlanService.getCrawlerContentByPlan(crawlerPlan.getCrawlerPlanId(), producePlanIds);
             List<String> produceContentIds = list.stream().map(ProduceContentCrawlerVO::getProduceContentId).collect(Collectors.toList());
-            List<ArticleCategory> exists = articleCategoryRepository.getByProduceContentIdIn(produceContentIds);
+            List<ArticleCategory> exists = articleCategoryRepository.getByProduceContentIdInAndVersion(produceContentIds, activeVersion);
             List<String> existsIds = exists.stream().map(ArticleCategory::getProduceContentId).collect(Collectors.toList());
             list = list.stream().filter(o -> !existsIds.contains(o.getProduceContentId())).collect(Collectors.toList());
             long now = System.currentTimeMillis();
@@ -207,7 +210,7 @@ public class ArticleCategoryService {
             Map<String, ArticlePoolPromotionSource> sourceMap = sourceList.stream()
                     .collect(Collectors.toMap(ArticlePoolPromotionSource::getChannelContentId, Function.identity()));
             // 根据produceContentId查询category
-            List<ArticleCategory> articleCategoryList = articleCategoryRepository.getByStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
+            List<ArticleCategory> articleCategoryList = articleCategoryRepository.getByStatusAndVersion(ArticleCategoryStatusEnum.SUCCESS.getCode(), activeVersion);
             Map<String, ArticleCategory> categoryMap = articleCategoryList.stream()
                     .collect(Collectors.toMap(ArticleCategory::getProduceContentId, Function.identity()));
             Map<String, ArticleCategory> coldStartCategoryMap = articleCategoryList.stream()
@@ -239,7 +242,7 @@ public class ArticleCategoryService {
     private void addVideoPoolArticleCategory() {
         List<ArticleCategory> saveList = new ArrayList<>();
         // 查找所有待处理视频内容池内容
-        List<PublishSingleVideoSource> dealList = articleCategoryMapper.getVideoPoolArticleCategoryDealList();
+        List<PublishSingleVideoSource> dealList = articleCategoryMapper.getVideoPoolArticleCategoryDealList(activeVersion);
         if (CollectionUtils.isEmpty(dealList)) {
             return;
         }
@@ -261,7 +264,7 @@ public class ArticleCategoryService {
     private List<ArticleCategory> addArticleCategoryByProducePlan(List<String> producePlanIds) {
         List<ProducePlanExeRecord> produceContentList = aigcBaseMapper.getAllByProducePlanId(producePlanIds);
         List<String> channelContentIds = produceContentList.stream().map(ProducePlanExeRecord::getChannelContentId).distinct().collect(Collectors.toList());
-        List<ArticleCategory> articleCategoryList = articleCategoryRepository.getAllByChannelContentIdIn(channelContentIds);
+        List<ArticleCategory> articleCategoryList = articleCategoryRepository.getAllByChannelContentIdInAndVersion(channelContentIds, activeVersion);
         List<String> articleCategoryIds = articleCategoryList.stream().map(ArticleCategory::getChannelContentId).collect(Collectors.toList());
         List<ProduceContentCrawlerVO> list = produceContentList.stream().filter(o -> !articleCategoryIds.contains(o.getChannelContentId())).map(o -> {
             ProduceContentCrawlerVO item = new ProduceContentCrawlerVO();
@@ -304,7 +307,7 @@ public class ArticleCategoryService {
     }
 
     public void articleCategoryJobRetry() {
-        List<ArticleCategory> dealList = articleCategoryRepository.getByStatusAndRetryTimesLessThan(ArticleCategoryStatusEnum.FAIL.getCode(), 3);
+        List<ArticleCategory> dealList = articleCategoryRepository.getByStatusAndRetryTimesLessThanAndVersion(ArticleCategoryStatusEnum.FAIL.getCode(), 3, activeVersion);
         for (ArticleCategory articleCategory : dealList) {
             List<String> partitionTitles = Collections.singletonList(articleCategory.getTitle());
             String prompt = buildKimiPrompt(partitionTitles);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -129,6 +129,8 @@ public class RecallService implements ApplicationContextAware {
     private Boolean contentHisFeishuEnable;
     @Value("${morning.noon.fission.rate:0.64}")
     private double morningNoonFissionRate;
+    @Value("${category.active.version:1}")
+    private Integer activeVersion;
 
 
     @PostConstruct
@@ -506,7 +508,7 @@ public class RecallService implements ApplicationContextAware {
                     .collect(Collectors.toMap(PublishContent::getId, Function.identity()));
         }
         // 根据produceContentId查询category
-        List<ArticleCategory> articleCategoryList = articleCategoryRepository.getByStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
+        List<ArticleCategory> articleCategoryList = articleCategoryRepository.getByStatusAndVersion(ArticleCategoryStatusEnum.SUCCESS.getCode(), activeVersion);
         Map<String, ArticleCategory> categoryMap = articleCategoryList.stream()
                 .collect(Collectors.toMap(ArticleCategory::getProduceContentId, Function.identity()));
         Map<String, ArticleCategory> coldStartCategoryMap = articleCategoryList.stream()

long-article-recommend-service/src/main/resources/mapper/longArticle/ArticleCategoryMapper.xml

@@ -57,7 +57,7 @@
         where bad_status = 0
           and audit_status = 1
           and content_trace_id not in (select produce_content_id
-                                       from article_category)
+                                       from article_category where version = #{version})
     </select>
 
 </mapper>

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/ArticleVideoAuditTest.java

@@ -83,8 +83,8 @@ public class ArticleVideoAuditTest {
 
     @Test
     public void ArticleCategoryTest() {
-        List<ArticleCategory> dealList = articleCategoryRepository.getByStatus(
-                ArticleCategoryStatusEnum.SUCCESS.getCode());
+        List<ArticleCategory> dealList = articleCategoryRepository.getByStatusAndVersion(
+                ArticleCategoryStatusEnum.SUCCESS.getCode(), 1);
         dealList = dealList.subList(0, 200);
         List<List<ArticleCategory>> partitionList = Lists.partition(dealList, 50);
         JSONArray result = new JSONArray();

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/XxlJobTest.java

@@ -71,7 +71,7 @@ public class XxlJobTest {
     public void articleCategoryTest() {
         List<ProducePlanExeRecord> produceContentList = aigcBaseMapper.getAllByProducePlanId(producePlanIds);
         List<String> channelContentIds = produceContentList.stream().map(ProducePlanExeRecord::getChannelContentId).distinct().collect(Collectors.toList());
-        List<ArticleCategory> articleCategoryList = articleCategoryRepository.getAllByChannelContentIdIn(channelContentIds);
+        List<ArticleCategory> articleCategoryList = articleCategoryRepository.getAllByChannelContentIdInAndVersion(channelContentIds, 1);
         List<String>  articleCategoryIds = articleCategoryList.stream().map(ArticleCategory::getChannelContentId).collect(Collectors.toList());
         List<ProduceContentCrawlerVO> list = produceContentList.stream().filter(o -> !articleCategoryIds.contains(o.getChannelContentId())).map(o -> {
             ProduceContentCrawlerVO item = new ProduceContentCrawlerVO();