Merge branch 'wyp/0311-categoryVersion' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/AccountCategory.java

@@ -26,6 +26,9 @@ public class AccountCategory {
     @Column(name = "status")
     private Integer status;
 
+    @Column(name = "version")
+    private Integer version;
+
     @Column(name = "create_timestamp")
     private Long createTimestamp;
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/DeepSeekApiService.java

@@ -25,9 +25,9 @@ public class DeepSeekApiService {
     @PostConstruct
     public void init() {
         client = new OkHttpClient().newBuilder()
-                .connectTimeout(15, TimeUnit.MINUTES)
-                .readTimeout(15, TimeUnit.MINUTES)
-                .writeTimeout(15, TimeUnit.MINUTES)
+                .connectTimeout(5, TimeUnit.MINUTES)
+                .readTimeout(5, TimeUnit.MINUTES)
+                .writeTimeout(5, TimeUnit.MINUTES)
                 .build();
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/AccountCategoryRepository.java

@@ -9,8 +9,8 @@ import java.util.List;
 @Repository
 public interface AccountCategoryRepository extends JpaRepository<AccountCategory, AccountCategory.PK> {
 
-    List<AccountCategory> getByGhIdAndStatus(String ghId, Integer status);
+    List<AccountCategory> getByGhIdAndStatusAndVersion(String ghId, Integer status, Integer version);
 
-    List<AccountCategory> getByStatus(Integer status);
+    List<AccountCategory> getByStatusAndVersion(Integer status, Integer version);
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/ArticleCategoryRepository.java

@@ -13,6 +13,8 @@ public interface ArticleCategoryRepository extends JpaRepository<ArticleCategory
 
     List<ArticleCategory> getByProduceContentIdInAndVersion(List<String> produceContentIds, Integer version);
 
+    ArticleCategory getByProduceContentIdAndVersion(String produceContentId, Integer version);
+
     List<ArticleCategory> getByStatusAndVersion(Integer status, Integer version);
 
     Page<ArticleCategory> getByStatusAndVersion(Integer status, Integer version, PageRequest pageRequest);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/DatastatScoreRepository.java

@@ -9,4 +9,6 @@ import java.util.List;
 @Repository
 public interface DatastatScoreRepository extends JpaRepository<DatastatScore, Long> {
     void deleteByDtIn(List<String> dateStrList);
+
+    List<DatastatScore> getBySourceIdIsNotNull();
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleCategoryService.java

@@ -66,7 +66,7 @@ public class ArticleCategoryService {
             "\"20241030070010871546586\"]}")
     private static List<String> producePlanIds;
     @Value("${kimiCategoryPrompt:}")
-    private String kimiCategoryPrompt;
+    private String categoryPrompt;
 
     @Value("${category.active.version:1}")
     private Integer activeVersion;
@@ -99,7 +99,7 @@ public class ArticleCategoryService {
         addPromotionArticleCategoryByProducePlan();
         // 视频内容池 添加品类处理任务
         addVideoPoolArticleCategory();
-        // 调用kimi进行内容分类
+        // 调用大模型进行内容分类
         dealArticleCategory();
     }
 
@@ -108,7 +108,7 @@ public class ArticleCategoryService {
         List<List<ArticleCategory>> partitionList = Lists.partition(dealList, 20);
         for (List<ArticleCategory> partition : partitionList) {
             List<String> partitionTitles = partition.stream().map(ArticleCategory::getTitle).distinct().collect(Collectors.toList());
-            String prompt = buildKimiPrompt(partitionTitles);
+            String prompt = buildCategoryPrompt(partitionTitles);
             KimiResult kimiResult = deepSeekApiService.requestOfficialApi(prompt, null, null, true);
             long now = System.currentTimeMillis();
             JSONObject obj = null;
@@ -120,7 +120,7 @@ public class ArticleCategoryService {
                 }
             }
             for (ArticleCategory articleCategory : partition) {
-                articleCategory.setKimiResult(kimiResult.getResponseStr());
+                articleCategory.setKimiResult(kimiResult.getResponseStr().trim());
                 articleCategory.setUpdateTimestamp(now);
                 if (kimiResult.isSuccess() && Objects.nonNull(obj) && obj.containsKey(articleCategory.getTitle())) {
                     articleCategory.setCategory(obj.getString(articleCategory.getTitle()));
@@ -164,6 +164,7 @@ public class ArticleCategoryService {
                 item.setTitle(vo.getTitle());
                 item.setTitleMd5(Md5Util.encoderByMd5(vo.getTitle()));
                 item.setCreateTimestamp(now);
+                item.setVersion(activeVersion);
                 saveList.add(item);
             }
             if (CollectionUtils.isNotEmpty(saveList)) {
@@ -231,7 +232,7 @@ public class ArticleCategoryService {
                 }
                 if (Objects.nonNull(category) && StringUtils.hasText(category.getCategory())) {
                     articleCategory.setCategory(category.getCategory());
-                    articleCategory.setKimiResult(category.getKimiResult());
+                    articleCategory.setKimiResult(category.getKimiResult().trim());
                     articleCategory.setStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
                 }
             }
@@ -253,6 +254,7 @@ public class ArticleCategoryService {
             item.setTitle(videoSource.getArticleTitle());
             item.setTitleMd5(Md5Util.encoderByMd5(videoSource.getArticleTitle()));
             item.setCreateTimestamp(now);
+            item.setVersion(activeVersion);
             saveList.add(item);
         }
         articleCategoryMapper.batchInsertArticleCategory(saveList);
@@ -291,14 +293,15 @@ public class ArticleCategoryService {
                 item.setTitle(title);
                 item.setTitleMd5(Md5Util.encoderByMd5(title));
                 item.setCreateTimestamp(now);
+                item.setVersion(activeVersion);
                 saveList.add(item);
             }
         }
         return saveList;
     }
 
-    private String buildKimiPrompt(List<String> titleList) {
-        StringBuilder prompt = new StringBuilder(kimiCategoryPrompt);
+    private String buildCategoryPrompt(List<String> titleList) {
+        StringBuilder prompt = new StringBuilder(categoryPrompt);
         prompt.append("\n");
         for (String title : titleList) {
             prompt.append(title).append("\n");
@@ -310,7 +313,7 @@ public class ArticleCategoryService {
         List<ArticleCategory> dealList = articleCategoryRepository.getByStatusAndRetryTimesLessThanAndVersion(ArticleCategoryStatusEnum.FAIL.getCode(), 3, activeVersion);
         for (ArticleCategory articleCategory : dealList) {
             List<String> partitionTitles = Collections.singletonList(articleCategory.getTitle());
-            String prompt = buildKimiPrompt(partitionTitles);
+            String prompt = buildCategoryPrompt(partitionTitles);
             KimiResult kimiResult = deepSeekApiService.requestOfficialApi(prompt, null, null, true);
             long now = System.currentTimeMillis();
             JSONObject obj = null;
@@ -321,7 +324,7 @@ public class ArticleCategoryService {
                     log.error(kimiResult.getResponse().getChoices().get(0).getMessage().getContent());
                 }
             }
-            articleCategory.setKimiResult(kimiResult.getResponseStr());
+            articleCategory.setKimiResult(kimiResult.getResponseStr().trim());
             articleCategory.setUpdateTimestamp(now);
             articleCategory.setRetryTimes(articleCategory.getRetryTimes() + 1);
             if (kimiResult.isSuccess() && Objects.nonNull(obj)) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -486,7 +486,7 @@ public class RecallService implements ApplicationContextAware {
                 .filter(o -> "1".equals(o.getPosition()))
                 .filter(o -> Objects.nonNull(o.getReadAvg()) && o.getReadAvg() > 0 && o.getFans() > 1000)
                 .collect(Collectors.groupingBy(AccountAvgInfo::getGhId, Collectors.toMap(AccountAvgInfo::getUpdateTime, o -> o)));
-        List<AccountCategory> accountCategoryList = accountCategoryRepository.getByStatus(StatusEnum.ONE.getCode());
+        List<AccountCategory> accountCategoryList = accountCategoryRepository.getByStatusAndVersion(StatusEnum.ONE.getCode(), activeVersion);
         Map<String, JSONObject> accountCategoryMap = accountCategoryList.stream().filter(o -> StringUtils.hasText(o.getCategoryMap()))
                 .collect(Collectors.toMap(AccountCategory::getGhId, o -> JSONObject.parseObject(o.getCategoryMap())));
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/strategy/CategoryStrategy.java

@@ -14,6 +14,7 @@ import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreParam;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreStrategy;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
@@ -34,6 +35,9 @@ public class CategoryStrategy implements ScoreStrategy {
     @ApolloJsonValue("${accountCategoryWeightConfig:{}}")
     private Map<String, AccountCategoryWeightConfig[]> accountCategoryWeightConfigMap;
 
+    @Value("${category.active.version:1}")
+    private Integer activeVersion;
+
 
     @Override
     public List<Score> score(ScoreParam param) {
@@ -42,7 +46,8 @@ public class CategoryStrategy implements ScoreStrategy {
         if (CollectionUtils.isEmpty(param.getContents())) {
             return scores;
         }
-        List<AccountCategory> accountCategoryList = accountCategoryRepository.getByGhIdAndStatus(param.getGhId(), StatusEnum.ONE.getCode());
+        List<AccountCategory> accountCategoryList = accountCategoryRepository.getByGhIdAndStatusAndVersion(
+                param.getGhId(), StatusEnum.ONE.getCode(), activeVersion);
         if (CollectionUtils.isEmpty(accountCategoryList)) {
             return scores;
         }

long-article-recommend-service/src/main/resources/mapper/longArticle/ArticleCategoryMapper.xml

@@ -5,11 +5,11 @@
     <insert id="batchInsertArticleCategory">
         INSERT INTO article_category
         (produce_content_id, channel_content_id, crawler_plan_id, title, title_md5, category,
-        kimi_result, status, create_timestamp)
+        kimi_result, status, create_timestamp, version)
         VALUES
         <foreach collection="list" item="item" separator=",">
             (#{item.produceContentId}, #{item.channelContentId}, #{item.crawlerPlanId}, #{item.title}, #{item.titleMd5},
-            #{item.category}, #{item.kimiResult}, #{item.status}, #{item.createTimestamp})
+            #{item.category}, #{item.kimiResult}, #{item.status}, #{item.createTimestamp}, #{item.version})
         </foreach>
     </insert>
 

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/ArticleVideoAuditTest.java

@@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleCategoryStatusEnum;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.ArticleAuditMapper;
+import com.tzld.longarticle.recommend.server.mapper.longArticle.ArticleCategoryMapper;
 import com.tzld.longarticle.recommend.server.model.dto.kimi.KimiResult;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.ProducePlan;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.ProducePlanExeRecord;
@@ -15,9 +16,9 @@ import com.tzld.longarticle.recommend.server.repository.aigc.ProducePlanExeRecor
 import com.tzld.longarticle.recommend.server.repository.aigc.ProducePlanRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleCategoryRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.LongArticleTitleAuditRepository;
-import com.tzld.longarticle.recommend.server.service.recommend.ArticleCategoryService;
 import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
@@ -32,6 +33,7 @@ import org.springframework.util.StringUtils;
 import javax.annotation.Resource;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -50,7 +52,7 @@ public class ArticleVideoAuditTest {
     @Resource
     private ProducePlanRepository producePlanRepository;
     @Resource
-    private ArticleCategoryService articleCategoryService;
+    private ArticleCategoryMapper articleCategoryMapper;
     @Autowired
     ArticleCategoryRepository articleCategoryRepository;
     @Autowired
@@ -177,4 +179,41 @@ public class ArticleVideoAuditTest {
         }
         return prompt.toString();
     }
+
+    @Test
+    public void articleCategoryAdd() {
+        List<ArticleCategory> oldList = articleCategoryRepository.getByStatusAndVersion(
+                ArticleCategoryStatusEnum.SUCCESS.getCode(), 1);
+        Map<String, ArticleCategory> oldMap = oldList.stream()
+                .filter(o -> !o.getProduceContentId().startsWith("video"))
+                .collect(Collectors.toMap(ArticleCategory::getProduceContentId, o -> o));
+        List<String> oldIds = oldList.stream().map(ArticleCategory::getProduceContentId)
+                .filter(o -> !o.startsWith("video"))
+                .collect(Collectors.toList());
+        List<ArticleCategory> newList = articleCategoryRepository.getByStatusAndVersion(
+                ArticleCategoryStatusEnum.SUCCESS.getCode(), 2);
+        List<String> newIds = newList.stream().map(ArticleCategory::getProduceContentId)
+               .filter(o ->!o.startsWith("video"))
+               .collect(Collectors.toList());
+        List<ArticleCategory> saveList = new ArrayList<>();
+        for (String oldId : oldIds) {
+            if (newIds.contains(oldId)) {
+                continue;
+            }
+            ArticleCategory articleCategory = oldMap.get(oldId);
+            articleCategory.setVersion(2);
+            articleCategory.setKimiResult(null);
+            articleCategory.setId(null);
+            articleCategory.setStatus(0);
+            articleCategory.setCategory(null);
+            articleCategory.setFailReason(null);
+            articleCategory.setRetryTimes(null);
+            articleCategory.setCreateTimestamp(System.currentTimeMillis());
+            saveList.add(articleCategory);
+        }
+
+        if (CollectionUtils.isNotEmpty(saveList)) {
+            articleCategoryMapper.batchInsertArticleCategory(saveList);
+        }
+    }
 }

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/DataDashboardTest.java

@@ -1,8 +1,14 @@
 package com.tzld.longarticle.recommend.server;
 
 import com.alibaba.fastjson.JSONObject;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleCategoryStatusEnum;
+import com.tzld.longarticle.recommend.server.mapper.longArticle.ArticleCategoryMapper;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatScore;
 import com.tzld.longarticle.recommend.server.model.vo.IntermediateIndicatorsExport;
 import com.tzld.longarticle.recommend.server.model.vo.NewSortStrategyExport;
+import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleCategoryRepository;
+import com.tzld.longarticle.recommend.server.repository.longArticle.DatastatScoreRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.DataDashboardService;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
 import com.tzld.longarticle.recommend.server.util.feishu.FeiShu;
@@ -14,6 +20,9 @@ import org.springframework.http.HttpHeaders;
 
 import javax.annotation.Resource;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 @SpringBootTest(classes = Application.class)
 @Slf4j
@@ -21,6 +30,12 @@ public class DataDashboardTest {
 
     @Resource
     DataDashboardService dataDashboardService;
+    @Resource
+    DatastatScoreRepository datastatScoreRepository;
+    @Resource
+    ArticleCategoryRepository articleCategoryRepository;
+    @Resource
+    ArticleCategoryMapper articleCategoryMapper;
 
     @Test
     public void test() {
@@ -63,4 +78,33 @@ public class DataDashboardTest {
         dataDashboardService.contentGroupFunnelExport(dateStrList);
     }
 
+    @Test
+    public void datastatScore() {
+        List<DatastatScore> scoreList = datastatScoreRepository.getBySourceIdIsNotNull();
+        List<ArticleCategory> categoryList = articleCategoryRepository.getByStatusAndVersion(ArticleCategoryStatusEnum.SUCCESS.getCode(), 2);
+        Map<String, ArticleCategory> categoryMap = categoryList.stream().collect(Collectors.toMap(ArticleCategory::getProduceContentId, category -> category));
+        Map<String, ArticleCategory> titleMap = categoryList.stream().collect(Collectors.toMap(ArticleCategory::getTitle, category -> category, (a, b) -> a));
+        Map<String, List<DatastatScore>> scoreMap = scoreList.stream().collect(Collectors.groupingBy(DatastatScore::getTitle));
+        for (Map.Entry<String, List<DatastatScore>> entry : scoreMap.entrySet()) {
+            String title = entry.getKey();
+            List<DatastatScore> list = entry.getValue();
+            ArticleCategory category = null;
+            for (DatastatScore score : list) {
+                category = categoryMap.get(score.getSourceId());
+                if (Objects.nonNull(category)) {
+                    category = titleMap.get(score.getTitle());
+                    break;
+                }
+            }
+            if (Objects.isNull(category)) {
+                category = titleMap.get(title);
+            }
+            if (Objects.isNull(category)) {
+                log.error("category not exists");
+                continue;
+            }
+            articleCategoryMapper.updateArticleCategory(title, category.getCategory());
+        }
+    }
+
 }