Merge branch 'wyp/0108-safeReduceV2' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -34,4 +34,5 @@ public interface AigcBaseMapper {
 
     List<ProduceTaskAtom> getProduceScoreGreaterThan(Long startTimeStamp, Long endTimeStamp);
 
+    List<ProduceTaskAtom> getProduceScoreByContentId(List<String> planExeIds);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/Content.java

@@ -22,6 +22,7 @@ public class Content {
     private String sourceId;
     private String title;
     private String titleMd5;
+    private String kimiSafeScore;
     private Long createTimestamp;
     private String producePlanName;
     private String contentPoolType; // 内容池类别

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleTitleHisCache.java

@@ -27,6 +27,8 @@ public class ArticleTitleHisCache implements Serializable {
     private String titleMd5;
     @Column(name = "crawler_title")
     private String crawlerTitle;
+    @Column(name = "kimi_safe_score")
+    private String kimiSafeScore;
     @Column(name = "channel_content_id")
     private String channelContentId;
     @Column(name = "root_publish_timestamp")

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/XxlJobService.java

@@ -716,7 +716,7 @@ public class XxlJobService {
     public ReturnT<String> refreshArticleHisCache(String param) {
         // 刷新历史表现缓存
         long count = articleTitleHisCacheRepository.count();
-        int pageSize = 100;
+        int pageSize = 1000;
         long page = (count / pageSize) + 1;
         for (int i = 0; i < page; i++) {
             Page<ArticleTitleHisCache> articleTitleHisCachePage = articleTitleHisCacheRepository.findAll(
@@ -754,6 +754,7 @@ public class XxlJobService {
                         if (CollectionUtil.isNotEmpty(content.getCategory())) {
                             cache.setCategory(JSONObject.toJSONString(content.getCategory()));
                         }
+                        cache.setKimiSafeScore(content.getKimiSafeScore());
                         cache.setUpdateTimestamp(System.currentTimeMillis());
                         articleTitleHisCacheRepository.save(cache);
                     }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/RankService.java

@@ -34,13 +34,18 @@ import static com.tzld.longarticle.recommend.server.common.constant.SceneConstan
 public class RankService implements InitializingBean {
 
     private static Map<String, Map<String, Integer>> staticStrategyPoolSourceTypeMap;
+    private static Map<String, Double> staticSafeReduceConfig;
 
     @ApolloJsonValue("${strategyPoolSourceTypeConfig:{}}")
     private Map<String, Map<String, Integer>> strategyPoolSourceTypeMap;
 
+    @ApolloJsonValue("${reduce.safe.config:{}}")
+    private Map<String, Double> safeReduceConfig;
+
     @Override
     public void afterPropertiesSet() throws Exception {
         RankService.staticStrategyPoolSourceTypeMap = strategyPoolSourceTypeMap;
+        RankService.staticSafeReduceConfig = safeReduceConfig;
     }
 
     @ApolloConfigChangeListener(interestedKeys = {"strategyPoolSourceTypeConfig"})
@@ -51,6 +56,14 @@ public class RankService implements InitializingBean {
                 JSONObject.toJSONString(staticStrategyPoolSourceTypeMap));
     }
 
+    @ApolloConfigChangeListener(interestedKeys = {"reduce.safe.config"})
+    public void configChangeSafeReduce(ConfigChangeEvent changeEvent) {
+        RankService.staticSafeReduceConfig = JSONObject.parseObject(changeEvent.getChange(
+                "reduce.safe.config").getNewValue(), Map.class);
+        log.info("reduce.safe.config change updateStaticValue success newValue:{}",
+                JSONObject.toJSONString(staticSafeReduceConfig));
+    }
+
     public static Integer getStrategyPoolSourceType(String strategy, Integer index) {
         Integer sourceType = 0;
         Map<String, Integer> indexSourceTypeMap = staticStrategyPoolSourceTypeMap.get(strategy);
@@ -168,4 +181,16 @@ public class RankService implements InitializingBean {
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
         }
     }
+
+    public static void safeScoreDecrease(List<RankItem> items) {
+        for (RankItem item : items) {
+            Content content = item.getContent();
+            if (Objects.nonNull(content.getKimiSafeScore())) {
+                Double safeScore = staticSafeReduceConfig.get(content.getKimiSafeScore());
+                if (Objects.nonNull(safeScore)) {
+                    content.setScore(content.getScore() - safeScore);
+                }
+            }
+        }
+    }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/FwhColdStartRankStrategy.java

@@ -4,10 +4,7 @@ package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
@@ -64,6 +61,8 @@ public class FwhColdStartRankStrategy implements RankStrategy {
             item.setScore(score);
             return item;
         });
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/HisJumpRankStrategy.java

@@ -69,6 +69,8 @@ public class HisJumpRankStrategy implements RankStrategy {
             item.setScore(score);
             return item;
         });
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/InfiniteRankStrategy.java

@@ -59,6 +59,8 @@ public class InfiniteRankStrategy implements RankStrategy {
             item.setScore(score);
             return item;
         });
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/LateRankStrategy.java

@@ -59,6 +59,8 @@ public class LateRankStrategy implements RankStrategy {
             item.setScore(score);
             return item;
         });
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV10Strategy.java

@@ -84,6 +84,8 @@ public class RankV10Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV11Strategy.java

@@ -104,6 +104,8 @@ public class RankV11Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV12Strategy.java

@@ -104,6 +104,8 @@ public class RankV12Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV13Strategy.java

@@ -105,6 +105,8 @@ public class RankV13Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV14Strategy.java

@@ -105,6 +105,8 @@ public class RankV14Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV15Strategy.java

@@ -94,6 +94,8 @@ public class RankV15Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV3Strategy.java

@@ -76,6 +76,8 @@ public class RankV3Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV4Strategy.java

@@ -74,6 +74,8 @@ public class RankV4Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV5Strategy.java

@@ -92,6 +92,8 @@ public class RankV5Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV7Strategy.java

@@ -77,6 +77,8 @@ public class RankV7Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV8Strategy.java

@@ -88,6 +88,8 @@ public class RankV8Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV9Strategy.java

@@ -76,6 +76,8 @@ public class RankV9Strategy implements RankStrategy {
         if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
             return new RankResult(result);
         }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
 
         // 1 排序
         Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -13,12 +13,14 @@ import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
 import com.tzld.longarticle.recommend.server.common.enums.longArticle.ArticleVideoAuditStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.*;
+import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.ArticleMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.dto.ContentHisPublishArticle;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.CrawlerMetaArticle;
+import com.tzld.longarticle.recommend.server.model.entity.aigc.ProduceTaskAtom;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishContent;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountCorrelation;
@@ -105,6 +107,8 @@ public class RecallService implements ApplicationContextAware {
     LongArticlesTextRepository longArticlesTextRepository;
     @Autowired
     LongArticleTitleAuditRepository titleAuditRepository;
+    @Autowired
+    AigcBaseMapper aigcBaseMapper;
 
     private final Map<String, RecallStrategy> strategyMap = new HashMap<>();
     private ApplicationContext applicationContext;
@@ -346,6 +350,7 @@ public class RecallService implements ApplicationContextAware {
                 if (StringUtils.hasText(cache.getCategory())) {
                     content.setCategory(JSONArray.parseArray(cache.getCategory(), String.class));
                 }
+                content.setKimiSafeScore(cache.getKimiSafeScore());
                 content.setRootPublishTimestamp(cache.getRootPublishTimestamp());
                 content.setHisPublishArticleList(hisPublishArticleList);
                 setT0Data(content);
@@ -357,6 +362,7 @@ public class RecallService implements ApplicationContextAware {
                 if (CollectionUtils.isNotEmpty(articleWithHistory.getCategory())) {
                     content.setCategory(articleWithHistory.getCategory());
                 }
+                content.setKimiSafeScore(articleWithHistory.getKimiSafeScore());
                 content.setRootPublishTimestamp(articleWithHistory.getRootPublishTimestamp());
                 for (ContentHisPublishArticle article : content.getHisPublishArticleList()) {
                     article.setCorrelation(Optional.ofNullable(accountCorrelationMap.get(article.getGhId())).orElse(0.0));
@@ -414,6 +420,7 @@ public class RecallService implements ApplicationContextAware {
     public Map<String, Content> getArticleWithHistory(List<TitleHisCacheParam> paramList, String type) {
         Map<String, Content> result = new HashMap<>();
         List<String> titleMd5List = paramList.stream().map(TitleHisCacheParam::getTitleMd5).collect(Collectors.toList());
+        List<String> sourceIds = paramList.stream().map(TitleHisCacheParam::getSourceId).collect(Collectors.toList());
         // 获取历史已发布文章
         List<Article> hisArticleList = new ArrayList<>();
         List<List<String>> titleMd5Partition = Lists.partition(new ArrayList<>(titleMd5List), 1000);
@@ -481,6 +488,10 @@ public class RecallService implements ApplicationContextAware {
                 .collect(Collectors.toMap(ArticleCategory::getChannelContentId, Function.identity(), (a, b) -> a));
         Map<String, ArticleCategory> titleCategoryMap = articleCategoryList.stream()
                 .collect(Collectors.toMap(ArticleCategory::getTitleMd5, Function.identity(), (a, b) -> a));
+        // 根据sourceId查询kimiSafeScore
+        List<ProduceTaskAtom> safeScoreList = aigcBaseMapper.getProduceScoreByContentId(sourceIds);
+        Map<String, String> safeScoreMap = safeScoreList.stream()
+               .collect(Collectors.toMap(ProduceTaskAtom::getPlanExeId, ProduceTaskAtom::getOutput));
 
         for (TitleHisCacheParam cacheParam : paramList) {
             Content res = new Content();
@@ -495,6 +506,14 @@ public class RecallService implements ApplicationContextAware {
             if (Objects.nonNull(category)) {
                 res.setCategory(Collections.singletonList(category.getCategory()));
             }
+            // 设置kimiSafeScore
+            String safeScore = safeScoreMap.get(cacheParam.getSourceId());
+            if (StringUtils.hasText(safeScore)) {
+                safeScore = safeScore.replaceAll("\n","");
+                if (safeScore.length() == 1) {
+                    res.setKimiSafeScore(safeScore);
+                }
+            }
             // 溯源查找源发布时间
             ArticlePoolPromotionSource source = sourceMap.get(cacheParam.getCrawlerChannelContentId());
             if (Objects.nonNull(source) && Objects.nonNull(source.getRootProduceContentId())) {

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -102,9 +102,8 @@
             resultType="com.tzld.longarticle.recommend.server.model.entity.aigc.ProduceTaskAtom">
         select atom.plan_exe_id, atom.input, atom.output, record.audit_timestamp as createTimestamp
         from produce_task_atom atom
-         join produce_plan_module_task_exe_record exe on exe.task_exe_id = atom.task_exe_id
-         join produce_plan_exe_record record on exe.plan_exe_id = record.plan_exe_id
-         join produce_plan_module_task task on task.task_id = exe.task_id
+         join produce_plan_exe_record record on atom.plan_exe_id = record.plan_exe_id
+         join produce_plan_module_task task on task.task_id = atom.task_id
         where task.name in ('title 安全分', 'titile 安全分')
           and record.audit_timestamp between #{startTimeStamp} and #{endTimeStamp}
           and atom.output >= 7
@@ -113,4 +112,20 @@
         order by record.audit_timestamp desc
     </select>
 
+    <select id="getProduceScoreByContentId"
+            resultType="com.tzld.longarticle.recommend.server.model.entity.aigc.ProduceTaskAtom">
+        select atom.plan_exe_id, atom.input, atom.output, record.audit_timestamp as createTimestamp
+        from produce_task_atom atom
+        join produce_plan_exe_record record on atom.plan_exe_id = record.plan_exe_id
+        join produce_plan_module_task task on task.task_id = atom.task_id
+        where task.name in ('title 安全分', 'titile 安全分')
+        and record.plan_exe_id in
+        <foreach collection="planExeIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and record.status in (2, 6, 9)
+        and record.audit_status in (1, 3)
+        order by record.audit_timestamp desc
+    </select>
+
 </mapper>

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -235,13 +235,13 @@
 
     <insert id="batchInsertArticleTitleHisCache">
         insert into article_title_his_cache
-        (source_id, type, title, title_md5, channel_content_id, root_publish_timestamp, crawler_title,
+        (source_id, type, title, title_md5, kimi_safe_score, channel_content_id, root_publish_timestamp, crawler_title,
          category, his_publish_article_list, create_timestamp)
         values
         <foreach collection="list" item="item" separator=",">
-            (#{item.sourceId}, #{item.type}, #{item.title}, #{item.titleMd5}, #{item.channelContentId},
-             #{item.rootPublishTimestamp}, #{item.crawlerTitle}, #{item.category}, #{item.hisPublishArticleList},
-             #{item.createTimestamp})
+            (#{item.sourceId}, #{item.type}, #{item.title}, #{item.titleMd5}, #{item.kimiSafeScore},
+             #{item.channelContentId}, #{item.rootPublishTimestamp}, #{item.crawlerTitle}, #{item.category},
+             #{item.hisPublishArticleList}, #{item.createTimestamp})
         </foreach>
     </insert>