Merge branch 'wyp/0226-topA' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/aigc/ProducePlanInputSourceTypeEnum.java

@@ -0,0 +1,54 @@
+package com.tzld.longarticle.recommend.server.common.enums.aigc;
+
+import lombok.Getter;
+
+import java.util.Objects;
+
+@Getter
+public enum ProducePlanInputSourceTypeEnum {
+    contentID(1, "内容ID"),
+    contentPlan(2, "内容计划"),
+    imageElementTag(3, "图片元素标签"),
+    manMadeImageElementTag(4, "人为图片元素标签"),
+    contentTag(5, "内容标签"),
+    manMadeContentTag(6, "人为内容标签"),
+    videoUploadMaterial(7, "视频上传素材"),
+    manMadeTag(8, "人为标签"),
+    customizeField(9, "输出自定义字段"),
+    point_pretreatPlan(10, "内容点处理计划"),
+    keyPointId(11, "内容要点ID"),
+    hotPointId(12, "平台热点ID"),
+    keyPointType(13, "要点类型"),
+    keyPointStrategy(14, "要点策略"),
+    hotPointStrategy(15, "热点策略"),
+    hotPointType(16, "热点类型"),
+    accountRole(17, "账号人设"),
+    aiModelRobot(18, "数字人分身"),
+    aiModelTts(19, "声音包"),
+    musicCategory(20, "音乐分类"),
+    textToMusic(21, "文字转语音"),
+    musicReplace(22, "音频音色替换"),
+    textImageTemplate(23, "文字配图模板"),
+    textImageTemplateName(24, "文字配图模板名称"),
+    textImageTemplateTag(25, "文字配图模板标签"),
+    aiModelBg(26, "数字人背景"),
+    other(999, ""),
+    ;
+
+    private final Integer val;
+    private final String description;
+
+    ProducePlanInputSourceTypeEnum(Integer val, String description) {
+        this.val = val;
+        this.description = description;
+    }
+
+    public static ProducePlanInputSourceTypeEnum from(Integer val) {
+        for (ProducePlanInputSourceTypeEnum typesEnum : ProducePlanInputSourceTypeEnum.values()) {
+            if (Objects.equals(typesEnum.val, val)) {
+                return typesEnum;
+            }
+        }
+        return null;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/RankStrategyEnum.java

@@ -21,6 +21,7 @@ public enum RankStrategyEnum {
     ArticleRankV14("ArticleRankV14", "ArticleRankV14", "rankV14Strategy"),
 
     ArticleRankV15("ArticleRankV15", "ArticleRankV15", "rankV15Strategy"),
+    ArticleRankV16("ArticleRankV16", "ArticleRankV16", "rankV16Strategy"),
 
     HIS_JUMP_STRATEGY("ArticleRankHisJump", "历史表现跳过相似度策略", "hisJumpRankStrategy"),
     INFINITE_STRATEGY("ArticleRankInfinite", "无限发表", "infiniteRankStrategy"),

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -63,4 +63,6 @@ public interface AigcBaseMapper {
     List<ProduceContentDTO> getProduceContentById(List<String> planExeIds);
 
     List<IdNameVO<String>> articleVideoAuditPlanFilterValue(List<String> planIds, String searchKeyword);
+
+    List<String> getProduceContentTitleByPlanId(String planId);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -83,4 +83,6 @@ public interface LongArticleBaseMapper {
     void batchInsertLongArticleAuditDelete(List<LongArticleAuditDelete> deleteList);
 
     List<String> getPassContentIds();
+
+    List<DatastatSortStrategy> getTopContent(String dateStr);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/Content.java

@@ -24,6 +24,7 @@ public class Content {
     private String titleMd5;
     private String kimiSafeScore;
     private Long createTimestamp;
+    private String producePlanId;
     private String producePlanName;
     private String contentPoolType; // 内容池类别
     private String crawlerChannelContentId; // 抓取内容channelContentId

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/CrawlerContent.java

@@ -4,6 +4,7 @@ import lombok.Data;
 
 @Data
 public class CrawlerContent {
+    private Long id;
     private String channelContentId;
     private String crawlerPlanId;
     private String ghId;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleTitleHisCache.java

@@ -25,6 +25,8 @@ public class ArticleTitleHisCache implements Serializable {
     private String title;
     @Column(name = "title_md5")
     private String titleMd5;
+    @Column(name = "plan_id")
+    private String planId;
     @Column(name = "crawler_title")
     private String crawlerTitle;
     @Column(name = "kimi_safe_score")

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/aigc/AIGCProducePlanSaveService.java

@@ -28,14 +28,14 @@ public class AIGCProducePlanSaveService {
 
     private static String apiUrl = "http://aigc-api.cybertogether.net/aigc/produce/plan/save";
 
-    public IdNameVO<String> save(String planName, String planId, ProducePlanDetailVO detail) {
+    public IdNameVO<String> save(String inputSourceLabel, String inputSourceValue, ProducePlanDetailVO detail, Integer inputSourceType) {
         // 添加新的依赖计划
         List<ProducePlanInputSourceParam> inputSources = detail.getInputSourceGroups().get(0).getInputSources();
         ProducePlanInputSourceParam newInputSource = new ProducePlanInputSourceParam();
         newInputSource.setContentType(1);
-        newInputSource.setInputSourceType(2);
-        newInputSource.setInputSourceValue(planId);
-        newInputSource.setInputSourceLabel(String.format("原始帖子-长文-微信公众号-内容添加计划-%s", planName));
+        newInputSource.setInputSourceType(inputSourceType);
+        newInputSource.setInputSourceValue(inputSourceValue);
+        newInputSource.setInputSourceLabel(inputSourceLabel);
         newInputSource.setInputSourceModal(3);
         newInputSource.setInputSourceChannel(5);
         inputSources.add(newInputSource);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/XxlJobService.java

@@ -809,6 +809,7 @@ public class XxlJobService {
                         cache.setCategory(JSONObject.toJSONString(content.getCategory()));
                     }
                     cache.setKimiSafeScore(content.getKimiSafeScore());
+                    cache.setPlanId(content.getProducePlanId());
                     cache.setUpdateTimestamp(System.currentTimeMillis());
                     articleTitleHisCacheRepository.save(cache);
                 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticlePromotionService.java

@@ -4,15 +4,21 @@ import cn.hutool.core.lang.UUID;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.CrawlerModeEnum;
+import com.tzld.longarticle.recommend.server.common.enums.aigc.ProducePlanInputSourceTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticlePoolPromotionSourceStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
+import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
+import com.tzld.longarticle.recommend.server.mapper.aigc.PublishContentMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
+import com.tzld.longarticle.recommend.server.model.dto.CrawlerContent;
+import com.tzld.longarticle.recommend.server.model.dto.PublishContentDTO;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.PublishSortLog;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticlePoolPromotionSource;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatSortStrategy;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource;
+import com.tzld.longarticle.recommend.server.model.param.PublishContentParam;
 import com.tzld.longarticle.recommend.server.model.vo.IdNameVO;
 import com.tzld.longarticle.recommend.server.model.vo.WxContentDetailResponse;
 import com.tzld.longarticle.recommend.server.model.vo.aigc.CommonListDataVO;
@@ -33,10 +39,13 @@ import com.tzld.longarticle.recommend.server.util.DateUtils;
 import com.tzld.longarticle.recommend.server.util.Md5Util;
 import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import com.tzld.longarticle.recommend.server.util.feishu.FeishuMessageSender;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
@@ -72,9 +81,15 @@ public class ArticlePromotionService {
     PublishSortLogRepository publishSortLogRepository;
     @Autowired
     PublishSingleVideoSourceRepository publishSingleVideoSourceRepository;
+    @Autowired
+    PublishContentMapper publishContentMapper;
+    @Autowired
+    AigcBaseMapper aigcBaseMapper;
 
     @ApolloJsonValue("${articlePromotionProduceConfig:{}}")
     private Map<String, Map<String, Map<String, String>>> produceConfig;
+    @Value("${topProducePlanId:}")
+    private String topProducePlanId;
 
     private final List<String> contentPoolType = Arrays.asList("autoArticlePoolLevel1", "autoArticlePoolLevel3", "autoArticlePoolLevel4");
 
@@ -180,7 +195,8 @@ public class ArticlePromotionService {
                 log.info("url_len: " + list.size() + ", " + filterUrlList.size());
                 IdNameVO<String> planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, filterUrlList, tag, CrawlerModeEnum.ContentIDs.getVal());
                 if (StringUtils.hasText(produceId)) {
-                    articleAddDependPlan(produceId, planInfo.getId(), planInfo.getName());
+                    String inputSourceLabel = String.format("原始帖子-长文-微信公众号-内容添加计划-%s", planInfo.getName());
+                    articleAddDependPlan(produceId, planInfo.getId(), inputSourceLabel, ProducePlanInputSourceTypeEnum.contentPlan.getVal());
                 }
                 log.info("{}, {}, produce plan not exist: {}, {}, {}", planInfo.getName(), planInfo.getId(), accountNickName, pos, way);
             }
@@ -188,7 +204,8 @@ public class ArticlePromotionService {
                 String planName = String.format("%d_%s_%s_%s【%s】_%s", publishContentIds.size(), today, accountNickName, pos, way, today);
                 IdNameVO<String> planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, publishContentIds, tag, CrawlerModeEnum.PublishContentIds.getVal());
                 if (StringUtils.hasText(produceId)) {
-                    articleAddDependPlan(produceId, planInfo.getId(), planInfo.getName());
+                    String inputSourceLabel = String.format("原始帖子-长文-微信公众号-内容添加计划-%s", planInfo.getName());
+                    articleAddDependPlan(produceId, planInfo.getId(), inputSourceLabel, ProducePlanInputSourceTypeEnum.contentPlan.getVal());
                 }
             }
             sendFeishuJobFinishMessage(accountNickName, filterUrlList.size(), publishContentIds.size());
@@ -342,7 +359,7 @@ public class ArticlePromotionService {
         articlePoolPromotionSourceRepository.save(articlePromotion);
     }
 
-    private void articleAddDependPlan(String produceId, String planId, String planName) {
+    private void articleAddDependPlan(String produceId, String inputSourceValue, String inputSourceLabel, Integer inputSourceType) {
         try {
             // 获取生产计划的详细信息
             ProducePlanDetailVO detail = aigcProducePlanDetailService.articleGetProducePlanDetail(produceId);
@@ -354,16 +371,16 @@ public class ArticlePromotionService {
             }
             // 获取依赖计划 ID 列表
             List<ProducePlanInputSourceParam> inputSources = detail.getInputSourceGroups().get(0).getInputSources();
-            List<String> dependPlanIds = new ArrayList<>();
+            List<String> dependValues = new ArrayList<>();
             for (ProducePlanInputSourceParam inputSource : inputSources) {
-                dependPlanIds.add(inputSource.getInputSourceValue());
+                dependValues.add(inputSource.getInputSourceValue());
             }
             // 如果计划 ID 已存在，直接返回
-            if (dependPlanIds.contains(planId)) {
-                log.info("depend_plan_id exist: {}", planId);
+            if (dependValues.contains(inputSourceValue)) {
+                log.info("depend_value exist: {}", inputSourceValue);
                 return;
             }
-            aigcProducePlanSaveService.save(planName, planId, detail);
+            aigcProducePlanSaveService.save(inputSourceLabel, inputSourceValue, detail, inputSourceType);
         } catch (Exception e) {
             FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.JOB.getRobotId(),
                     "文章晋升ERROR:\n" +
@@ -431,4 +448,43 @@ public class ArticlePromotionService {
         }
     }
 
+    @XxlJob("topContentReSendJob")
+    public ReturnT<String> topContentReSend(String param) {
+        String dateStr = DateUtils.getBeforeDaysDateStr("yyyyMMdd", 60);
+        if (StringUtils.hasText(param)) {
+            dateStr = param;
+        }
+        List<DatastatSortStrategy> topList = longArticleBaseMapper.getTopContent(dateStr);
+        List<String> ghIds = topList.stream().map(DatastatSortStrategy::getGhId).distinct().collect(Collectors.toList());
+        List<PublishAccount> publishAccountList = publishAccountRepository.getAllByGhIdIn(ghIds);
+        Map<String, String> publishContentMap = publishAccountList.stream().collect(Collectors.toMap(
+                PublishAccount::getGhId, PublishAccount::getId));
+        List<PublishContentParam> publishContentParamList = topList.stream().map(top -> {
+            PublishContentParam item = new PublishContentParam();
+            item.setTitle(top.getTitle());
+            String accountId = publishContentMap.get(top.getGhId());
+            item.setPublishAccountId(accountId);
+            return item;
+        }).collect(Collectors.toList());
+        List<PublishContentDTO> publishContentList = publishContentMapper.getPublishContentByTitle(publishContentParamList);
+        List<String> existsTitles = aigcBaseMapper.getProduceContentTitleByPlanId(topProducePlanId);
+        List<String> channelContentIds = publishContentList.stream().map(PublishContentDTO::getCrawlerChannelContentId).collect(Collectors.toList());
+        List<CrawlerContent> crawlerContentList = aigcBaseMapper.getCrawlerContentByChannelContentIdIn(channelContentIds);
+        Map<String, Long> crawlerContentMap = crawlerContentList.stream().collect(Collectors.toMap(
+                CrawlerContent::getChannelContentId, CrawlerContent::getId));
+        for (PublishContentDTO item : publishContentList) {
+            if (TitleSimilarCheckUtil.isDuplicateContent(item.getTitle(), existsTitles, TitleSimilarCheckUtil.ARTICLE_PROMOTION_THRESHOLD)) {
+                continue;
+            }
+            existsTitles.add(item.getTitle());
+            String inputSourceLabel = String.format("原始帖子-长文-微信公众号-内容IDs-%s", item.getTitle());
+            Long crawlerContentId = crawlerContentMap.get(item.getCrawlerChannelContentId());
+            if (Objects.isNull(crawlerContentId)) {
+                continue;
+            }
+            articleAddDependPlan(topProducePlanId, String.valueOf(crawlerContentId), inputSourceLabel,
+                    ProducePlanInputSourceTypeEnum.contentID.getVal());
+        }
+        return ReturnT.SUCCESS;
+    }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/HistoryTitleStrategy.java

@@ -2,23 +2,21 @@ package com.tzld.longarticle.recommend.server.service.recommend.filter.strategy;
 
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
-import com.tzld.longarticle.recommend.server.remote.ArticleListRemoteService;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
+import com.tzld.longarticle.recommend.server.remote.ArticleListRemoteService;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterStrategy;
+import com.tzld.longarticle.recommend.server.util.DateUtils;
 import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
-import org.springframework.util.StringUtils;
 
 import java.util.*;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
 import java.util.stream.Collectors;
 
 /**
@@ -45,8 +43,10 @@ public class HistoryTitleStrategy implements FilterStrategy {
         List<Content> filterContents = new ArrayList<>();
         List<Article> allArticleList = articleListRemoteService.articleList(param.getGhId(), allIndex, param.getType());
         List<String> allTitleList = allArticleList.stream().map(Article::getTitle).distinct().collect(Collectors.toList());
+        Long publishTimestampFilter = DateUtils.getBeforeDayStart(60);
         List<String> firstSecondTitleList = allArticleList.stream()
-                .filter(article -> firstSecondIndex.contains(article.getItemIndex()))
+                .filter(article -> firstSecondIndex.contains(article.getItemIndex())
+                        && article.getPublishTimestamp() > publishTimestampFilter)
                 .map(Article::getTitle).distinct().collect(Collectors.toList());
 
         List<String> firstSecondContentPool = new ArrayList<>();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV15Strategy.java

@@ -131,17 +131,19 @@ public class RankV15Strategy implements RankStrategy {
         // 3-8
         // RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
         List<Content> pool = contentMap.get(contentPools[2]);
-        Integer videoSourceType = PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal();
-        Queue<Content> videoPoolQueue = pool.stream().filter(o -> Objects.equals(o.getSourceType(), videoSourceType))
-                .collect(Collectors.toCollection(LinkedList::new));
-        Queue<Content> otherPoolQueue = pool.stream().filter(o -> !Objects.equals(o.getSourceType(), videoSourceType))
-                .collect(Collectors.toCollection(LinkedList::new));
-        for (int i = 3; i < param.getSize() + 1; i++) {
-            Integer sourceType = RankService.getStrategyPoolSourceType(param.getStrategy(), i);
-            if (Objects.equals(sourceType, videoSourceType) && !videoPoolQueue.isEmpty()) {
-                result.add(videoPoolQueue.poll());
-            } else if (!otherPoolQueue.isEmpty()) {
-                result.add(otherPoolQueue.poll());
+        if (CollectionUtils.isNotEmpty(pool)) {
+            Integer videoSourceType = PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal();
+            Queue<Content> videoPoolQueue = pool.stream().filter(o -> Objects.equals(o.getSourceType(), videoSourceType))
+                    .collect(Collectors.toCollection(LinkedList::new));
+            Queue<Content> otherPoolQueue = pool.stream().filter(o -> !Objects.equals(o.getSourceType(), videoSourceType))
+                    .collect(Collectors.toCollection(LinkedList::new));
+            for (int i = 3; i < param.getSize() + 1; i++) {
+                Integer sourceType = RankService.getStrategyPoolSourceType(param.getStrategy(), i);
+                if (Objects.equals(sourceType, videoSourceType) && !videoPoolQueue.isEmpty()) {
+                    result.add(videoPoolQueue.poll());
+                } else if (!otherPoolQueue.isEmpty()) {
+                    result.add(otherPoolQueue.poll());
+                }
             }
         }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV16Strategy.java

@@ -0,0 +1,172 @@
+package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
+
+
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
+import com.tzld.longarticle.recommend.server.model.dto.Content;
+import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
+import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
+import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
+import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
+import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
+import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
+import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Service
+@Slf4j
+public class RankV16Strategy implements RankStrategy {
+
+    @Autowired
+    private ScoreService scoreService;
+    @Autowired
+    private AccountContentPoolConfigService accountContentPoolConfigService;
+    @Autowired
+    private ArticleRepository articleRepository;
+    @Autowired
+    private StrategyIndexScoreWeightService weightService;
+
+    @ApolloJsonValue("${touliu.account.ghIds:[\"gh_93e00e187787\", \"gh_ac43e43b253b\", \"gh_68e7fdc09fe4\",\"gh_77f36c109fb1\", \"gh_b181786a6c8c\", \"gh_1ee2e1b39ccf\"]}")
+    private List<String> touliuAccountGhIds;
+    @Value("${topProducePlanId:}")
+    private String topProducePlanId;
+
+    public RankResult rank(RankParam param) {
+        List<Content> result = new ArrayList<>();
+
+        ScoreResult scoreResult = scoreService.score(RankStrategy.convertToScoreParam(param));
+
+        Map<String, Map<String, Double>> scoreMap = scoreResult.getScoreMap();
+        String[] contentPools = accountContentPoolConfigService.getContentPools(param.getAccountName());
+        Map<Integer, AccountIndexReplacePoolConfig> indexReplacePoolConfigMap = accountContentPoolConfigService.getContentReplacePools(param.getAccountName());
+
+        List<RankItem> items = CommonCollectionUtils.toList(param.getContents(), c -> {
+            RankItem item = new RankItem();
+            item.setContent(c);
+            c.setScoreMap(scoreMap.get(c.getId()));
+            item.setScoreMap(scoreMap.get(c.getId()));
+            double score;
+            int index = weightService.getIndex(item.getContent().getContentPoolType(), contentPools);
+            if (contentPools[0].equals(item.getContent().getContentPoolType())
+                    || contentPools[1].equals(item.getContent().getContentPoolType())) {
+                score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.SIMILARITY.value())
+                        + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
+                        + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value())
+                        + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value());
+                if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
+                    score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
+                            * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                            ScoreStrategyEnum.VIEW_COUNT_RATE.value());
+                }
+            } else {
+                score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.SIMILARITY.value())
+                        + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
+                        + item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
+                        + item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value())
+                        + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value())
+                        + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
+            }
+            c.setScore(score);
+            item.setScore(score);
+            return item;
+        });
+        // 相似度评分为0 报警返回
+        List<Article> hisPublishFirstArticleList = articleRepository.getByGhIdAndItemIndexAndTypeEqualsAndStatusEquals(
+                param.getGhId(), 1, param.getType(), 1);
+        if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
+            return new RankResult(result);
+        }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
+
+        // 1 排序
+        Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
+        // 2 相似去重
+        List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
+
+        // 3 文章按照内容池分组
+        Map<String, List<Content>> contentMap = new HashMap<>();
+        for (Content c : contents) {
+            List<Content> data = contentMap.computeIfAbsent(c.getContentPoolType(), k -> new ArrayList<>());
+            data.add(c);
+        }
+        // 4 选文章
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
+
+        // 头
+        List<Content> pool1 = contentMap.get(contentPools[0]);
+        if (CollectionUtils.isNotEmpty(pool1)) {
+            pool1 = RankService.contentSourceTypeFilter(param.getStrategy(), pool1, 1);
+        }
+        RankService.printSortLog(param.getStrategy(), param.getAccountName(), "头条", pool1);
+        if (CollectionUtils.isNotEmpty(pool1)) {
+            if (topProducePlanId.equals(pool1.get(0).getProducePlanId())) {
+                int i = RandomUtils.nextInt(0, 2);
+                if (i == 0) {
+                    for (Content content : pool1) {
+                        if (!topProducePlanId.equals(content.getProducePlanId())) {
+                            result.add(content);
+                            break;
+                        }
+                    }
+                }
+            }
+            if (CollectionUtils.isEmpty(result)) {
+                result.add(pool1.get(0));
+            }
+        } else {
+            RankStrategy.sendFeishuFirstPoolEmpty(param, contentPools[0]);
+            return new RankResult(result);
+        }
+
+        // 次
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap,
+                indexReplacePoolConfigMap, param.getStrategy());
+
+        // 3-8
+        // RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
+        List<Content> pool = contentMap.get(contentPools[2]);
+        if (CollectionUtils.isNotEmpty(pool)) {
+            Integer videoSourceType = PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal();
+            Queue<Content> videoPoolQueue = pool.stream().filter(o -> Objects.equals(o.getSourceType(), videoSourceType))
+                    .collect(Collectors.toCollection(LinkedList::new));
+            Queue<Content> otherPoolQueue = pool.stream().filter(o -> !Objects.equals(o.getSourceType(), videoSourceType))
+                    .collect(Collectors.toCollection(LinkedList::new));
+            for (int i = 3; i < param.getSize() + 1; i++) {
+                Integer sourceType = RankService.getStrategyPoolSourceType(param.getStrategy(), i);
+                if (Objects.equals(sourceType, videoSourceType) && !videoPoolQueue.isEmpty()) {
+                    result.add(videoPoolQueue.poll());
+                } else if (!otherPoolQueue.isEmpty()) {
+                    result.add(otherPoolQueue.poll());
+                }
+            }
+        }
+
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
+        return new RankResult(result);
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -2,7 +2,6 @@ package com.tzld.longarticle.recommend.server.service.recommend.recall;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Lists;
@@ -368,6 +367,7 @@ public class RecallService implements ApplicationContextAware {
                 if (StringUtils.hasText(cache.getCategory())) {
                     content.setCategory(JSONArray.parseArray(cache.getCategory(), String.class));
                 }
+                content.setProducePlanId(cache.getPlanId());
                 content.setKimiSafeScore(cache.getKimiSafeScore());
                 content.setRootPublishTimestamp(cache.getRootPublishTimestamp());
                 content.setHisPublishArticleList(hisPublishArticleList);
@@ -410,6 +410,7 @@ public class RecallService implements ApplicationContextAware {
                 ArticleTitleHisCache cache = new ArticleTitleHisCache();
                 BeanUtils.copyProperties(content, cache);
                 cache.setType(type);
+                cache.setPlanId(content.getProducePlanId());
                 cache.setChannelContentId(content.getCrawlerChannelContentId());
                 if (CollectionUtils.isNotEmpty(content.getCategory())) {
                     cache.setCategory(JSONObject.toJSONString(content.getCategory()));
@@ -508,8 +509,8 @@ public class RecallService implements ApplicationContextAware {
                 .collect(Collectors.toMap(ArticleCategory::getTitleMd5, Function.identity(), (a, b) -> a));
         // 根据sourceId查询kimiSafeScore
         List<ProduceTaskAtom> safeScoreList = aigcBaseMapper.getProduceScoreByContentId(sourceIds);
-        Map<String, String> safeScoreMap = safeScoreList.stream().filter(o -> StringUtils.hasText(o.getOutput()))
-               .collect(Collectors.toMap(ProduceTaskAtom::getPlanExeId, ProduceTaskAtom::getOutput));
+        Map<String, ProduceTaskAtom> safeScoreMap = safeScoreList.stream().filter(o -> StringUtils.hasText(o.getOutput()))
+               .collect(Collectors.toMap(ProduceTaskAtom::getPlanExeId, o -> o));
 
         for (TitleHisCacheParam cacheParam : paramList) {
             Content res = new Content();
@@ -525,12 +526,16 @@ public class RecallService implements ApplicationContextAware {
                 res.setCategory(Collections.singletonList(category.getCategory()));
             }
             // 设置kimiSafeScore
-            String safeScore = safeScoreMap.get(cacheParam.getSourceId());
-            if (StringUtils.hasText(safeScore)) {
-                safeScore = safeScore.replaceAll("\n","");
-                if (safeScore.length() == 1) {
-                    res.setKimiSafeScore(safeScore);
+            ProduceTaskAtom atom = safeScoreMap.get(cacheParam.getSourceId());
+            if (Objects.nonNull(atom)) {
+                String safeScore = atom.getOutput();
+                if (StringUtils.hasText(safeScore)) {
+                    safeScore = safeScore.replaceAll("\n", "");
+                    if (safeScore.length() == 1) {
+                        res.setKimiSafeScore(safeScore);
+                    }
                 }
+                res.setProducePlanId(atom.getPlanId());
             }
             // 溯源查找源发布时间
             ArticlePoolPromotionSource source = sourceMap.get(cacheParam.getCrawlerChannelContentId());

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/ScoreService.java

@@ -127,7 +127,8 @@ public class ScoreService implements ApplicationContextAware {
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV12.getStrategy())
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV13.getStrategy())
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV14.getStrategy())
-                || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV15.getStrategy())) {
+                || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV15.getStrategy())
+                || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV16.getStrategy())) {
             strategies.add(strategyMap.get(ScoreStrategyEnum.CATEGORY.value()));
             strategies.add(strategyMap.get(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value()));
             strategies.add(strategyMap.get(ScoreStrategyEnum.FLOW_CTL_DECREASE.value()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/XxlJobController.java

@@ -1,6 +1,7 @@
 package com.tzld.longarticle.recommend.server.web;
 
 import com.tzld.longarticle.recommend.server.service.XxlJobService;
+import com.tzld.longarticle.recommend.server.service.recommend.ArticlePromotionService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -13,6 +14,8 @@ import org.springframework.web.bind.annotation.RestController;
 public class XxlJobController {
     @Autowired
     private XxlJobService service;
+    @Autowired
+    private ArticlePromotionService articlePromotionService;
 
     @GetMapping("/checkPublishPlan")
     public void checkPublishPlan() {
@@ -99,4 +102,9 @@ public class XxlJobController {
         service.badAccountContentReject(null);
     }
 
+    @GetMapping("/topContentReSend")
+    public void topContentReSend() {
+        articlePromotionService.topContentReSend(null);
+    }
+
 }

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -20,7 +20,7 @@
 
     <select id="getCrawlerContentByChannelContentId"
             resultType="com.tzld.longarticle.recommend.server.model.dto.CrawlerContent">
-        select cc.channel_content_id, ca.wx_gh as ghId, cc.title, cc.publish_timestamp
+        select cc.id, cc.channel_content_id, ca.wx_gh as ghId, cc.title, cc.publish_timestamp
         from crawler_content cc
         join crawler_account ca on cc.channel_account_id = ca.channel_account_id
         where cc.channel_content_id = #{channelContentId}
@@ -48,7 +48,7 @@
 
     <select id="getCrawlerContentByChannelContentIdIn"
             resultType="com.tzld.longarticle.recommend.server.model.dto.CrawlerContent">
-        select cc.channel_content_id, cprr.plan_id as crawlerPlanId, ca.wx_gh as ghId, cc.title, cc.publish_timestamp
+        select cc.id, cc.channel_content_id, cprr.plan_id as crawlerPlanId, ca.wx_gh as ghId, cc.title, cc.publish_timestamp
         from crawler_content cc
         join (select channel_source_id, max(plan_id) as plan_id from crawler_plan_result_rel group by channel_source_id) cprr
             on cc.channel_content_id = cprr.channel_source_id
@@ -114,7 +114,7 @@
 
     <select id="getProduceScoreByContentId"
             resultType="com.tzld.longarticle.recommend.server.model.entity.aigc.ProduceTaskAtom">
-        select atom.plan_exe_id, atom.input, atom.output, record.audit_timestamp as createTimestamp
+        select atom.plan_exe_id, atom.plan_id, atom.input, atom.output, record.audit_timestamp as createTimestamp
         from produce_task_atom atom
         join produce_plan_exe_record record on atom.plan_exe_id = record.plan_exe_id
         join produce_plan_module_task task on task.task_id = atom.task_id
@@ -282,4 +282,12 @@
         limit 100
     </select>
 
+    <select id="getProduceContentTitleByPlanId" resultType="java.lang.String">
+        select output.output
+        from produce_plan_exe_record pper
+        join produce_plan_module_output output on pper.plan_exe_id = output.plan_exe_id
+        where pper.plan_id = #{planId} and pper.status = 2
+          and output.produce_module_type = 3
+    </select>
+
 </mapper>

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -297,4 +297,20 @@
         </foreach>
     </insert>
 
+    <select id="getTopContent"
+            resultType="com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatSortStrategy">
+        select *
+        from datastat_sort_strategy
+        where date_str &lt; #{dateStr}
+          and type = 9
+          and read_rate > 1
+          and view_count > 10000
+          and position in (1, 2)
+          and title not in (select title from article_unsafe_title)
+          and first_level is not null
+          and strategy is not null
+        order by read_rate desc
+        limit 200
+    </select>
+
 </mapper>