优质内容发布

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/aigc/ProducePlanInputSourceTypeEnum.java

@@ -0,0 +1,54 @@
+package com.tzld.longarticle.recommend.server.common.enums.aigc;
+
+import lombok.Getter;
+
+import java.util.Objects;
+
+@Getter
+public enum ProducePlanInputSourceTypeEnum {
+    contentID(1, "内容ID"),
+    contentPlan(2, "内容计划"),
+    imageElementTag(3, "图片元素标签"),
+    manMadeImageElementTag(4, "人为图片元素标签"),
+    contentTag(5, "内容标签"),
+    manMadeContentTag(6, "人为内容标签"),
+    videoUploadMaterial(7, "视频上传素材"),
+    manMadeTag(8, "人为标签"),
+    customizeField(9, "输出自定义字段"),
+    point_pretreatPlan(10, "内容点处理计划"),
+    keyPointId(11, "内容要点ID"),
+    hotPointId(12, "平台热点ID"),
+    keyPointType(13, "要点类型"),
+    keyPointStrategy(14, "要点策略"),
+    hotPointStrategy(15, "热点策略"),
+    hotPointType(16, "热点类型"),
+    accountRole(17, "账号人设"),
+    aiModelRobot(18, "数字人分身"),
+    aiModelTts(19, "声音包"),
+    musicCategory(20, "音乐分类"),
+    textToMusic(21, "文字转语音"),
+    musicReplace(22, "音频音色替换"),
+    textImageTemplate(23, "文字配图模板"),
+    textImageTemplateName(24, "文字配图模板名称"),
+    textImageTemplateTag(25, "文字配图模板标签"),
+    aiModelBg(26, "数字人背景"),
+    other(999, ""),
+    ;
+
+    private final Integer val;
+    private final String description;
+
+    ProducePlanInputSourceTypeEnum(Integer val, String description) {
+        this.val = val;
+        this.description = description;
+    }
+
+    public static ProducePlanInputSourceTypeEnum from(Integer val) {
+        for (ProducePlanInputSourceTypeEnum typesEnum : ProducePlanInputSourceTypeEnum.values()) {
+            if (Objects.equals(typesEnum.val, val)) {
+                return typesEnum;
+            }
+        }
+        return null;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/RankStrategyEnum.java

@@ -21,6 +21,7 @@ public enum RankStrategyEnum {
     ArticleRankV14("ArticleRankV14", "ArticleRankV14", "rankV14Strategy"),
 
     ArticleRankV15("ArticleRankV15", "ArticleRankV15", "rankV15Strategy"),
+    ArticleRankV16("ArticleRankV16", "ArticleRankV16", "rankV16Strategy"),
 
     HIS_JUMP_STRATEGY("ArticleRankHisJump", "历史表现跳过相似度策略", "hisJumpRankStrategy"),
     INFINITE_STRATEGY("ArticleRankInfinite", "无限发表", "infiniteRankStrategy"),

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/Content.java

@@ -24,6 +24,7 @@ public class Content {
     private String titleMd5;
     private String kimiSafeScore;
     private Long createTimestamp;
+    private String producePlanId;
     private String producePlanName;
     private String contentPoolType; // 内容池类别
     private String crawlerChannelContentId; // 抓取内容channelContentId

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleTitleHisCache.java

@@ -25,6 +25,8 @@ public class ArticleTitleHisCache implements Serializable {
     private String title;
     @Column(name = "title_md5")
     private String titleMd5;
+    @Column(name = "plan_id")
+    private String planId;
     @Column(name = "crawler_title")
     private String crawlerTitle;
     @Column(name = "kimi_safe_score")

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/HistoryTitleStrategy.java

@@ -2,23 +2,21 @@ package com.tzld.longarticle.recommend.server.service.recommend.filter.strategy;
 
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
-import com.tzld.longarticle.recommend.server.remote.ArticleListRemoteService;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
+import com.tzld.longarticle.recommend.server.remote.ArticleListRemoteService;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterStrategy;
+import com.tzld.longarticle.recommend.server.util.DateUtils;
 import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
-import org.springframework.util.StringUtils;
 
 import java.util.*;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
 import java.util.stream.Collectors;
 
 /**
@@ -45,8 +43,10 @@ public class HistoryTitleStrategy implements FilterStrategy {
         List<Content> filterContents = new ArrayList<>();
         List<Article> allArticleList = articleListRemoteService.articleList(param.getGhId(), allIndex, param.getType());
         List<String> allTitleList = allArticleList.stream().map(Article::getTitle).distinct().collect(Collectors.toList());
+        Long publishTimestampFilter = DateUtils.getBeforeDayStart(60);
         List<String> firstSecondTitleList = allArticleList.stream()
-                .filter(article -> firstSecondIndex.contains(article.getItemIndex()))
+                .filter(article -> firstSecondIndex.contains(article.getItemIndex())
+                        && article.getPublishTimestamp() > publishTimestampFilter)
                 .map(Article::getTitle).distinct().collect(Collectors.toList());
 
         List<String> firstSecondContentPool = new ArrayList<>();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV16Strategy.java

@@ -0,0 +1,170 @@
+package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
+
+
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
+import com.tzld.longarticle.recommend.server.model.dto.Content;
+import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
+import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
+import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
+import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
+import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
+import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
+import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Service
+@Slf4j
+public class RankV16Strategy implements RankStrategy {
+
+    @Autowired
+    private ScoreService scoreService;
+    @Autowired
+    private AccountContentPoolConfigService accountContentPoolConfigService;
+    @Autowired
+    private ArticleRepository articleRepository;
+    @Autowired
+    private StrategyIndexScoreWeightService weightService;
+
+    @ApolloJsonValue("${touliu.account.ghIds:[\"gh_93e00e187787\", \"gh_ac43e43b253b\", \"gh_68e7fdc09fe4\",\"gh_77f36c109fb1\", \"gh_b181786a6c8c\", \"gh_1ee2e1b39ccf\"]}")
+    private List<String> touliuAccountGhIds;
+    @Value("${topProducePlanId:}")
+    private String topProducePlanId;
+
+    public RankResult rank(RankParam param) {
+        List<Content> result = new ArrayList<>();
+
+        ScoreResult scoreResult = scoreService.score(RankStrategy.convertToScoreParam(param));
+
+        Map<String, Map<String, Double>> scoreMap = scoreResult.getScoreMap();
+        String[] contentPools = accountContentPoolConfigService.getContentPools(param.getAccountName());
+        Map<Integer, AccountIndexReplacePoolConfig> indexReplacePoolConfigMap = accountContentPoolConfigService.getContentReplacePools(param.getAccountName());
+
+        List<RankItem> items = CommonCollectionUtils.toList(param.getContents(), c -> {
+            RankItem item = new RankItem();
+            item.setContent(c);
+            c.setScoreMap(scoreMap.get(c.getId()));
+            item.setScoreMap(scoreMap.get(c.getId()));
+            double score;
+            int index = weightService.getIndex(item.getContent().getContentPoolType(), contentPools);
+            if (contentPools[0].equals(item.getContent().getContentPoolType())
+                    || contentPools[1].equals(item.getContent().getContentPoolType())) {
+                score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.SIMILARITY.value())
+                        + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
+                        + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value())
+                        + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value());
+                if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
+                    score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
+                            * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                            ScoreStrategyEnum.VIEW_COUNT_RATE.value());
+                }
+            } else {
+                score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.SIMILARITY.value())
+                        + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
+                        + item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
+                        + item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value())
+                        + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value())
+                        + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
+            }
+            c.setScore(score);
+            item.setScore(score);
+            return item;
+        });
+        // 相似度评分为0 报警返回
+        List<Article> hisPublishFirstArticleList = articleRepository.getByGhIdAndItemIndexAndTypeEqualsAndStatusEquals(
+                param.getGhId(), 1, param.getType(), 1);
+        if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
+            return new RankResult(result);
+        }
+        // 安全分降权
+        RankService.safeScoreDecrease(items);
+
+        // 1 排序
+        Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
+        // 2 相似去重
+        List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
+
+        // 3 文章按照内容池分组
+        Map<String, List<Content>> contentMap = new HashMap<>();
+        for (Content c : contents) {
+            List<Content> data = contentMap.computeIfAbsent(c.getContentPoolType(), k -> new ArrayList<>());
+            data.add(c);
+        }
+        // 4 选文章
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
+
+        // 头
+        List<Content> pool1 = contentMap.get(contentPools[0]);
+        if (CollectionUtils.isNotEmpty(pool1)) {
+            pool1 = RankService.contentSourceTypeFilter(param.getStrategy(), pool1, 1);
+        }
+        RankService.printSortLog(param.getStrategy(), param.getAccountName(), "头条", pool1);
+        if (CollectionUtils.isNotEmpty(pool1)) {
+            if (pool1.get(0).getProducePlanId().equals(topProducePlanId)) {
+                int i = RandomUtils.nextInt(0, 1);
+                if (i == 0) {
+                    for (Content content : pool1) {
+                        if (!content.getProducePlanId().equals(topProducePlanId)) {
+                            result.add(content);
+                            break;
+                        }
+                    }
+                }
+            }
+            if (CollectionUtils.isEmpty(result)) {
+                result.add(pool1.get(0));
+            }
+        } else {
+            RankStrategy.sendFeishuFirstPoolEmpty(param, contentPools[0]);
+            return new RankResult(result);
+        }
+
+        // 次
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap,
+                indexReplacePoolConfigMap, param.getStrategy());
+
+        // 3-8
+        // RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
+        List<Content> pool = contentMap.get(contentPools[2]);
+        Integer videoSourceType = PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal();
+        Queue<Content> videoPoolQueue = pool.stream().filter(o -> Objects.equals(o.getSourceType(), videoSourceType))
+                .collect(Collectors.toCollection(LinkedList::new));
+        Queue<Content> otherPoolQueue = pool.stream().filter(o -> !Objects.equals(o.getSourceType(), videoSourceType))
+                .collect(Collectors.toCollection(LinkedList::new));
+        for (int i = 3; i < param.getSize() + 1; i++) {
+            Integer sourceType = RankService.getStrategyPoolSourceType(param.getStrategy(), i);
+            if (Objects.equals(sourceType, videoSourceType) && !videoPoolQueue.isEmpty()) {
+                result.add(videoPoolQueue.poll());
+            } else if (!otherPoolQueue.isEmpty()) {
+                result.add(otherPoolQueue.poll());
+            }
+        }
+
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
+        return new RankResult(result);
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -2,7 +2,6 @@ package com.tzld.longarticle.recommend.server.service.recommend.recall;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Lists;
@@ -368,6 +367,7 @@ public class RecallService implements ApplicationContextAware {
                 if (StringUtils.hasText(cache.getCategory())) {
                     content.setCategory(JSONArray.parseArray(cache.getCategory(), String.class));
                 }
+                content.setProducePlanId(cache.getPlanId());
                 content.setKimiSafeScore(cache.getKimiSafeScore());
                 content.setRootPublishTimestamp(cache.getRootPublishTimestamp());
                 content.setHisPublishArticleList(hisPublishArticleList);
@@ -410,6 +410,7 @@ public class RecallService implements ApplicationContextAware {
                 ArticleTitleHisCache cache = new ArticleTitleHisCache();
                 BeanUtils.copyProperties(content, cache);
                 cache.setType(type);
+                cache.setPlanId(content.getProducePlanId());
                 cache.setChannelContentId(content.getCrawlerChannelContentId());
                 if (CollectionUtils.isNotEmpty(content.getCategory())) {
                     cache.setCategory(JSONObject.toJSONString(content.getCategory()));
@@ -508,8 +509,8 @@ public class RecallService implements ApplicationContextAware {
                 .collect(Collectors.toMap(ArticleCategory::getTitleMd5, Function.identity(), (a, b) -> a));
         // 根据sourceId查询kimiSafeScore
         List<ProduceTaskAtom> safeScoreList = aigcBaseMapper.getProduceScoreByContentId(sourceIds);
-        Map<String, String> safeScoreMap = safeScoreList.stream().filter(o -> StringUtils.hasText(o.getOutput()))
-               .collect(Collectors.toMap(ProduceTaskAtom::getPlanExeId, ProduceTaskAtom::getOutput));
+        Map<String, ProduceTaskAtom> safeScoreMap = safeScoreList.stream().filter(o -> StringUtils.hasText(o.getOutput()))
+               .collect(Collectors.toMap(ProduceTaskAtom::getPlanExeId, o -> o));
 
         for (TitleHisCacheParam cacheParam : paramList) {
             Content res = new Content();
@@ -525,12 +526,16 @@ public class RecallService implements ApplicationContextAware {
                 res.setCategory(Collections.singletonList(category.getCategory()));
             }
             // 设置kimiSafeScore
-            String safeScore = safeScoreMap.get(cacheParam.getSourceId());
-            if (StringUtils.hasText(safeScore)) {
-                safeScore = safeScore.replaceAll("\n","");
-                if (safeScore.length() == 1) {
-                    res.setKimiSafeScore(safeScore);
+            ProduceTaskAtom atom = safeScoreMap.get(cacheParam.getSourceId());
+            if (Objects.nonNull(atom)) {
+                String safeScore = atom.getOutput();
+                if (StringUtils.hasText(safeScore)) {
+                    safeScore = safeScore.replaceAll("\n", "");
+                    if (safeScore.length() == 1) {
+                        res.setKimiSafeScore(safeScore);
+                    }
                 }
+                res.setProducePlanId(atom.getPlanId());
             }
             // 溯源查找源发布时间
             ArticlePoolPromotionSource source = sourceMap.get(cacheParam.getCrawlerChannelContentId());

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/ScoreService.java

@@ -127,7 +127,8 @@ public class ScoreService implements ApplicationContextAware {
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV12.getStrategy())
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV13.getStrategy())
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV14.getStrategy())
-                || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV15.getStrategy())) {
+                || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV15.getStrategy())
+                || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV16.getStrategy())) {
             strategies.add(strategyMap.get(ScoreStrategyEnum.CATEGORY.value()));
             strategies.add(strategyMap.get(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value()));
             strategies.add(strategyMap.get(ScoreStrategyEnum.FLOW_CTL_DECREASE.value()));

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -114,7 +114,7 @@
 
     <select id="getProduceScoreByContentId"
             resultType="com.tzld.longarticle.recommend.server.model.entity.aigc.ProduceTaskAtom">
-        select atom.plan_exe_id, atom.input, atom.output, record.audit_timestamp as createTimestamp
+        select atom.plan_exe_id, atom.plan_id, atom.input, atom.output, record.audit_timestamp as createTimestamp
         from produce_task_atom atom
         join produce_plan_exe_record record on atom.plan_exe_id = record.plan_exe_id
         join produce_plan_module_task task on task.task_id = atom.task_id