Merge branch 'master' into wyp/1202-videoAuditJob

# Conflicts:
#	long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleAuditService.java

long-article-recommend-service/pom.xml

@@ -60,6 +60,11 @@
             <groupId>com.ctrip.framework.apollo</groupId>
             <artifactId>apollo-client</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.ctrip.framework.apollo</groupId>
+            <artifactId>apollo-openapi</artifactId>
+            <version>1.9.1</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/ApolloConfigModifier.java

@@ -0,0 +1,39 @@
+package com.tzld.longarticle.recommend.server.common;
+
+import com.ctrip.framework.apollo.openapi.client.ApolloOpenApiClient;
+import com.ctrip.framework.apollo.openapi.dto.NamespaceReleaseDTO;
+import com.ctrip.framework.apollo.openapi.dto.OpenItemDTO;
+import com.tzld.longarticle.recommend.server.util.DateUtils;
+
+public class ApolloConfigModifier {
+
+    public static void modifyConfig(String key, String value, String env) {
+        // 1. Apollo OpenAPI 客户端配置
+        ApolloOpenApiClient client = ApolloOpenApiClient.newBuilder()
+                .withPortalUrl("http://apolloportal-internal.piaoquantv.com/")
+                .withToken("a3234bae4cd0eab36be8107a13106d4a8c04f895")
+                .build();
+
+        // 2. 定义修改参数
+        String appId = "longarticle-recommend";
+        String clusterName = "default";
+        String namespace = "application";
+
+        // 3. 创建配置项
+        OpenItemDTO item = new OpenItemDTO();
+        item.setKey(key);
+        item.setValue(value);
+        item.setDataChangeCreatedBy("wangyunpeng");
+
+        // 4. 调用 API 更新配置
+        client.createOrUpdateItem(appId, env, clusterName, namespace, item);
+
+        // 5. 发布配置
+        NamespaceReleaseDTO releaseDTO = new NamespaceReleaseDTO();
+        releaseDTO.setReleaseTitle(DateUtils.getCurrentDateStr("yyyyMMddHHmmss") + "-release");
+        releaseDTO.setReleaseComment("Release Comment");
+        releaseDTO.setReleasedBy("wangyunpeng");
+        client.publishNamespace(appId, env, clusterName, namespace, releaseDTO);
+        System.out.println("配置已修改并发布成功！");
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/CostMonitor.java

@@ -0,0 +1,14 @@
+package com.tzld.longarticle.recommend.server.common;
+
+import com.tzld.longarticle.recommend.server.common.constant.LogConstants;
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
+
+@Slf4j
+public class CostMonitor {
+    public static void logCost(String type, String name, long cost) {
+        Marker marker = MarkerFactory.getMarker(LogConstants.MARKER_COST);
+        log.info(marker, "{}.{} cost: {}", type, name, cost);
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/HttpPoolFactory.java

@@ -14,10 +14,10 @@ public final class HttpPoolFactory {
             HttpClientFactory.create(60000, 60000, 200, 200, 0, 60000);
 
     private static CloseableHttpClient NLP =
-            HttpClientFactory.create(15000, 15000, 200, 200, 0, 15000);
+            HttpClientFactory.create(1000, 30000, 200, 200, 0, 5000);
 
     private static CloseableHttpClient ThirtySecond =
-            HttpClientFactory.create(30000, 30000, 200, 200, 0, 30000);
+            HttpClientFactory.create(1000, 60000, 200, 200, 0, 5000);
 
 
     public static CloseableHttpClient defaultPool() {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/RecommendLoghubAppender.java

@@ -13,6 +13,7 @@ import com.aliyun.openservices.aliyun.log.producer.ProducerConfig;
 import com.aliyun.openservices.aliyun.log.producer.ProjectConfig;
 import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException;
 import com.aliyun.openservices.log.common.LogItem;
+import com.tzld.longarticle.recommend.server.common.constant.LogConstants;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
@@ -103,62 +104,105 @@ public class RecommendLoghubAppender<E> extends UnsynchronizedAppenderBase<E> {
         } catch (Exception var3) {
             this.addError("Failed to append event.", var3);
         }
-
     }
 
     private void appendEvent(E eventObject) {
-        if (eventObject instanceof LoggingEvent) {
-            LoggingEvent event = (LoggingEvent) eventObject;
-            List<LogItem> logItems = new ArrayList();
-            LogItem item = new LogItem();
-            logItems.add(item);
-            item.SetTime((int) (event.getTimeStamp() / 1000L));
-            if (this.formatter != null) {
-                DateTime dateTime = new DateTime(event.getTimeStamp());
-                item.PushBack("time", dateTime.toString(this.formatter));
-            } else {
-                Instant instant = Instant.ofEpochMilli(event.getTimeStamp());
-                item.PushBack("time", this.formatter1.format(instant));
-            }
-
-            item.PushBack("level", event.getLevel().toString());
-            item.PushBack("thread", event.getThreadName());
-            StackTraceElement[] caller = event.getCallerData();
-            if (caller != null && caller.length > 0) {
-                item.PushBack("location", caller[0].toString());
-            }
-
-            String message = event.getFormattedMessage();
-            item.PushBack("message", message);
-            IThrowableProxy iThrowableProxy = event.getThrowableProxy();
-            if (iThrowableProxy != null) {
-                String throwable = this.getExceptionInfo(iThrowableProxy);
-                throwable = throwable + this.fullDump(event.getThrowableProxy().getStackTraceElementProxyArray());
-                item.PushBack("throwable", throwable);
-            }
-
-            if (this.encoder != null) {
-                item.PushBack("log", new String(this.encoder.encode(eventObject)));
-            }
-
-            Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
-                event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
-                    return Arrays.stream(f.split(",")).anyMatch((i) -> {
-                        return i.equals(v.getKey());
-                    });
-                }).forEach((map) -> {
-                    item.PushBack((String) map.getKey(), (String) map.getValue());
+        if (!(eventObject instanceof LoggingEvent)) {
+            return;
+        }
+        LoggingEvent event = (LoggingEvent) eventObject;
+        if (event.getMarker() == null) {
+            appendPlainEvent(eventObject);
+        } else if (LogConstants.MARKER_COST.equals(event.getMarker().getName())) {
+            appendCostEvent(event);
+        }
+    }
+
+    private void appendCostEvent(LoggingEvent event) {
+        List<LogItem> logItems = new ArrayList();
+        LogItem item = new LogItem();
+        logItems.add(item);
+        item.SetTime((int) (event.getTimeStamp() / 1000L));
+
+        item.PushBack("thread", event.getThreadName());
+        Object[] args = event.getArgumentArray();
+        try {
+            item.PushBack("type", (String) args[0]);
+            item.PushBack("name", (String) args[1]);
+            item.PushBack("cost", String.valueOf(args[2]));
+        } catch (Exception e) {
+            this.addError("invalid event argument ", e);
+        }
+
+        Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
+            event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
+                return Arrays.stream(f.split(",")).anyMatch((i) -> {
+                    return i.equals(v.getKey());
                 });
+            }).forEach((map) -> {
+                item.PushBack((String) map.getKey(), (String) map.getValue());
             });
+        });
+
+        try {
+            this.producer.send(this.projectConfig.getProject(), this.logStore, "CostMonitor", this.source, logItems,
+                    new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
+                            this.topic, this.source, logItems));
+        } catch (Exception e) {
+            this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, e);
+        }
+    }
+
+    private void appendPlainEvent(E eventObject) {
+        LoggingEvent event = (LoggingEvent) eventObject;
+        List<LogItem> logItems = new ArrayList();
+        LogItem item = new LogItem();
+        logItems.add(item);
+        item.SetTime((int) (event.getTimeStamp() / 1000L));
+        if (this.formatter != null) {
+            DateTime dateTime = new DateTime(event.getTimeStamp());
+            item.PushBack("time", dateTime.toString(this.formatter));
+        } else {
+            Instant instant = Instant.ofEpochMilli(event.getTimeStamp());
+            item.PushBack("time", this.formatter1.format(instant));
+        }
+
+        item.PushBack("level", event.getLevel().toString());
+        item.PushBack("thread", event.getThreadName());
+        StackTraceElement[] caller = event.getCallerData();
+        if (caller != null && caller.length > 0) {
+            item.PushBack("location", caller[0].toString());
+        }
+
+        String message = event.getFormattedMessage();
+        item.PushBack("message", message);
+        IThrowableProxy iThrowableProxy = event.getThrowableProxy();
+        if (iThrowableProxy != null) {
+            String throwable = this.getExceptionInfo(iThrowableProxy);
+            throwable = throwable + this.fullDump(event.getThrowableProxy().getStackTraceElementProxyArray());
+            item.PushBack("throwable", throwable);
+        }
+
+        if (this.encoder != null) {
+            item.PushBack("log", new String(this.encoder.encode(eventObject)));
+        }
 
-            try {
-                this.producer.send(this.projectConfig.getProject(), this.logStore, this.topic, this.source, logItems,
-                        new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
-                                this.topic, this.source, logItems));
-            } catch (Exception var9) {
-                this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, var9);
-            }
+        Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
+            event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
+                return Arrays.stream(f.split(",")).anyMatch((i) -> {
+                    return i.equals(v.getKey());
+                });
+            }).forEach((map) -> {
+                item.PushBack((String) map.getKey(), (String) map.getValue());
+            });
+        });
 
+        try {
+            this.producer.send(this.projectConfig.getProject(), this.logStore, this.topic, this.source, logItems,
+                    new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
+                            this.topic, this.source, logItems));
+        } catch (Exception var9) {
+            this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, var9);
         }
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/constant/LogConstants.java

@@ -0,0 +1,5 @@
+package com.tzld.longarticle.recommend.server.common.constant;
+
+public class LogConstants {
+    public static final String MARKER_COST = "COST";
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/SecretEnum.java

@@ -8,7 +8,8 @@ public enum SecretEnum {
 
     SECRET_ENUM_1("3b83574b477d4c5b8508a6e33f6e35ec", "魅力", "ml"),
     SECRET_ENUM_2("70d342bf11a84ac7aca6b3e99541e085", "老来福", "llf"),
-    SECRET_ENUM_3("595db67618174499b2bed23d8be6a3c1", "微小盟", "wxm");
+    SECRET_ENUM_3("595db67618174499b2bed23d8be6a3c1", "微小盟", "wxm"),
+    SECRET_ENUM_4("9a48498757774ddf8a3878b9c02d0fef", "福州像素", "xs");
 
     SecretEnum(String secret, String desc, String channel) {
         this.secret = secret;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/StatusEnum.java

@@ -7,7 +7,6 @@ public enum StatusEnum {
 
     ZERO(0, "0"),
     ONE(1, "1"),
-    TWO(2, "2"),
     ;
 
     private int code;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/aigc/ProduceContentAuditStatusEnum.java

@@ -0,0 +1,34 @@
+package com.tzld.longarticle.recommend.server.common.enums.aigc;
+
+import lombok.Getter;
+
+import java.util.Objects;
+
+@Getter
+public enum ProduceContentAuditStatusEnum {
+    // 0-待审核，1-审核通过，2-审核不通过，3-不审核
+    waiting(0, "待审核"),
+    pass(1, "审核通过"),
+    reject(2, "审核不通过"),
+    not_audit(3, "不审核"),
+
+    other(999, "其他"),
+    ;
+
+    private final Integer val;
+    private final String description;
+
+    ProduceContentAuditStatusEnum(Integer val, String description) {
+        this.val = val;
+        this.description = description;
+    }
+
+    public static ProduceContentAuditStatusEnum from(Integer val) {
+        for (ProduceContentAuditStatusEnum statusEnum : ProduceContentAuditStatusEnum.values()) {
+            if (Objects.equals(statusEnum.val, val)) {
+                return statusEnum;
+            }
+        }
+        return other;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/aigc/PublishPlanInputSourceTypesEnum.java

@@ -0,0 +1,40 @@
+package com.tzld.longarticle.recommend.server.common.enums.aigc;
+
+import lombok.Getter;
+
+import java.util.Objects;
+
+@Getter
+public enum PublishPlanInputSourceTypesEnum {
+    producePlan(1, "生成计划"),
+    produceContent(2, "单个生成内容"),
+    crawlerPlan(3, "抓取计划"),
+    crawlerContent(4, "单个抓取内容"),
+    publishPlan(5, "发布计划"),
+    publishContent(6, "单个发布内容"),
+    pretreatPlan(7, "处理计划"),
+    pretreatContent(8, "单个处理内容"),
+    monitor_plan_query_contents(9, "监控计划查找输入源"),
+    template(10, "评论/私信模板"),
+    monitor_plan(11, "监控计划内容"),
+    longArticleVideoPoolSource(12, "长文视频池内容"),
+    other(999, ""),
+    ;
+
+    private final Integer val;
+    private final String description;
+
+    PublishPlanInputSourceTypesEnum(Integer val, String description) {
+        this.val = val;
+        this.description = description;
+    }
+
+    public static PublishPlanInputSourceTypesEnum from(Integer val) {
+        for (PublishPlanInputSourceTypesEnum typesEnum : PublishPlanInputSourceTypesEnum.values()) {
+            if (Objects.equals(typesEnum.val, val)) {
+                return typesEnum;
+            }
+        }
+        return null;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/ArticleDeleteStatusEnum.java

@@ -0,0 +1,29 @@
+package com.tzld.longarticle.recommend.server.common.enums.recommend;
+
+import lombok.Getter;
+
+@Getter
+public enum ArticleDeleteStatusEnum {
+
+    WAITING(0, "WAITING"),
+    SUCCESS(1, "SUCCESS"),
+    FAIL(2, "FAIL"),
+    ;
+
+    private int code;
+    private String msg;
+
+    ArticleDeleteStatusEnum(int code, String msg) {
+        this.code = code;
+        this.msg = msg;
+    }
+
+    public static ArticleDeleteStatusEnum getByCode(int code) {
+        for (ArticleDeleteStatusEnum statusEnum : ArticleDeleteStatusEnum.values()) {
+            if (statusEnum.getCode() == code) {
+                return statusEnum;
+            }
+        }
+        return null;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/ContentPoolEnum.java

@@ -8,19 +8,20 @@ import java.util.Objects;
 
 @Getter
 public enum ContentPoolEnum {
-    autoArticlePoolLevel1("autoArticlePoolLevel1", "内容池1层"),
-    autoArticlePoolLevel2("autoArticlePoolLevel2", "内容池2层"),
-    autoArticlePoolLevel3("autoArticlePoolLevel3", "内容池3层"),
-    autoArticlePoolLevel4("autoArticlePoolLevel4", "冷启层"),
+    autoArticlePoolLevel1("autoArticlePoolLevel1", 1, "内容池1层"),
+    autoArticlePoolLevel2("autoArticlePoolLevel2", 2, "内容池2层"),
+    autoArticlePoolLevel3("autoArticlePoolLevel3", 3, "内容池3层"),
+    autoArticlePoolLevel4("autoArticlePoolLevel4", 4, "冷启层"),
 
     ;
 
-
     private final String contentPool;
+    private final Integer value;
     private final String description;
 
-    ContentPoolEnum(String contentPool, String description) {
+    ContentPoolEnum(String contentPool, Integer value, String description) {
         this.contentPool = contentPool;
+        this.value = value;
         this.description = description;
     }
 
@@ -33,6 +34,15 @@ public enum ContentPoolEnum {
         return autoArticlePoolLevel4;
     }
 
+    public static ContentPoolEnum from(Integer value) {
+        for (ContentPoolEnum poolEnum : ContentPoolEnum.values()) {
+            if (Objects.equals(poolEnum.value, value)) {
+                return poolEnum;
+            }
+        }
+        return autoArticlePoolLevel4;
+    }
+
     public static List<String> getOrderContentPool() {
         List<String> result = new ArrayList<>();
         result.add(autoArticlePoolLevel1.getContentPool());

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/FeishuRobotIdEnum.java

@@ -8,7 +8,8 @@ import java.util.Objects;
 public enum FeishuRobotIdEnum {
     RECOMMEND("长文排序报警群", "07026a9f-43f5-448b-ba40-a8d71bd6e634"),
     JOB("定时任务报警群", "186c9798-5b6a-4ff8-b7fc-4ce4b6ea5076"),
-    ARTICLE_DELETE("文章删除报警群", "6e43785b-19f6-4d28-a369-829ba8d7bf5d"),
+    DAILY("长文每日更新报警群", "b44333f2-16c0-4cb1-af01-d135f8704410"),
+    ARTICLE_DELETE("文章删除报警群", "2262caec-3f52-4a93-9fa5-da2382a21482"),
 
     ;
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/RankStrategyEnum.java

@@ -20,6 +20,8 @@ public enum RankStrategyEnum {
     ArticleRankV13("ArticleRankV13", "ArticleRankV13", "rankV13Strategy"),
     ArticleRankV14("ArticleRankV14", "ArticleRankV14", "rankV14Strategy"),
 
+    ArticleRankV15("ArticleRankV15", "ArticleRankV15", "rankV15Strategy"),
+
     HIS_JUMP_STRATEGY("ArticleRankHisJump", "历史表现跳过相似度策略", "hisJumpRankStrategy"),
     INFINITE_STRATEGY("ArticleRankInfinite", "无限发表", "infiniteRankStrategy"),
     LATE_STRATEGY("ArticleRankLate", "晚间策略", "lateRankStrategy"),

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -33,6 +33,8 @@ public interface AigcBaseMapper {
 
     List<PublishContent> getNearestPublishContent(String publishAccountId, Long publishTimestamp, Integer size);
 
+    List<PublishContent> getLateNearestPublishContent(String publishAccountId, Long publishTimestamp);
+
     CrawlerContent getCrawlerContentByChannelContentId(String channelContentId);
 
     List<CrawlerContent> getCrawlerContentByChannelContentIdIn(List<String> channelContentIds);
@@ -52,4 +54,10 @@ public interface AigcBaseMapper {
     List<PublishGzhPushContentRelDTO> getGroupPushRelByPushIdIn(List<String> pushIds);
 
     PublishContentDTO getPublishContentById(String publishContentId);
+
+    List<PublishContentDTO> getPublishContentBySourceIdIn(List<String> sourceIds);
+
+    List<CrawlerPlan> getColdCrawlerPlan(Long timeStart, Long timeEnd, List<String> planTags);
+
+    List<ProducePlanAuditCheckDTO> getColdProducePlanAudit(Long timeStart, Long timeEnd, List<String> planIds);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/crawler/CrawlerBaseMapper.java

@@ -7,9 +7,11 @@ import com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideoDTO;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountCorrelation;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
+import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleDetailInfo;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesRootSourceId;
 
 import java.util.List;
+import java.util.Set;
 
 public interface CrawlerBaseMapper {
 
@@ -46,4 +48,13 @@ public interface CrawlerBaseMapper {
     List<LongArticlesText> getLongArticlesText();
 
     List<LongArticlesVideoDTO> getLongArticlesVideo(List<String> traceIds);
+
+    List<Article> getByTitleMd5InAndTypeEqualsAndStatusEquals(List<String> titleMd5s, String type, Integer status);
+
+    List<ArticleDetailInfo> getAllByWxSnIn(List<String> wxSnList);
+
+    List<AccountAvgInfo> getAllByGhIdIn(Set<String> ghIdList);
+
+    List<Article> getByGhIdInAndAppMsgIdInAndItemIndexAndTypeEqualsAndStatusEquals(
+            Set<String> ghIdList, Set<String> appMsgIdList, Integer itemIndex, String type, Integer status);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -1,11 +1,11 @@
 package com.tzld.longarticle.recommend.server.mapper.longArticle;
 
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesRootSourceId;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatScore;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticlePoolPromotionSource;
-import com.tzld.longarticle.recommend.server.model.dto.*;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatSortStrategy;
+import com.tzld.longarticle.recommend.server.model.dto.GetOffVideos;
+import com.tzld.longarticle.recommend.server.model.dto.LongArticlesCrawlerVideos;
+import com.tzld.longarticle.recommend.server.model.dto.LongArticlesMatchVideos;
+import com.tzld.longarticle.recommend.server.model.dto.LongArticlesText;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
+import com.tzld.longarticle.recommend.server.model.param.ArticleVideoPoolSourceParam;
 import org.apache.ibatis.annotations.Mapper;
 
 import java.util.List;
@@ -21,6 +21,8 @@ public interface LongArticleBaseMapper {
 
     void batchInsertArticlePoolPromotionSource(List<ArticlePoolPromotionSource> list);
 
+    void batchInsertArticleTitleHisCache(List<ArticleTitleHisCache> list);
+
     void updateRootProduceContentLevel(String rootProduceContentId, String level);
 
     void deleteDatastatScoreByDtIn(List<String> dateStrList);
@@ -61,4 +63,8 @@ public interface LongArticleBaseMapper {
     void updateLongArticleMatchVideosResponse(LongArticlesMatchVideos longArticlesMatchVideos);
 
     int countNeedMatchVideos(Long id);
+
+    List<PublishSingleVideoSource> getPublishSingleVideoSource(ArticleVideoPoolSourceParam param);
+
+    List<String> getFilterColdLongArticleTitle();
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/Content.java

@@ -18,8 +18,10 @@ import java.util.Map;
 @AllArgsConstructor
 public class Content {
     private String id;
+    private Integer sourceType;
     private String sourceId;
     private String title;
+    private String titleMd5;
     private Long createTimestamp;
     private String producePlanName;
     private String contentPoolType; // 内容池类别

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/ProducePlanAuditCheckDTO.java

@@ -0,0 +1,10 @@
+package com.tzld.longarticle.recommend.server.model.dto;
+
+import lombok.Data;
+
+@Data
+public class ProducePlanAuditCheckDTO {
+    private String planId;
+    private Integer auditStatus;
+    private Integer num;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/crawler/PublishContentSortLog.java

@@ -1,30 +0,0 @@
-package com.tzld.longarticle.recommend.server.model.entity.crawler;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import javax.persistence.*;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-@Entity
-@Table(name = "publish_content_sort_log")
-public class PublishContentSortLog {
-
-    @Id
-    @GeneratedValue(strategy = GenerationType.IDENTITY)
-    private Long id;
-    @Column(name = "gh_id")
-    private String ghId;
-    @Column(name = "account_name")
-    private String accountName;
-    @Column(name = "strategy")
-    private String strategy;
-    @Column(name = "publish_content_id")
-    private String publishContentId;
-    @Column(name = "create_timestamp")
-    private Long createTimestamp;
-
-}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/crawler/PublishSortLog.java

@@ -22,8 +22,14 @@ public class PublishSortLog {
     private String ghId;
     @Column(name = "account_name")
     private String accountName;
+    @Column(name = "publish_content_id")
+    private String publishContentId;
     @Column(name = "crawler_channel_content_id")
     private String crawlerChannelContentId;
+    @Column(name = "source_type")
+    private Integer sourceType;
+    @Column(name = "source_id")
+    private String sourceId;
     @Column(name = "title")
     private String title;
     @Column(name = "`index`")

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleCategory.java

@@ -39,7 +39,7 @@ public class ArticleCategory {
     private String kimiResult;
 
     @Column(name = "status")
-    private Integer status;
+    private Integer status = 0;
 
     @Column(name = "fail_reason")
     private String failReason;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleTitleHisCache.java

@@ -0,0 +1,56 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.*;
+import java.io.Serializable;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "article_title_his_cache")
+@IdClass(ArticleTitleHisCache.PK.class)
+public class ArticleTitleHisCache implements Serializable {
+
+    @Id
+    @Column(name = "source_id")
+    private String sourceId;
+    @Id
+    @Column(name = "type")
+    private String type;
+    @Column(name = "title")
+    private String title;
+    @Column(name = "title_md5")
+    private String titleMd5;
+    @Column(name = "crawler_title")
+    private String crawlerTitle;
+    @Column(name = "channel_content_id")
+    private String channelContentId;
+    @Column(name = "root_publish_timestamp")
+    private Long rootPublishTimestamp;
+    @Column(name = "category")
+    private String category;
+    @Column(name = "his_publish_article_list")
+    private String hisPublishArticleList;
+    @Column(name = "create_timestamp")
+    private Long createTimestamp;
+    @Column(name = "update_timestamp")
+    private Long updateTimestamp;
+
+
+    @Data
+    public static class PK implements Serializable {
+
+        @Column(name = "source_id")
+        private String sourceId;
+        @Column(name = "type")
+        private String type;
+
+        public PK() {
+        }
+
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleUnsafeTitle.java

@@ -0,0 +1,29 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "article_unsafe_title")
+public class ArticleUnsafeTitle {
+    @Id
+    @Column(name = "title_md5")
+    private String titleMd5;
+    @Column(name = "title")
+    private String title;
+    @Column(name = "status")
+    private Integer status;
+    @Column(name = "create_timestamp")
+    private Long createTimestamp;
+    @Column(name = "update_timestamp")
+    private Long updateTimestamp;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/DatastatScore.java

@@ -74,4 +74,16 @@ public class DatastatScore {
     @Column(name = "first_pub_interval")
     private Integer firstPubInterval;
 
+    @Column(name = "publish_content_id")
+    private String publishContentId;
+
+    @Column(name = "crawler_channel_content_id")
+    private String crawlerChannelContentId;
+
+    @Column(name = "source_id")
+    private String sourceId;
+
+    @Column(name = "publish_timestamp")
+    private Long publishTimestamp;
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/DatastatSortStrategy.java

@@ -73,23 +73,23 @@ public class DatastatSortStrategy implements Serializable {
     @Column(name = "third_fission2")
     private Integer thirdFission2;
     @Column(name = "read_rate")
-    private Double readRate;
+    private Double readRate = 0.0;
     @Column(name = "read_fans_rate")
-    private Double readFansRate;
+    private Double readFansRate = 0.0;
     @Column(name = "first_read_rate")
-    private Double firstReadRate;
+    private Double firstReadRate = 0.0;
     @Column(name = "fission0_first_rate")
-    private Double fission0FirstRate;
+    private Double fission0FirstRate = 0.0;
     @Column(name = "fission1_fission0_rate")
-    private Double fission1Fission0Rate;
+    private Double fission1Fission0Rate = 0.0;
     @Column(name = "fission0_read_avg_rate")
-    private Double fission0ReadAvgRate;
+    private Double fission0ReadAvgRate = 0.0;
     @Column(name = "his_read_rate")
-    private Double hisReadRate;
+    private Double hisReadRate = 0.0;
     @Column(name = "his_first_read_rate")
-    private Double hisFirstReadRate;
+    private Double hisFirstReadRate = 0.0;
     @Column(name = "his_fission0_first_rate")
-    private Double hisFission0FirstRate;
+    private Double hisFission0FirstRate = 0.0;
     @Column(name = "gh_id")
     private String ghId;
     @Column(name = "account_create_timestamp")
@@ -103,11 +103,11 @@ public class DatastatSortStrategy implements Serializable {
     @Column(name = "publish_timestamp")
     private Long publishTimestamp;
     @Column(name = "fission0_read_avg_100_rate")
-    private Double fission0ReadAvg100Rate;
+    private Double fission0ReadAvg100Rate = 0.0;
     @Column(name = "fission0_read_avg_500_rate")
-    private Double fission0ReadAvg500Rate;
+    private Double fission0ReadAvg500Rate = 0.0;
     @Column(name = "fission0_read_avg_1000_rate")
-    private Double fission0ReadAvg1000Rate;
+    private Double fission0ReadAvg1000Rate = 0.0;
     @Column(name = "crawler_plan_name")
     private String crawlerPlanName;
     @Column(name = "crawler_plan_tag")
@@ -126,6 +126,10 @@ public class DatastatSortStrategy implements Serializable {
     private Integer publishMiniProgramNum;
     @Column(name = "source_produce_plan_name")
     private String sourceProducePlanName;
+    @Column(name = "source_type")
+    private Integer sourceType;
+    @Column(name = "source_id")
+    private String sourceId;
 
 
     @Data

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/LongArticleAuditDelete.java

@@ -10,8 +10,9 @@ import javax.persistence.*;
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
-@Table(name = "long_articles_audit_delete")
+@Table(name = "long_article_audit_delete")
 public class LongArticleAuditDelete {
+
     @Id
     @GeneratedValue(strategy = GenerationType.IDENTITY)
     @Column(name = "id")
@@ -23,7 +24,7 @@ public class LongArticleAuditDelete {
     @Column(name = "msg_id")
     private String msgId;
 
-    @Column(name = "index")
+    @Column(name = "`index`")
     private Integer index;
 
     @Column(name = "push_id")

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/PublishSingleVideoSource.java

@@ -0,0 +1,79 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.*;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "publish_single_video_source")
+public class PublishSingleVideoSource {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @Column(name = "content_trace_id")
+    private String contentTraceId;
+
+    @Column(name = "article_title")
+    private String articleTitle;
+
+    @Column(name = "out_account_id")
+    private String outAccountId;
+
+    @Column(name = "out_account_name")
+    private String outAccountName;
+
+    @Column(name = "read_cnt")
+    private Integer readCount;
+
+    @Column(name = "like_cnt")
+    private Integer likeCount;
+
+    @Column(name = "article_index")
+    private Byte articleIndex;
+
+    @Column(name = "article_publish_type")
+    private String articlePublishType;
+
+    @Column(name = "article_url")
+    private String articleUrl;
+
+    @Column(name = "cover_url")
+    private String coverUrl;
+
+    @Column(name = "video_oss_path")
+    private String videoOssPath;
+
+    @Column(name = "flow_pool_level")
+    private Integer flowPoolLevel;
+
+    @Column(name = "bad_status")
+    private Byte badStatus;
+
+    @Column(name = "publish_timestamp")
+    private Long publishTimestamp;
+
+    @Column(name = "crawler_timestamp")
+    private Long crawlerTimestamp;
+
+    @Column(name = "url_unique_md5")
+    private String urlUniqueMd5;
+
+    @Column(name = "up_level_timestamp")
+    private Long upLevelTimestamp;
+
+    @Column(name = "exit_timestamp")
+    private Long exitTimestamp;
+
+    @Column(name = "source_account")
+    private Integer sourceAccount;
+
+    @Column(name = "audit_status")
+    private Integer auditStatus;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/ArticleDangerFindDeleteParam.java

@@ -0,0 +1,13 @@
+package com.tzld.longarticle.recommend.server.model.param;
+
+import lombok.Data;
+
+@Data
+public class ArticleDangerFindDeleteParam {
+
+    private String ghId;
+    private String accountName;
+    private String title;
+    private String wxSn;
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/ArticleVideoPoolSourceParam.java

@@ -0,0 +1,8 @@
+package com.tzld.longarticle.recommend.server.model.param;
+
+import lombok.Data;
+
+@Data
+public class ArticleVideoPoolSourceParam {
+    private Integer flowPoolLevel;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/PublishContentFilterContentItem.java

@@ -0,0 +1,10 @@
+package com.tzld.longarticle.recommend.server.model.param;
+
+import lombok.Data;
+
+@Data
+public class PublishContentFilterContentItem {
+    private String sourceId;
+    private String poolLevel;
+    private String title;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/PublishContentFilterParam.java

@@ -0,0 +1,11 @@
+package com.tzld.longarticle.recommend.server.model.param;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class PublishContentFilterParam {
+    private String ghId;
+    private List<PublishContentFilterContentItem> contentList;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/RecommendParam.java

@@ -27,5 +27,6 @@ public class RecommendParam {
     private boolean excludeLog = false;
     private String scene;
     private List<Integer> userGroupIds;
+    private Boolean replaceSimilarityAccount = false;
 }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/TitleHisCacheParam.java

@@ -0,0 +1,16 @@
+package com.tzld.longarticle.recommend.server.model.param;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class TitleHisCacheParam {
+
+    private String sourceId;
+    private String title;
+    private String crawlerTitle;
+    private String crawlerChannelContentId;
+    private String titleMd5;
+    private List<String> category;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/ArticleSortResponseDataItem.java

@@ -11,6 +11,7 @@ import java.util.Map;
 @Accessors(chain = true)
 public class ArticleSortResponseDataItem {
     private String id;
+    private Integer sourceType;
     private String sourceId;
     private String contentPoolType; // 内容池类别
     private String title;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/ArticleVideoPoolSourceVO.java

@@ -0,0 +1,28 @@
+package com.tzld.longarticle.recommend.server.model.vo;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource;
+import lombok.Data;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Data
+public class ArticleVideoPoolSourceVO {
+    private Integer poolId;
+    private Integer flowPoolLevel;
+    private String sourceId;
+    private String title;
+    private String coverUrl;
+
+    public static List<ArticleVideoPoolSourceVO> convertFromPublishSingleVideoSource(List<PublishSingleVideoSource> list) {
+        return list.stream().map(publishSingleVideoSource -> {
+            ArticleVideoPoolSourceVO articleVideoPoolSourceVO = new ArticleVideoPoolSourceVO();
+            articleVideoPoolSourceVO.setPoolId(publishSingleVideoSource.getFlowPoolLevel());
+            articleVideoPoolSourceVO.setFlowPoolLevel(publishSingleVideoSource.getFlowPoolLevel());
+            articleVideoPoolSourceVO.setSourceId(publishSingleVideoSource.getContentTraceId());
+            articleVideoPoolSourceVO.setTitle(publishSingleVideoSource.getArticleTitle());
+            articleVideoPoolSourceVO.setCoverUrl(publishSingleVideoSource.getCoverUrl());
+            return articleVideoPoolSourceVO;
+        }).collect(Collectors.toList());
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/NewSortStrategyExport.java

@@ -1,39 +1,45 @@
 package com.tzld.longarticle.recommend.server.model.vo;
 
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatSortStrategy;
 import lombok.Data;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.BeanUtils;
+
+import java.util.ArrayList;
+import java.util.List;
 
 @Data
 public class NewSortStrategyExport {
 
     private String dateStr;
     private String publishTime;
-    private String accountMode;
+//    private String accountMode;
     private String accountSource;
-    private String accountType;
+//    private String accountType;
     private String accountStatus;
     private String businessType;
     private String accountName;
     private String strategy;
     private long fans;
-    private Integer viewCount;
+//    private Integer viewCount;
     private Double avgViewCount;
-    private Integer firstViewCount;
-    private Double firstAvgViewCount;
-    // 首日
-    private Integer firstLevel;
-    private Integer fission0;
-    private Integer fission1;
-    private Integer fission2;
-    // 次日
-    private Integer secondFirstLevel;
-    private Integer secondFission0;
-    private Integer secondFission1;
-    private Integer secondFission2;
-    // 第三日
-    private Integer thirdFirstLevel;
-    private Integer thirdFission0;
-    private Integer thirdFission1;
-    private Integer thirdFission2;
+//    private Integer firstViewCount;
+//    private Double firstAvgViewCount;
+//    // 首日
+//    private Integer firstLevel;
+//    private Integer fission0;
+//    private Integer fission1;
+//    private Integer fission2;
+//    // 次日
+//    private Integer secondFirstLevel;
+//    private Integer secondFission0;
+//    private Integer secondFission1;
+//    private Integer secondFission2;
+//    // 第三日
+//    private Integer thirdFirstLevel;
+//    private Integer thirdFission0;
+//    private Integer thirdFission1;
+//    private Integer thirdFission2;
     private Double readRate = 0.0;
     private Double readFansRate = 0.0;
     private Double firstReadRate = 0.0;
@@ -49,28 +55,41 @@ public class NewSortStrategyExport {
     private String ghId;
     private String title;
     private String link;
-    private String wxSn;
-    private Double fission0ReadAvg100Rate = 0.0;
-    private Double fission0ReadAvg500Rate = 0.0;
-    private Double fission0ReadAvg1000Rate = 0.0;
+//    private String wxSn;
+//    private Double fission0ReadAvg100Rate = 0.0;
+//    private Double fission0ReadAvg500Rate = 0.0;
+//    private Double fission0ReadAvg1000Rate = 0.0;
 
     // 抓取计划名称
     private String crawlerPlanName;
-    // 抓取计划策略
-    private String crawlerPlanTag;
+//    // 抓取计划策略
+//    private String crawlerPlanTag;
     // 生成计划名称
     private String producePlanName;
-    // 生成计划策略
-    private String producePlanTag;
+//    // 生成计划策略
+//    private String producePlanTag;
     // 发布计划名称
     private String publishPlanName;
-    // 发布匹配小程序策略
-    private String publishMiniProgramInsertStrategy;
-    // 发布匹配小程序小程序用途
-    private String publishMiniProgramInsertUseType;
-    // 发布内容小程序数量
-    private Integer publishMiniProgramNum;
+//    // 发布匹配小程序策略
+//    private String publishMiniProgramInsertStrategy;
+//    // 发布匹配小程序小程序用途
+//    private String publishMiniProgramInsertUseType;
+//    // 发布内容小程序数量
+//    private Integer publishMiniProgramNum;
     // 源生成计划名称
     private String sourceProducePlanName;
 
+    public static List<NewSortStrategyExport> dbObjToExportObj(List<DatastatSortStrategy> list) {
+        if (CollectionUtils.isEmpty(list)) {
+            return null;
+        }
+        List<NewSortStrategyExport> result = new ArrayList<>();
+        list.forEach(item -> {
+            NewSortStrategyExport export = new NewSortStrategyExport();
+            BeanUtils.copyProperties(item, export);
+            result.add(export);
+        });
+        return result;
+    }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/PublishContentFilterResultVO.java

@@ -0,0 +1,9 @@
+package com.tzld.longarticle.recommend.server.model.vo;
+
+import lombok.Data;
+
+@Data
+public class PublishContentFilterResultVO {
+    private String sourceId;
+    private String filterReason;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/NLPRemoteService.java

@@ -57,7 +57,6 @@ public class NLPRemoteService {
         String url = scoreListUrl;
         JSONObject bodyParam = new JSONObject();
         bodyParam.put("gh_id_list", Collections.singletonList(ghId));
-        bodyParam.put("account_nickname_list", Collections.singletonList(accountName));
         bodyParam.put("text_list", titleList);
         bodyParam.put("interest_type", accountScoreInterestTypeMap.getOrDefault(accountName, "avg"));
         bodyParam.put("sim_type", accountScoreSimTypeMap.getOrDefault(accountName, "mean"));
@@ -69,7 +68,7 @@ public class NLPRemoteService {
             bodyParam.put("min_time", accountScoreMinTimeMap.get(accountName));
         }
         int retry = 0;
-        while (retry < 3) {
+        while (retry < 2) {
             retry++;
             try {
                 HttpPost httpPost = new HttpPost(url);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/WxAccessTokenRemoteService.java

@@ -42,6 +42,11 @@ public class WxAccessTokenRemoteService {
     private Map<String, Map<String, String>> gzhConfig;
 
     public String getAccessToken(String gzhId) {
+        try {
+            return cache.getUnchecked(gzhId);
+        } catch (Exception e) {
+            log.error("get user error gzhId {} ", gzhId, e);
+        }
         return cache.getUnchecked(gzhId);
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/aigc/AIGCWaitingPublishContentService.java

@@ -29,7 +29,6 @@ public class AIGCWaitingPublishContentService {
 
 
     public List<Content> getAllContent(RecallParam param) {
-        long start = System.currentTimeMillis();
         List<Content> result = new ArrayList<>();
         JSONObject bodyParam = new JSONObject();
         JSONObject bodyParamParams = new JSONObject();
@@ -58,7 +57,6 @@ public class AIGCWaitingPublishContentService {
         } catch (Exception e) {
             log.error("getAllContent error", e);
         }
-        log.info("getAllContent耗时：{}", System.currentTimeMillis() - start);
         return result;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/aigc/PublishAccountRepository.java

@@ -14,4 +14,6 @@ public interface PublishAccountRepository extends JpaRepository<PublishAccount,
     PublishAccount getByGhId(String ghId);
 
     List<PublishAccount> getByIdIn(List<String> publishAccountIds);
+
+    PublishAccount getById(String publishAccountId);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/ArticleRepository.java

@@ -36,4 +36,6 @@ public interface ArticleRepository extends JpaRepository<Article, String> {
     int countByGhIdAndTypeAndItemIndex(String ghId, String val, Integer itemIndex);
 
     List<Article> getByGhIdInAndUpdateTimeGreaterThanAndTypeEquals(List<String> ghIds, long l, String number);
+
+    List<Article> getByTitleMd5(String titleMd5);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/PublishContentSortLogRepository.java

@@ -1,10 +0,0 @@
-package com.tzld.longarticle.recommend.server.repository.crawler;
-
-import com.tzld.longarticle.recommend.server.model.entity.crawler.PublishContentSortLog;
-import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.stereotype.Repository;
-
-@Repository
-public interface PublishContentSortLogRepository extends JpaRepository<PublishContentSortLog, Long> {
-
-}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/AccountCategoryRepository.java

@@ -9,7 +9,7 @@ import java.util.List;
 @Repository
 public interface AccountCategoryRepository extends JpaRepository<AccountCategory, AccountCategory.PK> {
 
-    AccountCategory getByGhIdAndStatus(String ghId, Integer status);
+    List<AccountCategory> getByGhIdAndStatus(String ghId, Integer status);
 
     List<AccountCategory> getByStatus(Integer status);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/ArticleTitleHisCacheRepository.java

@@ -0,0 +1,13 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleTitleHisCache;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+@Repository
+public interface ArticleTitleHisCacheRepository extends JpaRepository<ArticleTitleHisCache, ArticleTitleHisCache.PK> {
+
+    List<ArticleTitleHisCache> getBySourceIdInAndType(List<String> sourceIdList, String type);
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/ArticleUnsafeTitleRepository.java

@@ -0,0 +1,15 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleUnsafeTitle;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+@Repository
+public interface ArticleUnsafeTitleRepository extends JpaRepository<ArticleUnsafeTitle, String> {
+
+    ArticleUnsafeTitle getByTitleMd5(String titleMd5);
+
+    List<ArticleUnsafeTitle> getByStatus(Integer status);
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/LongArticleAuditDeleteRepository.java

@@ -9,4 +9,6 @@ import java.util.List;
 @Repository
 public interface LongArticleAuditDeleteRepository extends JpaRepository<LongArticleAuditDelete, Long> {
     List<LongArticleAuditDelete> getByStatus(Integer status);
+
+    List<LongArticleAuditDelete> getByPublishContentIdIn(List<String> publishContentIds);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/PublishSingleVideoSourceRepository.java

@@ -0,0 +1,12 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+@Repository
+public interface PublishSingleVideoSourceRepository extends JpaRepository<PublishSingleVideoSource, Long> {
+    List<PublishSingleVideoSource> getByContentTraceIdIn(List<String> contentTraceIds);
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/ArticleVideoPoolService.java

@@ -0,0 +1,22 @@
+package com.tzld.longarticle.recommend.server.service;
+
+import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource;
+import com.tzld.longarticle.recommend.server.model.param.ArticleVideoPoolSourceParam;
+import com.tzld.longarticle.recommend.server.model.vo.ArticleVideoPoolSourceVO;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service
+public class ArticleVideoPoolService {
+
+    @Autowired
+    private LongArticleBaseMapper longArticleBaseMapper;
+
+    public List<ArticleVideoPoolSourceVO> getSource(ArticleVideoPoolSourceParam param) {
+        List<PublishSingleVideoSource> list = longArticleBaseMapper.getPublishSingleVideoSource(param);
+        return ArticleVideoPoolSourceVO.convertFromPublishSingleVideoSource(list);
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/XxlJobService.java

@@ -8,31 +8,32 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.collect.Lists;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.tzld.longarticle.recommend.server.common.CommonThreadPoolExecutor;
+import com.tzld.longarticle.recommend.server.common.enums.aigc.ProduceContentAuditStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.AccountBusinessTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
 import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.growth.NewPushMessageCallbackMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
-import com.tzld.longarticle.recommend.server.model.dto.AccountTypeFansDTO;
-import com.tzld.longarticle.recommend.server.model.dto.NotPublishPlan;
-import com.tzld.longarticle.recommend.server.model.dto.PublishPlanAccountNotifyDTO;
+import com.tzld.longarticle.recommend.server.model.dto.*;
+import com.tzld.longarticle.recommend.server.model.entity.aigc.CrawlerPlan;
+import com.tzld.longarticle.recommend.server.model.entity.aigc.ProducePlan;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.GetOffVideoCrawler;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.LongArticlesVideo;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.GetOffVideoArticle;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesMatchVideo;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesReadRate;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesRootSourceId;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
 import com.tzld.longarticle.recommend.server.model.param.ArticleFindSourceParam;
+import com.tzld.longarticle.recommend.server.model.param.TitleHisCacheParam;
 import com.tzld.longarticle.recommend.server.remote.ODPSManager;
+import com.tzld.longarticle.recommend.server.repository.aigc.ProducePlanRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.GetOffVideoCrawlerRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.LongArticlesVideoRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.*;
 import com.tzld.longarticle.recommend.server.repository.model.PushMessageCallbackExample;
 import com.tzld.longarticle.recommend.server.service.recommend.ArticlePromotionService;
 import com.tzld.longarticle.recommend.server.service.recommend.ArticleService;
+import com.tzld.longarticle.recommend.server.service.recommend.recall.RecallService;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
 import com.tzld.longarticle.recommend.server.util.LarkRobotUtil;
 import com.tzld.longarticle.recommend.server.util.feishu.FeishuMessageSender;
@@ -42,14 +43,18 @@ import lombok.Getter;
 import lombok.Setter;
 import lombok.experimental.Accessors;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.dao.DuplicateKeyException;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
 import java.time.LocalTime;
 import java.util.*;
 import java.util.concurrent.*;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static com.tzld.longarticle.recommend.server.common.constant.TimeConstant.MILLISECOND_DAY;
@@ -85,6 +90,12 @@ public class XxlJobService {
     private ArticleService articleService;
     @Autowired
     private ArticlePromotionService articlePromotionService;
+    @Autowired
+    private ArticleTitleHisCacheRepository articleTitleHisCacheRepository;
+    @Autowired
+    private RecallService recallService;
+    @Autowired
+    private ProducePlanRepository producePlanRepository;
 
     ExecutorService thread = new CommonThreadPoolExecutor(
             5,
@@ -161,7 +172,8 @@ public class XxlJobService {
                 accountNames.remove(errorMsgDetail.getAccountName());
             }
         }
-        if (CollectionUtil.isEmpty(planErrorMsg.getErrorMsgDetails())) {
+        if (CollectionUtil.isEmpty(planErrorMsg.getErrorMsgDetails())
+                || CollectionUtil.isEmpty(accountNames)) {
             return;
         }
         FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
@@ -200,6 +212,87 @@ public class XxlJobService {
         return ReturnT.SUCCESS;
     }
 
+    @XxlJob("checkColdCrawlerPlan")
+    public ReturnT<String> checkColdCrawlerPlan(String param) {
+        long timeStamp = DateUtils.getBeforeDayStart(1);
+        if (StringUtils.hasText(param)) {
+            timeStamp = DateUtils.getStartOfDay(param, "yyyyMMdd");
+        }
+        String dateStr = DateUtils.timestampToYMDStr(timeStamp, "yyyyMMdd");
+        timeStamp = timeStamp * 1000;
+        List<String> planTags = Arrays.asList("账号联想_v1", "文章联想_v1", "品类冷启动");
+        List<CrawlerPlan> crawlerPlanList = aigcBaseMapper.getColdCrawlerPlan(timeStamp, timeStamp + 86400000, planTags);
+        if (CollectionUtil.isNotEmpty(crawlerPlanList)) {
+            for (CrawlerPlan crawlerPlan : crawlerPlanList) {
+                JSONObject crawlerModeValue = JSONObject.parseObject(crawlerPlan.getCrawlerModeValue());
+                if (crawlerModeValue == null) {
+                    continue;
+                }
+                JSONObject sourceModeValues = crawlerModeValue.getJSONObject("sourceModeValues");
+                if (sourceModeValues == null) {
+                    continue;
+                }
+                JSONArray inputModeValues = sourceModeValues.getJSONArray("inputModeValues");
+                if (inputModeValues == null) {
+                    continue;
+                }
+                log.info("checkColdCrawlerPlan crawlerPlan: {}", JSONObject.toJSONString(crawlerPlan));
+                FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.DAILY.getRobotId(),
+                        "【" + dateStr + "冷启计划抓取数量】\n"
+                                + "计划ID: " + crawlerPlan.getId() + "\n"
+                                + "计划名称: " + crawlerPlan.getName() + "\n"
+                                + "计划标签: " + crawlerPlan.getPlanTag() + "\n"
+                                + "计划抓取数量: " + inputModeValues.size() + "\n"
+                                + "实际抓取数量: " + crawlerPlan.getCrawlerTotalNum() + "\n"
+                                + "<at user_id=\"all\">所有人</at> ");
+            }
+        }
+        return ReturnT.SUCCESS;
+    }
+
+    @XxlJob("checkColdProducePlanAudit")
+    public ReturnT<String> checkColdProducePlanAudit(String param) {
+        long timeStamp = DateUtils.getBeforeDayStart(1);
+        if (StringUtils.hasText(param)) {
+            timeStamp = DateUtils.getStartOfDay(param, "yyyyMMdd");
+        }
+        timeStamp = timeStamp * 1000;
+        List<String> planIds = Arrays.asList("20241030070010871546586","20240802080355355308981","20240802021606053813696");
+        List<ProducePlan> producePlans = producePlanRepository.findByIdIn(planIds);
+        Map<String, String> producePlanMap = producePlans.stream().collect(Collectors.toMap(ProducePlan::getId, ProducePlan::getName));
+        List<ProducePlanAuditCheckDTO> producePlanList = aigcBaseMapper.getColdProducePlanAudit(
+                timeStamp, timeStamp + 86400000, planIds);
+        Map<String, List<ProducePlanAuditCheckDTO>> producePlanAuditMap = producePlanList.stream()
+                .collect(Collectors.groupingBy(ProducePlanAuditCheckDTO::getPlanId));
+        for (String planId : planIds) {
+            List<ProducePlanAuditCheckDTO> auditCheckDTOList = producePlanAuditMap.get(planId);
+            Integer yesterdayAuditNum = 0, yesterdayAuditPassNum = 0, waitingAuditNum = 0;
+            if (CollectionUtils.isNotEmpty(auditCheckDTOList)) {
+                for (ProducePlanAuditCheckDTO producePlanAuditCheckDTO : auditCheckDTOList) {
+                    if (Objects.equals(producePlanAuditCheckDTO.getAuditStatus(), ProduceContentAuditStatusEnum.waiting.getVal())) {
+                        waitingAuditNum = producePlanAuditCheckDTO.getNum();
+                    } else if (Objects.equals(producePlanAuditCheckDTO.getAuditStatus(), ProduceContentAuditStatusEnum.pass.getVal())) {
+                        yesterdayAuditPassNum = producePlanAuditCheckDTO.getNum();
+                        yesterdayAuditNum += producePlanAuditCheckDTO.getNum();
+                    } else if (Objects.equals(producePlanAuditCheckDTO.getAuditStatus(), ProduceContentAuditStatusEnum.reject.getVal())) {
+                        yesterdayAuditNum += producePlanAuditCheckDTO.getNum();
+                    }
+                }
+            }
+            log.info("checkColdProducePlanAudit planId: {}, planName: {}, yesterdayAuditNum: {}, " +
+                            "yesterdayAuditPassNum: {}, waitingAuditNum: {}",
+                    planId, producePlanMap.get(planId), yesterdayAuditNum, yesterdayAuditPassNum, waitingAuditNum);
+            FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.DAILY.getRobotId(),
+                    "【冷启生成计划审核数量】\n"
+                            + "planId: " + planId + "\n"
+                            + "planName: " + producePlanMap.get(planId) + "\n"
+                            + "前一天审核数量: " + yesterdayAuditNum + "\n"
+                            + "前一天审核通过数量: " + yesterdayAuditPassNum + "\n"
+                            + "当前待审核数量: " + waitingAuditNum);
+        }
+        return ReturnT.SUCCESS;
+    }
+
     @XxlJob("migrateCrawlerRootSourceId")
     public ReturnT<String> migrateCrawlerRootSourceId(String param) {
         try {
@@ -512,7 +605,59 @@ public class XxlJobService {
         articlePromotionService.articlePromotion("【1】", "direct", "头条利用池_v2",
                 "头条利用池_v4", 250, 1.21, Lists.newArrayList(2));
         articlePromotionService.articlePromotion("【2】", "direct", "次条利用池_v1",
-                "头条利用池_v4", 100, 1.33, Lists.newArrayList(3, 4, 5, 6, 7, 8));
+                "次条利用池_v4", 100, 1.33, Lists.newArrayList(3, 4, 5, 6, 7, 8));
+        return ReturnT.SUCCESS;
+    }
+
+    @XxlJob("refreshArticleHisCache")
+    public ReturnT<String> refreshArticleHisCache(String param) {
+        // 刷新历史表现缓存
+        long count = articleTitleHisCacheRepository.count();
+        int pageSize = 100;
+        long page = (count / pageSize) + 1;
+        for (int i = 0; i < page; i++) {
+            Page<ArticleTitleHisCache> articleTitleHisCachePage = articleTitleHisCacheRepository.findAll(
+                    PageRequest.of(i, pageSize));
+            List<ArticleTitleHisCache> cacheList = articleTitleHisCachePage.getContent();
+            if (CollectionUtils.isEmpty(cacheList)) {
+                continue;
+            }
+            Map<String, Map<String, ArticleTitleHisCache>> cacheMap = cacheList.stream().collect(
+                    Collectors.groupingBy(ArticleTitleHisCache::getType,
+                            Collectors.toMap(ArticleTitleHisCache::getSourceId, Function.identity())));
+            for (Map.Entry<String, Map<String, ArticleTitleHisCache>> typeEntry : cacheMap.entrySet()) {
+                String type = typeEntry.getKey();
+                Map<String, ArticleTitleHisCache> sourceIdToCacheMap = typeEntry.getValue();
+                Set<String> sourceIdList = sourceIdToCacheMap.keySet();
+                List<TitleHisCacheParam> paramList = sourceIdList.stream().map(sourceId -> {
+                    ArticleTitleHisCache cache = cacheMap.get(type).get(sourceId);
+                    TitleHisCacheParam cacheParam = new TitleHisCacheParam();
+                    cacheParam.setSourceId(sourceId);
+                    cacheParam.setTitleMd5(cache.getTitleMd5());
+                    cacheParam.setTitle(cache.getTitle());
+                    cacheParam.setCrawlerTitle(cache.getCrawlerTitle());
+                    cacheParam.setCrawlerChannelContentId(cache.getChannelContentId());
+                    if (StringUtils.hasText(cache.getCategory())) {
+                        cacheParam.setCategory(JSONArray.parseArray(cache.getCategory(), String.class));
+                    }
+                    return cacheParam;
+                }).collect(Collectors.toList());
+                Map<String, Content> articlesWithHistory = recallService.getArticleWithHistory(paramList, type);
+                for (String sourceId : sourceIdList) {
+                    Content content = articlesWithHistory.get(sourceId);
+                    if (Objects.nonNull(content) && CollectionUtils.isNotEmpty(content.getHisPublishArticleList())) {
+                        ArticleTitleHisCache cache = sourceIdToCacheMap.get(sourceId);
+                        cache.setHisPublishArticleList(JSONObject.toJSONString(content.getHisPublishArticleList()));
+                        if (CollectionUtil.isNotEmpty(content.getCategory())) {
+                            cache.setCategory(JSONObject.toJSONString(content.getCategory()));
+                        }
+                        cache.setUpdateTimestamp(System.currentTimeMillis());
+                        articleTitleHisCacheRepository.save(cache);
+                    }
+                }
+            }
+        }
+
         return ReturnT.SUCCESS;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/exterior/impl/GhDetailServiceImpl.java

@@ -124,9 +124,7 @@ public class GhDetailServiceImpl implements GhDetailService {
             }
             ghDetail.setGhId(ghDetailVo.getAccountId());
             ghDetail.setGhName(ghDetailVo.getAccountName());
-            if (!CollectionUtils.isEmpty(ghDetailVo.getVideoIds())) {
-                ghDetail.setVideoIds(JSONObject.toJSONString(ghDetailVo.getVideoIds()));
-            }
+            ghDetail.setVideoIds(JSONObject.toJSONString(ghDetailVo.getVideoIds()));
             ghDetailMapper.updateByPrimaryKeySelective(ghDetail);
             return CommonResponse.success();
         } catch (Exception e) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/exterior/impl/ThirdPartyServiceImpl.java

@@ -86,24 +86,24 @@ public class ThirdPartyServiceImpl implements ThirdPartyService {
         example.createCriteria().andTypeEqualTo(GhTypeEnum.THIRD_PARTY_GH.type).andGhIdEqualTo(param.getGhId());
         List<GhDetail> ghDetails = ghDetailMapper.selectByExample(example);
         if (CollectionUtils.isEmpty(ghDetails)) {
-            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "ghId不存在，请联系管理员配置");
+            return CommonResponse.create(404, "ghId不存在，请联系管理员配置");
         }
         String channel = ghDetails.get(0).getChannel();
         if (channel == null) {
             LarkRobotUtil.sendMessage("channel不存在，请查看详情 ghId=", param.getGhId());
-            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "ghId异常，请联系管理员检查");
+            return CommonResponse.create(404, "ghId异常，请联系管理员检查");
         }
         if (!Objects.equals(secretEnum.channel, channel)) {
             LarkRobotUtil.sendMessage(String.format("channel异常 secretEnum.channel=%s ghDetail.channel=%s ghId=%s",
                     secretEnum.channel, channel, param.getGhId()));
-            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "ghId异常，请联系管理员检查");
+            return CommonResponse.create(404, "ghId异常，请联系管理员检查");
         }
         List<PushMessageVo> pushMessageVoList = new ArrayList<>();
         ReplyBucketData replyBucketData = getPushMessageData(param, channel);
         log.info("replyBucketData={}", JSON.toJSONString(replyBucketData));
         if (replyBucketData == null) {
             log.error("获取推送策略数据失败");
-            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "系统异常，获取失败");
+            return CommonResponse.create(500, "系统异常，获取失败");
         }
         List<GroupData> groupList = replyBucketData.getGroupList();
         for (GroupData groupData : groupList) {
@@ -139,9 +139,13 @@ public class ThirdPartyServiceImpl implements ThirdPartyService {
         if (!DateUtils.isValidDate(canViewReportDate)) {
             return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "系统异常");
         }
-        //llf暂时不返回数据
+        //llf只返回2024-11-30后的数据
         if (secretEnum == SecretEnum.SECRET_ENUM_2) {
-            return CommonResponse.create(500, "数据不存在");
+            long targetTime = DateUtils.dateStrToTimestamp(date, "yyyy-MM-dd");
+            long limitTime = DateUtils.dateStrToTimestamp("2024-11-30", "yyyy-MM-dd");
+            if (targetTime < limitTime) {
+                return CommonResponse.create(500, "数据不存在");
+            }
         }
 
         //10点后可查询前一天数据

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleAuditService.java

@@ -2,31 +2,37 @@ package com.tzld.longarticle.recommend.server.service.recommend;
 
 import cn.hutool.core.collection.CollectionUtil;
 import com.alibaba.fastjson.JSONArray;
+import com.google.common.collect.Lists;
 import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PushTypeEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleDeleteStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
 import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
 import com.tzld.longarticle.recommend.server.model.dto.*;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
+import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
+import com.tzld.longarticle.recommend.server.model.param.ArticleDangerFindDeleteParam;
 import com.tzld.longarticle.recommend.server.model.param.ArticleVideoAuditResultParam;
+import com.tzld.longarticle.recommend.server.model.param.PublishContentParam;
 import com.tzld.longarticle.recommend.server.remote.WxAccessTokenRemoteService;
 import com.tzld.longarticle.recommend.server.remote.WxArticleDeleteService;
 import com.tzld.longarticle.recommend.server.remote.pq.PQVideoAuditStartProcessService;
 import com.tzld.longarticle.recommend.server.repository.aigc.PublishAccountRepository;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.*;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
+import com.tzld.longarticle.recommend.server.util.Md5Util;
 import com.tzld.longarticle.recommend.server.util.feishu.FeishuMessageSender;
 import com.xxl.job.core.biz.model.ReturnT;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -53,6 +59,10 @@ public class ArticleAuditService {
     @Autowired
     private PublishAccountRepository publishAccountRepository;
     @Autowired
+    private ArticleRepository articleRepository;
+    @Autowired
+    private ArticleUnsafeTitleRepository articleUnsafeTitleRepository;
+    @Autowired
     private PQVideoAuditStartProcessService pqVideoAuditStartProcessService;
 
 
@@ -110,12 +120,13 @@ public class ArticleAuditService {
         LongArticleVideoAudit longArticleVideoAudit = longArticleVideoAuditRepository.getByVideoId(param.getVideoId());
         if (param.getResult() == 1) {
             // 审核通过，更新文章状态
-            longArticleVideoAudit.setStatus(StatusEnum.ONE.getCode());
+            longArticleVideoAudit.setStatus(ArticleDeleteStatusEnum.SUCCESS.getCode());
             longArticleVideoAudit.setFinishTimestamp(System.currentTimeMillis());
             longArticleVideoAuditRepository.save(longArticleVideoAudit);
         } else {
             // 审核不通过，删除文章
-            longArticleVideoAudit.setStatus(StatusEnum.TWO.getCode());
+            longArticleVideoAudit.setStatus(ArticleDeleteStatusEnum.FAIL.getCode());
+            longArticleVideoAudit.setFailReason(param.getFailReason());
             longArticleVideoAudit.setFinishTimestamp(System.currentTimeMillis());
             longArticleVideoAuditRepository.save(longArticleVideoAudit);
             // 构建删除文章记录 并保存
@@ -128,16 +139,32 @@ public class ArticleAuditService {
         List<String> traceIds = crawlerVideoList.stream().map(LongArticleCrawlerVideo::getTraceId).collect(Collectors.toList());
         List<PublishContentMiniprogramDTO> publishContentList = aigcBaseMapper.getPublishContentByTraceIdIn(traceIds);
         List<String> publishContentIds = publishContentList.stream().map(PublishContentMiniprogramDTO::getPublishContentId).collect(Collectors.toList());
-        Map<String, String> publishContentIdMap = publishContentList.stream()
-                .collect(Collectors.toMap(PublishContentMiniprogramDTO::getTraceId, PublishContentMiniprogramDTO::getPublishContentId));
+        buildArticleAuditDelete(publishContentIds);
+    }
+
+    private void buildArticleAuditDelete(List<String> publishContentIds) {
+        if (CollectionUtils.isEmpty(publishContentIds)) {
+            return;
+        }
+        List<LongArticleAuditDelete> existList = longArticleAuditDeleteRepository.getByPublishContentIdIn(publishContentIds);
+        List<String> existContentIds = existList.stream().map(LongArticleAuditDelete::getPublishContentId).collect(Collectors.toList());
+        publishContentIds = publishContentIds.stream().filter(id -> !existContentIds.contains(id)).collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(publishContentIds)) {
+            return;
+        }
         List<PublishGzhPushContentRelDTO> pushContentRelList = aigcBaseMapper.getPushContentRelByPublishContentIdIn(publishContentIds);
+        if (CollectionUtils.isEmpty(pushContentRelList)) {
+            return;
+        }
         List<String> pushIds = pushContentRelList.stream().map(PublishGzhPushContentRelDTO::getPushId).collect(Collectors.toList());
         Map<String, String> publishPushIdMap = pushContentRelList.stream()
-                .collect(Collectors.toMap(PublishGzhPushContentRelDTO::getPublishContentId, PublishGzhPushContentRelDTO::getPushId));
+                .collect(Collectors.toMap(PublishGzhPushContentRelDTO::getPublishContentId,
+                        PublishGzhPushContentRelDTO::getPushId,
+                         (o1, o2) -> o2));
         List<PublishGzhPushDTO> pushList = aigcBaseMapper.getPushByPushIdIn(pushIds);
         Map<String, PublishGzhPushDTO> pushDTOMap = pushList.stream()
                 .collect(Collectors.toMap(PublishGzhPushDTO::getPushId, Function.identity()));
-        Map<String, String> pushIdMap = pushList.stream()
+        Map<String, String> pushIdMap = pushList.stream().filter(o -> StringUtils.hasText(o.getGroupPushMsgId()))
                 .collect(Collectors.toMap(PublishGzhPushDTO::getPushId, PublishGzhPushDTO::getGroupPushMsgId));
         List<String> publishAccountIds = pushList.stream().map(PublishGzhPushDTO::getPublishAccountId).collect(Collectors.toList());
         Map<String, String> pushAccountMap = pushList.stream()
@@ -148,18 +175,22 @@ public class ArticleAuditService {
         List<PublishGzhPushContentRelDTO> groupPushRelList = aigcBaseMapper.getGroupPushRelByPushIdIn(pushIds);
         Map<String, List<PublishGzhPushContentRelDTO>> groupPushRelMap = groupPushRelList.stream()
                 .collect(Collectors.groupingBy(PublishGzhPushContentRelDTO::getPushId));
-        for (String traceId : traceIds) {
-            String publishContentId = publishContentIdMap.get(traceId);
+        // 删除文章
+        for (String publishContentId : publishContentIds) {
             String pushId = publishPushIdMap.get(publishContentId);
             if (!StringUtils.hasText(pushId)) {
-                // todo 未发布文章如何进行小程序替换
+                deleteFailAlarm(publishContentId, "无推送记录", 0);
                 continue;
             }
             PublishGzhPushDTO publishGzhPushDTO = pushDTOMap.get(pushId);
-            String groupPushMsgId = pushIdMap.get(pushId);
+            if (Objects.isNull(publishGzhPushDTO)) {
+                deleteFailAlarm(publishContentId, "无推送记录", 0);
+                continue;
+            }
             String publishAccountId = pushAccountMap.get(pushId);
             String ghId = publishAccountMap.get(publishAccountId);
-            List<PublishGzhPushContentRelDTO> relList = groupPushRelMap.get(pushId);
+            List<PublishGzhPushContentRelDTO> relList = groupPushRelMap.get(pushId).stream()
+                    .sorted(Comparator.comparing(PublishGzhPushContentRelDTO::getId)).collect(Collectors.toList());
             int index = 1;
             for (int i = 0; i < relList.size(); i++) {
                 if (relList.get(i).getPublishContentId().equals(publishContentId)) {
@@ -167,6 +198,16 @@ public class ArticleAuditService {
                     break;
                 }
             }
+            if (!publishGzhPushDTO.getPushType().equals(PushTypeEnum.AUTO_GROUP_PUBLISH.getVal())) {
+                PushTypeEnum pushTypeEnum = PushTypeEnum.from(publishGzhPushDTO.getPushType());
+                deleteFailAlarm(publishContentId, "推送类型为" + pushTypeEnum.getDescription(), index);
+                continue;
+            }
+            String groupPushMsgId = pushIdMap.get(pushId);
+            if (!StringUtils.hasText(groupPushMsgId)) {
+                deleteFailAlarm(publishContentId, "无推送MsgId", index);
+                continue;
+            }
             LongArticleAuditDelete delete = new LongArticleAuditDelete();
             delete.setGhId(ghId);
             delete.setMsgId(groupPushMsgId);
@@ -185,36 +226,84 @@ public class ArticleAuditService {
      */
     @XxlJob("articleVideoDelete")
     public ReturnT<String> articleVideoDelete(String param) {
-        List<LongArticleAuditDelete> dealList = longArticleAuditDeleteRepository.getByStatus(StatusEnum.ZERO.getCode());
+        List<LongArticleAuditDelete> dealList = longArticleAuditDeleteRepository.getByStatus(ArticleDeleteStatusEnum.WAITING.getCode());
         for (LongArticleAuditDelete delete : dealList) {
-            if (Objects.equals(delete.getPushType(), PushTypeEnum.AUTO_GROUP_PUBLISH.getVal())) {
-                // 获取token
-                String token = wxAccessTokenRemoteService.getAccessToken(delete.getGhId());
-                // 删除文章
-                RequestResult<String> result = wxArticleDeleteService.deleteArticle(token, delete.getMsgId(), delete.getIndex());
-                if (result.isSuccess()) {
-                    delete.setStatus(StatusEnum.ONE.getCode());
+            try {
+                if (Objects.equals(delete.getPushType(), PushTypeEnum.AUTO_GROUP_PUBLISH.getVal())) {
+                    // 获取token
+                    String token = wxAccessTokenRemoteService.getAccessToken(delete.getGhId());
+                    // 删除文章
+                    RequestResult<String> result = wxArticleDeleteService.deleteArticle(token, delete.getMsgId(), delete.getIndex());
+                    if (result.isSuccess()) {
+                        delete.setStatus(ArticleDeleteStatusEnum.SUCCESS.getCode());
+                    } else {
+                        delete.setStatus(ArticleDeleteStatusEnum.FAIL.getCode());
+                        delete.setFailReason(result.getFailReason());
+                    }
                 } else {
-                    delete.setStatus(StatusEnum.TWO.getCode());
-                    delete.setFinishTimestamp(System.currentTimeMillis());
+                    deleteFailAlarm(delete.getPublishContentId(), "非自动群发", delete.getIndex());
+                    delete.setStatus(ArticleDeleteStatusEnum.SUCCESS.getCode());
                 }
+                delete.setFinishTimestamp(System.currentTimeMillis());
                 longArticleAuditDeleteRepository.save(delete);
-            } else {
-                PublishAccount publishAccount = publishAccountRepository.getByGhId(delete.getGhId());
-                PublishContentDTO publishContent = aigcBaseMapper.getPublishContentById(delete.getPublishContentId());
-                String publishTime = DateUtils.timestampToYMDStr(publishContent.getPublishTimestamp() / 1000, "yyyyMMdd");
-                // 无法自动删除 发送飞书通知 人工删除
-                FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.ARTICLE_DELETE.getRobotId(),
-                        "视频审核不通过文章删除\n"
-                                + "账号名称: " + publishAccount.getName() + "\n"
-                                + "发布日期: " + publishTime + "\n"
-                                + "位置: " + delete.getIndex() + "\n"
-                                + "标题: " + publishContent.getTitle());
-
-                delete.setStatus(StatusEnum.ONE.getCode());
-                longArticleAuditDeleteRepository.save(delete);
+            } catch (Exception e) {
+                log.error("articleVideoDelete ghId:{} error", delete.getGhId(), e);
             }
         }
         return ReturnT.SUCCESS;
     }
+
+    private void deleteFailAlarm(String publishContentId, String typeMsg, Integer index) {
+        PublishContentDTO publishContent = aigcBaseMapper.getPublishContentById(publishContentId);
+        PublishAccount publishAccount = publishAccountRepository.getById(publishContent.getPublishAccountId());
+        String publishTime = DateUtils.timestampToYMDStr(publishContent.getPublishTimestamp() / 1000, "yyyyMMdd");
+        // 无法自动删除 发送飞书通知 人工删除
+        FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.ARTICLE_DELETE.getRobotId(),
+                "【文章删除失败】\n"
+                        + "账号名称: " + publishAccount.getName() + "\n"
+                        + "账号ghId: " + publishAccount.getGhId() + "\n"
+                        + "发布日期: " + publishTime + "\n"
+                        + "位置: " + index + "\n"
+                        + "标题: " + publishContent.getTitle() + "\n"
+                        + "原因: " + typeMsg);
+    }
+
+    public void titleDangerFindDelete(ArticleDangerFindDeleteParam param) {
+        String titleMd5 = Md5Util.encoderByMd5(param.getTitle());
+        // 根据标题查找已发布文章
+        List<Article> articleList = articleRepository.getByTitleMd5(titleMd5);
+        List<String> ghIds = articleList.stream().map(Article::getGhId).distinct().collect(Collectors.toList());
+        List<PublishAccount> publishAccountList = publishAccountRepository.getAllByGhIdIn(ghIds);
+        List<PublishContentParam> publishContentParamList = publishAccountList.stream().map(account -> {
+            PublishContentParam item = new PublishContentParam();
+            item.setTitle(param.getTitle());
+            item.setPublishAccountId(account.getId());
+            return item;
+        }).collect(Collectors.toList());
+        List<PublishContentDTO> publishContents = new ArrayList<>();
+        for (List<PublishContentParam> partitions : Lists.partition(publishContentParamList, 100)) {
+            publishContents.addAll(aigcBaseMapper.getPublishContentByTitle(partitions));
+        }
+        if (CollectionUtils.isEmpty(publishContents)) {
+            return;
+        }
+        // 查找该生成内容下所有已发布内容
+        List<String> sourceIds = publishContents.stream().map(PublishContentDTO::getSourceId).distinct().collect(Collectors.toList());
+        publishContents = aigcBaseMapper.getPublishContentBySourceIdIn(sourceIds);
+
+        // 根据已发布文章查找推送id 并删除
+        List<String> publishContentIds = publishContents.stream().map(PublishContentDTO::getId).collect(Collectors.toList());
+        buildArticleAuditDelete(publishContentIds);
+        buildArticleUnsafeTitle(titleMd5, param.getTitle());
+    }
+
+    private void buildArticleUnsafeTitle(String titleMd5, String title) {
+        ArticleUnsafeTitle unsafeTitle = new ArticleUnsafeTitle();
+        unsafeTitle.setTitle(title);
+        unsafeTitle.setTitleMd5(titleMd5);
+        unsafeTitle.setStatus(StatusEnum.ONE.getCode());
+        unsafeTitle.setCreateTimestamp(System.currentTimeMillis());
+        articleUnsafeTitleRepository.save(unsafeTitle);
+    }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticlePromotionService.java

@@ -1,7 +1,10 @@
 package com.tzld.longarticle.recommend.server.service.recommend;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.CrawlerModeEnum;
+import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticlePoolPromotionSourceStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
@@ -88,7 +91,8 @@ public class ArticlePromotionService {
         list = list.stream().filter(o -> {
             long publishTime = DateUtils.dateStrToTimestamp(o.getDateStr(), "yyyyMMdd");
             Long accountCreateTime = publishAccountCreateTimeMap.get(o.getGhId());
-            return publishTime * 1000 > accountCreateTime;
+            return publishTime * 1000 > accountCreateTime
+                    && !PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal().equals(o.getSourceType());
         }).collect(Collectors.toList());
         return list;
     }
@@ -118,6 +122,8 @@ public class ArticlePromotionService {
             aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, urlList, tag, CrawlerModeEnum.ContentIDs.getVal());
             return;
         }
+        List<String> publishContentIds = new ArrayList<>();
+        List<String> filterUrlList = new ArrayList<>();
         try {
             String level = pos.equals("【1】") ? contentPoolType.get(0) : contentPoolType.get(1);
             String produceId = produceConfig.get(accountNickName).get(pos).get(way).trim();
@@ -129,8 +135,6 @@ public class ArticlePromotionService {
             Set<String> visitedUrlIdList = contentList.stream().map(content -> getUrlId(content.getReferContentLink()))
                     .collect(Collectors.toSet());
             // 筛选URL和标题
-            List<String> publishContentIds = new ArrayList<>();
-            List<String> filterUrlList = new ArrayList<>();
             for (DatastatSortStrategy item : list) {
                 String url = item.getLink();
                 String urlId = getUrlId(item.getLink());
@@ -142,45 +146,56 @@ public class ArticlePromotionService {
                 if (TitleSimilarCheckUtil.isDuplicateContent(title, visitedTitleList, TitleSimilarCheckUtil.ARTICLE_PROMOTION_THRESHOLD)) {
                     continue;
                 }
-                filterUrlList.add(url);
-                // 调用爬虫 detail 接口并保存数据
-                WxContentDetailResponse detail = getArticleDetail(url);
-                if (detail != null && StringUtils.hasText(detail.getChannelContentId())) {
-                    saveArticlePoolPromotionSource(detail.getChannelContentId(), wxSn, title, level);
+                // 能匹配到aigc发布记录则使用id抓取  否则使用url抓取
+                String publishContentId = articleService.getPublishContentByWxSn(wxSn);
+                if (StringUtils.hasText(publishContentId)) {
+                    publishContentIds.add(publishContentId);
+                    saveArticlePoolPromotionSource(Md5Util.encoderByMd5(publishContentId), wxSn, title, level);
                 } else {
-                    String publishContentId = articleService.getPublishContentByWxSn(wxSn);
-                    if (StringUtils.hasText(publishContentId)) {
-                        publishContentIds.add(publishContentId);
-                        saveArticlePoolPromotionSource(Md5Util.encoderByMd5(publishContentId), wxSn, title, level);
+                    filterUrlList.add(url);
+                    // 调用爬虫 detail 接口并保存数据
+                    WxContentDetailResponse detail = getArticleDetail(url);
+                    if (detail != null && StringUtils.hasText(detail.getChannelContentId())) {
+                        saveArticlePoolPromotionSource(detail.getChannelContentId(), wxSn, title, level);
                     }
                 }
             }
-            if (filterUrlList.isEmpty()) {
-                log.info("url_list empty: " + accountNickName + ", " + pos + ", " + way);
-                return;
-            }
-            String planName = String.format("%d_%s_%s_%s【%s】_%s", filterUrlList.size(), today, accountNickName, pos, way, today);
-            log.info("url_len: " + list.size() + ", " + filterUrlList.size());
-            IdNameVO<String> planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, filterUrlList, tag, CrawlerModeEnum.ContentIDs.getVal());
-            if (StringUtils.hasText(produceId)) {
-                articleAddDependPlan(produceId, planInfo.getId(), planInfo.getName());
+            if (CollectionUtils.isNotEmpty(filterUrlList)) {
+                String planName = String.format("%d_%s_%s_%s【%s】_%s", filterUrlList.size(), today, accountNickName, pos, way, today);
+                log.info("url_len: " + list.size() + ", " + filterUrlList.size());
+                IdNameVO<String> planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, filterUrlList, tag, CrawlerModeEnum.ContentIDs.getVal());
+                if (StringUtils.hasText(produceId)) {
+                    articleAddDependPlan(produceId, planInfo.getId(), planInfo.getName());
+                }
+                log.info("{}, {}, produce plan not exist: {}, {}, {}", planInfo.getName(), planInfo.getId(), accountNickName, pos, way);
             }
-            log.info("{}, {}, produce plan not exist: {}, {}, {}", planInfo.getName(), planInfo.getId(), accountNickName, pos, way);
             if (CollectionUtils.isNotEmpty(publishContentIds)) {
-                planName = String.format("%d_%s_%s_%s【%s】_%s", publishContentIds.size(), today, accountNickName, pos, way, today);
-                planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, publishContentIds, tag, CrawlerModeEnum.PublishContentIds.getVal());
+                String planName = String.format("%d_%s_%s_%s【%s】_%s", publishContentIds.size(), today, accountNickName, pos, way, today);
+                IdNameVO<String> planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, publishContentIds, tag, CrawlerModeEnum.PublishContentIds.getVal());
                 if (StringUtils.hasText(produceId)) {
                     articleAddDependPlan(produceId, planInfo.getId(), planInfo.getName());
                 }
             }
+            sendFeishuJobFinishMessage(accountNickName, filterUrlList.size(), publishContentIds.size());
         } catch (Exception e) {
             log.error("articlePromotion error: ", e);
-            FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.JOB.getRobotId(),
-                    "文章晋升ERROR:\n" +
-                            "articlePromotion error: " + e.getMessage());
+            FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.DAILY.getRobotId(),
+                    "【文章晋升ERROR】\n" +
+                            "晋级任务：" + accountNickName + "\n" +
+                            "articlePromotion error: " + e.getMessage() + "\n"
+                            + "<at user_id=\"all\">所有人</at> ");
         }
     }
 
+    private void sendFeishuJobFinishMessage(String accountNickName, Integer urlListSize, Integer contentListSize) {
+        log.info("articlePromotion finish: 晋级任务:{}, id晋级数量：{}, url晋级数量：{}", accountNickName, contentListSize, urlListSize);
+        FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.DAILY.getRobotId(),
+                "【文章晋级job完成】\n" +
+                        "晋级任务：" + accountNickName + "\n" +
+                        "id晋级数量：" + contentListSize + "\n" +
+                        "url晋级数量：" + urlListSize + "\n");
+    }
+
     private List<ProduceContentListItemVO> getProduceContentList(String accountNickName, String pos, String way) {
         List<String> planIdList = getProducePlanIdList(accountNickName, pos, way);
         CommonListDataVO<ProduceContentListItemVO> contentData = getProduceContentListByPlanIdList(planIdList);
@@ -303,6 +318,8 @@ public class ArticlePromotionService {
             articlePromotion.setTitle(title);
             articlePromotion.setTitleMd5(Md5Util.encoderByMd5(title));
             articlePromotion.setCreateTimestamp(System.currentTimeMillis());
+            articlePromotion.setStatus(ArticlePoolPromotionSourceStatusEnum.WAITING.getCode());
+            articlePromotion.setDeleted(StatusEnum.ZERO.getCode());
         }
         articlePromotion.setWxSn(wxSn);
         articlePromotion.setLevel(level);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleService.java

@@ -42,6 +42,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
@@ -88,6 +89,11 @@ public class ArticleService {
             "\"20240805154433785506170\", \"20240805154359027876170\", \"20241024100016206421084\", " +
             "\"20241030070010871546586\"]}")
     private static List<String> producePlanIds;
+    @Value("${kimiCategoryPrompt:}")
+    private String kimiCategoryPrompt;
+
+    @ApolloJsonValue("${articlePromotionProduceConfig:{}}")
+    private Map<String, Map<String, Map<String, String>>> produceConfig;
 
     private final static ExecutorService pool = new CommonThreadPoolExecutor(
             32,
@@ -150,6 +156,13 @@ public class ArticleService {
         if (Objects.nonNull(publishContent)) {
             publishContentId = publishContent.getId();
             channelContentId = publishContent.getCrawlerChannelContentId();
+        } else {
+            publishContentList = aigcBaseMapper.getLateNearestPublishContent(publishAccount.getId(), publishTimestamp);
+            publishContent = findPublishContent(publishContentList, article.getTitle(), publishTimestamp);
+            if (Objects.nonNull(publishContent)) {
+                publishContentId = publishContent.getId();
+                channelContentId = publishContent.getCrawlerChannelContentId();
+            }
         }
         log.info("syncAigcIdByWxSn titleMatch finish");
         if (Objects.isNull(channelContentId)) {
@@ -169,6 +182,9 @@ public class ArticleService {
     private PublishContent findPublishContent(List<PublishContent> publishContentList,
                                               String title,
                                               Long publishTimestamp) {
+        if (CollectionUtils.isEmpty(publishContentList)) {
+            return null;
+        }
         Map<String, PublishContent> publishContentMap = publishContentList.stream().collect(
                 Collectors.toMap(PublishContent::getId, publishContent -> publishContent));
         List<String> publishContentIds = publishContentList.stream().map(PublishContent::getId).collect(Collectors.toList());
@@ -259,6 +275,17 @@ public class ArticleService {
             result.setRootPublishContentId(publishContent.getId());
             result.setRootProduceContentId(publishContent.getSourceId());
             channelContentId = publishContent.getCrawlerChannelContentId();
+        } else {
+            publishContentList = aigcBaseMapper.getLateNearestPublishContent(publishAccount.getId(), publishTimestamp);
+            publishContent = findPublishContent(publishContentList, title, publishTimestamp);
+            if (Objects.nonNull(publishContent)) {
+                if (!StringUtils.hasText(sourcePublishContentId)) {
+                    result.setSourcePublishContentId(publishContent.getId());
+                }
+                result.setRootPublishContentId(publishContent.getId());
+                result.setRootProduceContentId(publishContent.getSourceId());
+                channelContentId = publishContent.getCrawlerChannelContentId();
+            }
         }
         // channelContentId未被修改，说明未找到
         if (channelContentId.equals(crawlerContent.getChannelContentId())) {
@@ -284,72 +311,85 @@ public class ArticleService {
         }
         long now = System.currentTimeMillis();
         for (ArticlePoolPromotionSource task : tasks) {
-            // 判断文章是否被抓回来，如果没有抓回来，不进行处理 临时方案
-            CrawlerContent crawlerContent = aigcBaseMapper.getCrawlerContentByChannelContentId(task.getChannelContentId());
-            if (Objects.isNull(crawlerContent)) {
-                continue;
-            }
-            // 溯源
-            Article article = articleRepository.getByWxSn(task.getWxSn());
-            if (Objects.isNull(article)) {
-                task.setDeleted(StatusEnum.ONE.getCode());
-                articlePoolPromotionSourceRepository.save(task);
-                continue;
-            }
-            PublishAccount publishAccount = publishAccountRepository.getByGhId(article.getGhId());
-            if (Objects.isNull(publishAccount)) {
-                continue;
-            }
-            long publishTimestamp = article.getPublishTimestamp() > 0 ? article.getPublishTimestamp() * 1000 : article.getUpdateTime() * 1000;
-            List<PublishContent> publishContentList = aigcBaseMapper.getNearestPublishContent(publishAccount.getId(), publishTimestamp, 100);
-            PublishContent publishContent = findPublishContent(publishContentList, task.getTitle(), publishTimestamp);
-            if (Objects.isNull(publishContent)) {
-                task.setDeleted(StatusEnum.ONE.getCode());
-                articlePoolPromotionSourceRepository.save(task);
-                continue;
-            }
-            RootPublishContentVO source = getRootPublishContent(publishContent.getCrawlerChannelContentId(), null, publishContent.getId(), publishContent.getSourceId(), 0);
-            // 更新
-            if (StringUtils.hasText(source.getRootProduceContentId())) {
-                task.setStatus(ArticlePoolPromotionSourceStatusEnum.FINISH.getCode());
-                task.setSourcePublishContentId(source.getSourcePublishContentId());
-                task.setRootProduceContentId(source.getRootProduceContentId());
-                task.setRootPublishContentId(source.getRootPublishContentId());
-                task.setUpdateTimestamp(now);
-                articlePoolPromotionSourceRepository.save(task);
-                // 保存中间环节 晋级溯源
-                for (String midChannelContentId : source.getMidChannelContentIds()) {
-                    ArticlePoolPromotionSource item = new ArticlePoolPromotionSource();
-                    BeanUtils.copyProperties(task, item);
-                    ArticlePoolPromotionSource dto = articlePoolPromotionSourceRepository.getByChannelContentId(midChannelContentId);
-                    if (Objects.nonNull(dto)) {
-                        // 以dto为基础
-                        dto.setRootProduceContentId(task.getRootProduceContentId());
-                        dto.setRootPublishContentId(task.getRootPublishContentId());
-                        dto.setStatus(task.getStatus());
-                        dto.setLevel(task.getLevel());
-                        dto.setUpdateTimestamp(task.getUpdateTimestamp());
-                        item = dto;
-                    } else {
-                        // 以新item为基础
-                        item.setChannelContentId(midChannelContentId);
-                        item.setCreateTimestamp(item.getUpdateTimestamp());
+            try {
+                // 判断文章是否被抓回来，如果没有抓回来，不进行处理 临时方案
+                CrawlerContent crawlerContent = aigcBaseMapper.getCrawlerContentByChannelContentId(task.getChannelContentId());
+                if (Objects.isNull(crawlerContent)) {
+                    continue;
+                }
+                // 溯源
+                Article article = articleRepository.getByWxSn(task.getWxSn());
+                if (Objects.isNull(article)) {
+                    task.setDeleted(StatusEnum.ONE.getCode());
+                    articlePoolPromotionSourceRepository.save(task);
+                    continue;
+                }
+                PublishAccount publishAccount = publishAccountRepository.getByGhId(article.getGhId());
+                if (Objects.isNull(publishAccount)) {
+                    continue;
+                }
+                long publishTimestamp = article.getPublishTimestamp() > 0 ? article.getPublishTimestamp() * 1000 : article.getUpdateTime() * 1000;
+                List<PublishContent> publishContentList = aigcBaseMapper.getNearestPublishContent(publishAccount.getId(), publishTimestamp, 100);
+                PublishContent publishContent = findPublishContent(publishContentList, task.getTitle(), publishTimestamp);
+                if (Objects.isNull(publishContent)) {
+                    publishContentList = aigcBaseMapper.getLateNearestPublishContent(publishAccount.getId(), publishTimestamp);
+                    publishContent = findPublishContent(publishContentList, task.getTitle(), publishTimestamp);
+                    if (Objects.isNull(publishContent)) {
+                        task.setDeleted(StatusEnum.ONE.getCode());
+                        articlePoolPromotionSourceRepository.save(task);
+                        continue;
                     }
-                    item.setDeleted(0);
-                    articlePoolPromotionSourceRepository.save(item);
                 }
-                longArticleBaseMapper.updateRootProduceContentLevel(task.getRootProduceContentId(), task.getLevel());
-            } else {
-                task.setDeleted(StatusEnum.ONE.getCode());
-                articlePoolPromotionSourceRepository.save(task);
+                RootPublishContentVO source = getRootPublishContent(publishContent.getCrawlerChannelContentId(), null, publishContent.getId(), publishContent.getSourceId(), 0);
+                // 更新
+                if (StringUtils.hasText(source.getRootProduceContentId())) {
+                    task.setStatus(ArticlePoolPromotionSourceStatusEnum.FINISH.getCode());
+                    task.setSourcePublishContentId(source.getSourcePublishContentId());
+                    task.setRootProduceContentId(source.getRootProduceContentId());
+                    task.setRootPublishContentId(source.getRootPublishContentId());
+                    task.setUpdateTimestamp(now);
+                    articlePoolPromotionSourceRepository.save(task);
+                    // 保存中间环节 晋级溯源
+                    for (String midChannelContentId : source.getMidChannelContentIds()) {
+                        ArticlePoolPromotionSource item = new ArticlePoolPromotionSource();
+                        BeanUtils.copyProperties(task, item);
+                        ArticlePoolPromotionSource dto = articlePoolPromotionSourceRepository.getByChannelContentId(midChannelContentId);
+                        if (Objects.nonNull(dto)) {
+                            // 以dto为基础
+                            dto.setRootProduceContentId(task.getRootProduceContentId());
+                            dto.setRootPublishContentId(task.getRootPublishContentId());
+                            dto.setStatus(task.getStatus());
+                            dto.setLevel(task.getLevel());
+                            dto.setUpdateTimestamp(task.getUpdateTimestamp());
+                            item = dto;
+                        } else {
+                            // 以新item为基础
+                            item.setChannelContentId(midChannelContentId);
+                            item.setCreateTimestamp(item.getUpdateTimestamp());
+                        }
+                        item.setDeleted(StatusEnum.ZERO.getCode());
+                        articlePoolPromotionSourceRepository.save(item);
+                    }
+                    longArticleBaseMapper.updateRootProduceContentLevel(task.getRootProduceContentId(), task.getLevel());
+                } else {
+                    task.setDeleted(StatusEnum.ONE.getCode());
+                    articlePoolPromotionSourceRepository.save(task);
+                }
+            } catch (Exception e) {
+                log.error("articlePromotionTraceability error channelContentId:{}", task.getChannelContentId(), e);
             }
         }
     }
 
 
     public void articleCategory() {
+        // 根据抓取计划 添加品类处理任务
         addArticleCategoryByCrawlerPlan();
-        addArticleCategoryByProducePlan();
+        // 冷启层 生成计划 添加品类处理任务
+        addColdArticleCategoryByProducePlan();
+        // 晋级 生成计划 添加品类处理任务
+        addPromotionArticleCategoryByProducePlan();
+        // 调用kimi进行内容分类
         dealArticleCategory();
     }
 
@@ -417,11 +457,74 @@ public class ArticleService {
         }
     }
 
-    private void addArticleCategoryByProducePlan() {
+    private void addColdArticleCategoryByProducePlan() {
+        List<ArticleCategory> saveList = addArticleCategoryByProducePlan(producePlanIds);
+        if (CollectionUtils.isNotEmpty(saveList)) {
+            longArticleBaseMapper.batchInsertArticleCategory(saveList);
+        }
+    }
+
+    private void addPromotionArticleCategoryByProducePlan() {
+        List<String> articlePromotionProducePlanIds = new ArrayList<>();
+        // 获取晋级生成计划Id
+        for (Map.Entry<String, Map<String, Map<String, String>>> oneEntry : produceConfig.entrySet()) {
+            for (Map.Entry<String, Map<String, String>> twoEntry : oneEntry.getValue().entrySet()) {
+                twoEntry.getValue().forEach((key, value) -> {
+                    if (StringUtils.hasText(value) && !producePlanIds.contains(value)) {
+                        articlePromotionProducePlanIds.add(value);
+                    }
+                });
+            }
+        }
+        List<ArticleCategory> saveList = addArticleCategoryByProducePlan(articlePromotionProducePlanIds);
+        // 已晋级文章 先溯源查找源内容品类，查询不到再用kimi进行分类
+        if (CollectionUtils.isNotEmpty(saveList)) {
+            List<String> channelContentIds = saveList.stream().map(ArticleCategory::getChannelContentId)
+                    .collect(Collectors.toList());
+            // 查询晋升rootProduceContentId
+            List<ArticlePoolPromotionSource> sourceList = articlePoolPromotionSourceRepository
+                    .getByChannelContentIdInAndStatusAndDeleted(channelContentIds,
+                            ArticlePoolPromotionSourceStatusEnum.FINISH.getCode(), 0);
+            Map<String, ArticlePoolPromotionSource> sourceMap = sourceList.stream()
+                    .collect(Collectors.toMap(ArticlePoolPromotionSource::getChannelContentId, Function.identity()));
+            // 根据produceContentId查询category
+            List<ArticleCategory> articleCategoryList = articleCategoryRepository.getByStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
+            Map<String, ArticleCategory> categoryMap = articleCategoryList.stream()
+                    .collect(Collectors.toMap(ArticleCategory::getProduceContentId, Function.identity()));
+            Map<String, ArticleCategory> coldStartCategoryMap = articleCategoryList.stream()
+                    .collect(Collectors.toMap(ArticleCategory::getChannelContentId, Function.identity(), (a, b) -> a));
+            Map<String, ArticleCategory> titleCategoryMap = articleCategoryList.stream()
+                    .collect(Collectors.toMap(ArticleCategory::getTitleMd5, Function.identity(), (a, b) -> a));
+            for (ArticleCategory articleCategory : saveList) {
+                ArticlePoolPromotionSource source = sourceMap.get(articleCategory.getChannelContentId());
+                ArticleCategory category = null;
+                if (Objects.nonNull(source) && Objects.nonNull(source.getRootProduceContentId())) {
+                    category = categoryMap.get(source.getRootProduceContentId());
+                }
+                if (Objects.isNull(category)) {
+                    category = coldStartCategoryMap.get(articleCategory.getChannelContentId());
+                }
+                if (Objects.isNull(category)) {
+                    category = titleCategoryMap.get(articleCategory.getTitleMd5());
+                }
+                if (Objects.nonNull(category) && StringUtils.hasText(category.getCategory())) {
+                    articleCategory.setCategory(category.getCategory());
+                    articleCategory.setKimiResult(category.getKimiResult());
+                    articleCategory.setStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
+                }
+            }
+            longArticleBaseMapper.batchInsertArticleCategory(saveList);
+        }
+    }
+
+    /**
+     * 根据生成计划获取需要进行分类内容
+     */
+    private List<ArticleCategory> addArticleCategoryByProducePlan(List<String> producePlanIds) {
         List<ProducePlanExeRecord> produceContentList = aigcBaseMapper.getAllByProducePlanId(producePlanIds);
         List<String> channelContentIds = produceContentList.stream().map(ProducePlanExeRecord::getChannelContentId).distinct().collect(Collectors.toList());
         List<ArticleCategory> articleCategoryList = articleCategoryRepository.getAllByChannelContentIdIn(channelContentIds);
-        List<String>  articleCategoryIds = articleCategoryList.stream().map(ArticleCategory::getChannelContentId).collect(Collectors.toList());
+        List<String> articleCategoryIds = articleCategoryList.stream().map(ArticleCategory::getChannelContentId).collect(Collectors.toList());
         List<ProduceContentCrawlerVO> list = produceContentList.stream().filter(o -> !articleCategoryIds.contains(o.getChannelContentId())).map(o -> {
             ProduceContentCrawlerVO item = new ProduceContentCrawlerVO();
             item.setChannelContentId(o.getChannelContentId());
@@ -429,6 +532,9 @@ public class ArticleService {
             return item;
         }).collect(Collectors.toList());
         channelContentIds = channelContentIds.stream().filter(o -> !articleCategoryIds.contains(o)).collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(channelContentIds)) {
+            return Collections.emptyList();
+        }
         List<CrawlerContent> crawlerContentList = aigcBaseMapper.getCrawlerContentByChannelContentIdIn(channelContentIds);
         Map<String, CrawlerContent> map = crawlerContentList.stream().collect(Collectors.toMap(CrawlerContent::getChannelContentId, Function.identity()));
         long now = System.currentTimeMillis();
@@ -447,48 +553,12 @@ public class ArticleService {
                 saveList.add(item);
             }
         }
-        if (CollectionUtils.isNotEmpty(saveList)) {
-            longArticleBaseMapper.batchInsertArticleCategory(saveList);
-        }
+        return saveList;
     }
 
     private String buildKimiPrompt(List<String> titleList) {
-        StringBuilder prompt = new StringBuilder(
-                "请帮我完成以下任务：输入为文章的标题，根据标题判断其内容所属的类目，输出为文章标题及其对应的类目。\n" +
-                "类目需从以下范围内选择：\n" +
-                "奇闻趣事\n" +
-                "历史人物\n" +
-                "家长里短\n" +
-                "温情故事\n" +
-                "健康养生\n" +
-                "生活知识\n" +
-                "名人八卦\n" +
-                "政治新闻\n" +
-                "军事新闻\n" +
-                "影视解读\n" +
-                "为了更好地完成任务，可参考下列对文章标题的分类：\n" +
-                "{" +
-                "\"大舅病了，我取了三万元送过去，病房门口听到舅妈的话我改了主意\": \"家长里短\",\n" +
-                "\"能活到90岁的老人，基本上在70岁的时候，就不再做这些事了！\": \"健康养生\",\n" +
-                "\"去医院看望病人时，切忌带这4样东西，再亲近也不行，这是做人的根本\": \"生活知识\",\n" +
-                "\"上海一女子去饭店吃生煎包，戳了个洞想凉一凉，往里一看，瞬间惊呆了\": \"奇闻趣事\",\n" +
-                "\"卫生间放一把食盐，一年能省下好几百，涨新知识\": \"生活知识\",\n" +
-                "\"中国有一古寺，庙不大，却有武警24小时站岗，到底有何“过人”之处\": \"奇闻趣事\",\n" +
-                "\"1974年，苏联外长故意拿邓小平身高“取笑”，邓小平一句话轻松反击\": \"历史人物\",\n" +
-                "\"中国最美的女将军：上世纪曾家喻户晓，如今仍然健在\": \"历史人物\",\n" +
-                "\"北大才女蒙曼48岁仍未婚，被问最想嫁给谁，一个名字让全场笑喷\": \"名人八卦\",\n" +
-                "\"广东一老人去世，家人把老人的旧床垫扔了，环卫工人看到后，竟发现里面藏了15万元现金！家人傻眼了\": \"奇闻趣事\",\n" +
-                "\"《孤舟》结局：要不是海沫2年不生娃，周知非至死不知，顾易中要3000元的报销费的真实目的！\": \"影视解读\"\n" +
-                "}" +
-                "最后输出结果请用JSON格式输出，key为title，value为类目，仅输出JSON，不要markdown格式，不要任何其他内容，" +
-                "并且内容可以被 fastJSON 的JSONObject.parseObject转换为JSON对象\n" +
-                "当标题的开头或结尾为以下字符时“”“，则在标题的开头或结尾增加\" " +
-                "输出结果格式如下：\n" +
-                "{" +
-                "\"浙江老人用“假钱”吃霸王餐9年，离世后，老板却崩溃大哭：“每天都在等他！”\": \"奇闻趣事\"," +
-                "\"“最美婴儿”迅速走红，像在娘胎里整过容，网友：看到第一眼就想抱回家\": \"奇闻趣事\"" +
-                "}" +
-                "以下是需要分析的文章标题列表，每一行是一个标题：\n");
+        StringBuilder prompt = new StringBuilder(kimiCategoryPrompt);
+        prompt.append("\n");
         for (String title : titleList) {
             prompt.append(title).append("\n");
         }
@@ -538,6 +608,12 @@ public class ArticleService {
         PublishContent publishContent = findPublishContent(publishContentList, article.getTitle(), publishTimestamp);
         if (Objects.nonNull(publishContent)) {
             return publishContent.getId();
+        } else {
+            publishContentList = aigcBaseMapper.getLateNearestPublishContent(publishAccount.getId(), publishTimestamp);
+            publishContent = findPublishContent(publishContentList, article.getTitle(), publishTimestamp);
+            if (Objects.nonNull(publishContent)) {
+                return publishContent.getId();
+            }
         }
         return null;
     }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/DataDashboardService.java

@@ -45,7 +45,7 @@ import com.xxl.job.core.biz.model.ReturnT;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.beans.BeanUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.util.Pair;
 import org.springframework.http.*;
@@ -99,35 +99,39 @@ public class DataDashboardService {
 
     public void export(String beginDate, String endDate) {
         List<String> dateStrList = DateUtils.getBeforeDays(beginDate, endDate, 5);
+        // 滚动删除，仅保留最近14天内容
+        List<String> delDateStrList = DateUtils.getBeforeDays(null, null, 14);
         exportFeishuNewSortStrategy(dateStrList, ArticleTypeEnum.QUNFA.getVal(), StatusEnum.ZERO.getCode(),
-                sheetToken, "7d4e12");
+                sheetToken, "7d4e12", delDateStrList);
     }
 
     @XxlJob("scheduledExport")
     public ReturnT<String> scheduledExport(String param) {
         List<String> dateStrList = DateUtils.getBeforeDays(null, null, 5);
+        // 滚动删除，仅保留最近14天内容
+        List<String> delDateStrList = DateUtils.getBeforeDays(null, null, 14);
         exportFeishuNewSortStrategy(dateStrList, ArticleTypeEnum.QUNFA.getVal(), StatusEnum.ZERO.getCode(),
-                sheetToken, "7d4e12");
+                sheetToken, "7d4e12", delDateStrList);
         return ReturnT.SUCCESS;
     }
 
     public void exportWuXianLiu(String beginDate, String endDate) {
         List<String> dateStrList = DateUtils.getBeforeDays(beginDate, endDate, 5);
         exportFeishuNewSortStrategy(dateStrList, ArticleTypeEnum.WUXIANLIU.getVal(), StatusEnum.ONE.getCode(),
-                sheetToken, "Ucycvw");
+                sheetToken, "Ucycvw", null);
     }
 
     @XxlJob("scheduledExportWuXianLiu")
     public ReturnT<String> scheduledExportWuXianLiu(String param) {
         List<String> dateStrList = DateUtils.getBeforeDays(null, null, 5);
         exportFeishuNewSortStrategy(dateStrList, ArticleTypeEnum.WUXIANLIU.getVal(), StatusEnum.ONE.getCode(),
-                sheetToken, "Ucycvw");
+                sheetToken, "Ucycvw", null);
         return ReturnT.SUCCESS;
     }
 
 
     private void exportFeishuNewSortStrategy(List<String> dateStrList, String articleType, Integer filter,
-                                             String sheetToken, String sheetId) {
+                                             String sheetToken, String sheetId, List<String> delDateStrList) {
         String minDate = dateStrList.stream().min(String::compareTo).orElse("");
         String maxDate = dateStrList.stream().max(String::compareTo).orElse("");
         List<NewSortStrategyExport> newContentsYesData = newSortStrategyData(minDate, maxDate, articleType, filter);
@@ -156,21 +160,17 @@ public class DataDashboardService {
 
         List<Pair<String, String>> styles = Arrays
                 .asList(
-                        Pair.of("AA", "0.00%"),
-                        Pair.of("AB", "0.00%"),
-                        Pair.of("AC", "0.00%"),
-                        Pair.of("AD", "0.00%"),
-                        Pair.of("AE", "0.00%"),
-                        Pair.of("AF", "0.00%"),
-                        Pair.of("AG", "0.00%"),
-                        Pair.of("AH", "0.00%"),
-                        Pair.of("AI", "0.00%"),
-                        Pair.of("AO", "0.00%"),
-                        Pair.of("AP", "0.00%"),
-                        Pair.of("AQ", "0.00%")
+                        Pair.of("J", "0.00%"),
+                        Pair.of("K", "0.00%"),
+                        Pair.of("L", "0.00%"),
+                        Pair.of("M", "0.00%"),
+                        Pair.of("N", "0.00%"),
+                        Pair.of("O", "0.00%"),
+                        Pair.of("P", "0.00%"),
+                        Pair.of("Q", "0.00%"),
+                        Pair.of("R", "0.00%")
                 );
-
-        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles);
+        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles, delDateStrList);
     }
 
     private List<NewSortStrategyExport> newSortStrategyData(String beginDate, String endDate,
@@ -187,7 +187,6 @@ public class DataDashboardService {
                 Collectors.groupingBy(Article::getAppMsgId, Collectors.toMap(Article::getItemIndex, o -> o))));
         log.info("newSortStrategyData articleList finish");
         Set<String> snList = articleList.stream().map(Article::getWxSn).collect(Collectors.toSet());
-        Map<String, Article> wxSnMap = articleList.stream().collect(Collectors.toMap(Article::getWxSn, Function.identity()));
         List<ArticleDetailInfo> articleDetailInfoList = new ArrayList<>();
         for (List<String> partitions : Lists.partition(new ArrayList<>(snList), 1000)) {
             articleDetailInfoList.addAll(articleDetailInfoRepository.getAllByWxSnIn(partitions));
@@ -197,10 +196,10 @@ public class DataDashboardService {
                 .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
         List<PublishSortLog> sortLogList = publishSortLogRepository.findByGhIdInAndDateStrBetween(ghIds, beginDate, endDate);
         log.info("newSortStrategyData sortLogList finish");
-        Map<String, Map<String, Map<String, String>>> sortStrategyMap = sortLogList.stream()
+        Map<String, Map<String, Map<String, PublishSortLog>>> sortStrategyMap = sortLogList.stream()
                 .collect(Collectors.groupingBy(PublishSortLog::getGhId,
                         Collectors.groupingBy(PublishSortLog::getDateStr, Collectors.toMap(PublishSortLog::getTitle,
-                                PublishSortLog::getStrategy, (existing, replacement) -> existing))));
+                                Function.identity(), (existing, replacement) -> existing))));
         Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap = accountAvgInfoList.stream()
                 .filter(o -> Objects.nonNull(o.getReadAvg()) && o.getReadAvg() > 0 && o.getFans() > 1000)
                 .collect(Collectors.groupingBy(AccountAvgInfo::getGhId, Collectors.groupingBy(AccountAvgInfo::getUpdateTime,
@@ -332,14 +331,16 @@ public class DataDashboardService {
                 .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
 
         // result
-        List<NewSortStrategyExport> result = new ArrayList<>();
+        List<DatastatSortStrategy> saveList = new ArrayList<>();
         for (Article article : articleList) {
-            NewSortStrategyExport obj = new NewSortStrategyExport();
+            DatastatSortStrategy obj = new DatastatSortStrategy();
+            obj.setType(articleType);
+            obj.setPublishTimestamp(article.getPublishTimestamp());
             List<ArticleDetailInfo> articleDetailInfos = articleDetailInfoMap.get(article.getWxSn());
             setObjArticleDetailInfo(article, obj, articleDetailInfos);
             Article firstArticle = articleMap.get(article.getGhId()).get(article.getAppMsgId()).get(1);
-            Map<String, Map<String, String>> dateStrategy = sortStrategyMap.get(article.getGhId());
-            AccountAvgInfo avgInfo = getAccountAvgInfo(accountAvgInfoIndexMap, article.getGhId(),
+            Map<String, Map<String, PublishSortLog>> dateStrategy = sortStrategyMap.get(article.getGhId());
+            Map<String, AccountAvgInfo> indexAvgInfoMap = getDateAccountAvgInfo(accountAvgInfoIndexMap, article.getGhId(),
                     article.getPublishTimestamp(), article.getItemIndex());
             AccountAvgInfo firstAvgInfo = getAccountAvgInfo(accountAvgInfoIndexMap, article.getGhId(),
                     article.getPublishTimestamp(), 1);
@@ -352,64 +353,58 @@ public class DataDashboardService {
                 obj.setFirstAvgViewCount(firstAvgInfo.getReadAvg());
             }
             if (Objects.nonNull(dateStrategy)) {
-                Map<String, String> titleStrategyMap = dateStrategy.get(date);
+                Map<String, PublishSortLog> titleStrategyMap = dateStrategy.get(date);
                 if (Objects.nonNull(titleStrategyMap)) {
-                    String strategy = titleStrategyMap.get(article.getTitle());
-                    if (!StringUtils.hasText(strategy)) {
-                        if (Objects.equals(articleType, ArticleTypeEnum.WUXIANLIU.getVal())) {
-                            strategy = RankStrategyEnum.INFINITE_STRATEGY.getStrategy();
-                        } else {
-                            for (Map.Entry<String, String> entry : titleStrategyMap.entrySet()) {
-                                strategy = entry.getValue();
-                                break;
+                    PublishSortLog sortLog = titleStrategyMap.get(article.getTitle());
+                    if (Objects.nonNull(sortLog)) {
+                        String strategy = sortLog.getStrategy();
+                        if (!StringUtils.hasText(sortLog.getStrategy())) {
+                            if (Objects.equals(articleType, ArticleTypeEnum.WUXIANLIU.getVal())) {
+                                strategy = RankStrategyEnum.INFINITE_STRATEGY.getStrategy();
+                            } else {
+                                for (Map.Entry<String, PublishSortLog> entry : titleStrategyMap.entrySet()) {
+                                    strategy = entry.getValue().getStrategy();
+                                    break;
+                                }
                             }
                         }
+                        obj.setSourceType(sortLog.getSourceType());
+                        obj.setSourceId(sortLog.getSourceId());
+                        obj.setStrategy(strategy);
                     }
-                    obj.setStrategy(strategy);
                 }
             }
             if (!StringUtils.hasText(obj.getStrategy()) &&
                     Objects.equals(articleType, ArticleTypeEnum.WUXIANLIU.getVal())) {
                 obj.setStrategy(RankStrategyEnum.INFINITE_STRATEGY.getStrategy());
             }
-            setObjAvgInfo(article, obj, avgInfo);
+            setObjAvgInfo(article, obj, indexAvgInfoMap);
             setObjHisRateInfo(article, obj, hisArticleMap, accountAvgInfoIndexMap, hisArticleDetailInfoMap);
             // aigc 数据
             setObjAigcInfo(article, obj, date, publishAccountMap, publishContentMap, publishContentLayoutMap,
                     publishPlanMap, miniprogramTaskMap, planExeRecordMap, producePlanMap, inputSourceMap,
                     resultRelMap, crawlerPlanMap, sourceTitlePlanMap);
-            result.add(obj);
+            saveList.add(obj);
         }
         log.info("newSortStrategyData buildData finish");
-        if (CollectionUtil.isNotEmpty(result) && filter == StatusEnum.ONE.getCode()) {
-            result = result.stream()
+        if (CollectionUtil.isNotEmpty(saveList) && filter == StatusEnum.ONE.getCode()) {
+            saveList = saveList.stream()
                     .filter(o -> Objects.nonNull(o.getPublishMiniProgramNum()) && o.getPublishMiniProgramNum() > 0)
                     .collect(Collectors.toList());
         }
-        result.sort(Comparator.comparing(NewSortStrategyExport::getDateStr).reversed()
-                .thenComparing(NewSortStrategyExport::getGhId).thenComparing(NewSortStrategyExport::getPosition));
-        if (CollectionUtils.isNotEmpty(result)) {
+        saveList.sort(Comparator.comparing(DatastatSortStrategy::getDateStr).reversed()
+                .thenComparing(DatastatSortStrategy::getGhId).thenComparing(DatastatSortStrategy::getPosition));
+        if (CollectionUtils.isNotEmpty(saveList)) {
             longArticleBaseMapper.deleteByDateStrBetween(beginDate, endDate, articleType);
-            List<DatastatSortStrategy> saveList = new ArrayList<>();
-            for (NewSortStrategyExport newSortStrategyExport : result) {
-                DatastatSortStrategy item = new DatastatSortStrategy();
-                BeanUtils.copyProperties(newSortStrategyExport, item);
-                item.setType(articleType);
-                PublishAccount publishAccount = publishAccountMap.get(item.getGhId());
-                item.setAccountCreateTimestamp(publishAccount.getCreateTimestamp() / 1000);
-                Article article = wxSnMap.get(item.getWxSn());
-                item.setPublishTimestamp(article.getPublishTimestamp());
-                saveList.add(item);
-            }
             for (List<DatastatSortStrategy> saveListPartition : Lists.partition(saveList, 1000)) {
                 longArticleBaseMapper.batchInsertDatastatSortStrategy(saveListPartition);
             }
         }
         log.info("newSortStrategyData finish");
-        return result;
+        return NewSortStrategyExport.dbObjToExportObj(saveList);
     }
 
-    private void setObjAigcInfo(Article article, NewSortStrategyExport obj, String date,
+    private void setObjAigcInfo(Article article, DatastatSortStrategy obj, String date,
                                 Map<String, PublishAccount> publishAccountMap,
                                 Map<String, Map<String, Map<Long, PublishContentDTO>>> publishContentMap,
                                 Map<String, PublishContentLayout> publishContentLayoutMap,
@@ -422,6 +417,7 @@ public class DataDashboardService {
                                 Map<String, CrawlerPlan> crawlerPlanMap,
                                 Map<String, ProducePlan> sourceTitlePlanMap) {
         PublishAccount publishAccount = publishAccountMap.get(article.getGhId());
+        obj.setAccountCreateTimestamp(publishAccount.getCreateTimestamp() / 1000);
         Map<String, Map<Long, PublishContentDTO>> titleContentMap = publishContentMap.get(publishAccount.getId());
         if (Objects.isNull(titleContentMap)) {
             return;
@@ -494,7 +490,7 @@ public class DataDashboardService {
         }
     }
 
-    private void setObjHisRateInfo(Article article, NewSortStrategyExport obj,
+    private void setObjHisRateInfo(Article article, DatastatSortStrategy obj,
                                    Map<String, List<Article>> hisArticleMap,
                                    Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap,
                                    Map<String, List<ArticleDetailInfo>> hisArticleDetailInfoMap) {
@@ -539,7 +535,7 @@ public class DataDashboardService {
         }
     }
 
-    private void setObjBaseInfo(Article article, NewSortStrategyExport obj, String date) {
+    private void setObjBaseInfo(Article article, DatastatSortStrategy obj, String date) {
         obj.setGhId(article.getGhId());
         obj.setAccountName(article.getAccountName());
         obj.setTitle(article.getTitle());
@@ -551,7 +547,7 @@ public class DataDashboardService {
         obj.setWxSn(article.getWxSn());
     }
 
-    private void setObjArticleDetailInfo(Article article, NewSortStrategyExport obj, List<ArticleDetailInfo> articleDetailInfos) {
+    private void setObjArticleDetailInfo(Article article, DatastatSortStrategy obj, List<ArticleDetailInfo> articleDetailInfos) {
         if (CollectionUtils.isNotEmpty(articleDetailInfos)) {
             Date publishDate = DateUtils.getStartDateOfDay(article.getPublishTimestamp());
             articleDetailInfos = articleDetailInfos.stream().filter(o -> !o.getRecallDt().before(publishDate))
@@ -606,14 +602,24 @@ public class DataDashboardService {
         }
     }
 
-    private void setObjAvgInfo(Article article, NewSortStrategyExport obj, AccountAvgInfo avgInfo) {
+    private void setObjAvgInfo(Article article, DatastatSortStrategy obj, Map<String, AccountAvgInfo> indexAvgInfoMap) {
+        if (MapUtils.isNotEmpty(indexAvgInfoMap)) {
+            AccountAvgInfo info = null;
+            for (String index : indexAvgInfoMap.keySet()) {
+                info = indexAvgInfoMap.get(index);
+                break;
+            }
+            if (Objects.nonNull(info)) {
+                obj.setAccountMode(info.getAccountMode());
+                obj.setAccountSource(info.getAccountSource());
+                obj.setAccountType(info.getAccountType());
+                obj.setAccountStatus(info.getAccountStatus());
+                obj.setBusinessType(AccountBusinessTypeEnum.from(info.getBusinessType()).getDescription());
+                obj.setFans(info.getFans());
+            }
+        }
+        AccountAvgInfo avgInfo = indexAvgInfoMap.get(article.getItemIndex().toString());
         if (Objects.nonNull(avgInfo)) {
-            obj.setAccountMode(avgInfo.getAccountMode());
-            obj.setAccountSource(avgInfo.getAccountSource());
-            obj.setAccountType(avgInfo.getAccountType());
-            obj.setAccountStatus(avgInfo.getAccountStatus());
-            obj.setBusinessType(AccountBusinessTypeEnum.from(avgInfo.getBusinessType()).getDescription());
-            obj.setFans(avgInfo.getFans());
             obj.setAvgViewCount(avgInfo.getReadAvg());
             if (avgInfo.getReadAvg() > 0) {
                 obj.setReadRate((article.getShowViewCount() * 1.0) / avgInfo.getReadAvg());
@@ -671,12 +677,14 @@ public class DataDashboardService {
 
     private static void doSendFeishuSheet(List<String> dateStrList, String sheetToken, String sheetId,
                                           int rowNum, List<List<Object>> rows, Integer startRowIndex,
-                                          List<Pair<String, String>> styles) {
+                                          List<Pair<String, String>> styles,
+                                          List<String> delDateStrList) {
         Pair<String, Integer> token = FeiShu.requestAccessToken();
         RestTemplate restTemplate = new RestTemplate();
         HttpHeaders httpHeaders = new HttpHeaders();
         httpHeaders.add("Authorization", "Bearer " + token.getFirst());
 
+        delFeishuSheet(httpHeaders, sheetId, startRowIndex, delDateStrList);
         // 先删除掉已存在的dateStr数据
         HttpEntity<Object> queryEntity = new HttpEntity<>(httpHeaders);
         int deleteRowNum = rowNum < 20 ? startRowIndex + dateStrList.size() * rowNum : startRowIndex + (rowNum * 2);
@@ -795,6 +803,63 @@ public class DataDashboardService {
         }
     }
 
+    public static void delFeishuSheet(HttpHeaders httpHeaders, String sheetId, Integer startRowIndex,
+                                       List<String> delDateStrList) {
+        if (CollectionUtil.isEmpty(delDateStrList)) {
+            return;
+        }
+        RestTemplate restTemplate = new RestTemplate();
+        HttpEntity<Object> queryEntity = new HttpEntity<>(httpHeaders);
+        ResponseEntity<String> queryResponseEntity = restTemplate.exchange(
+                String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/values/%s!A"
+                        + startRowIndex + ":A", sheetToken, sheetId),
+                HttpMethod.GET, queryEntity, String.class);
+        JSONArray values = JSON.parseObject(queryResponseEntity.getBody())
+                .getJSONObject("data")
+                .getJSONObject("valueRange")
+                .getJSONArray("values");
+        int deleteStartRow = values.size();
+        if (!values.isEmpty() && Objects.nonNull(values.get(0)) && !((JSONArray) values.get(0)).isEmpty()
+                && Objects.nonNull(((JSONArray) values.get(0)).get(0))) {
+            for (int i = 0; i < values.size(); i++) {
+                JSONArray value = values.getJSONArray(i);
+                if (value.get(0) != null) {
+                    List<String> dates = value.stream().map(Object::toString).collect(Collectors.toList());
+                    if (!delDateStrList.contains(dates.get(0))) {
+                        deleteStartRow = i + startRowIndex;
+                        break;
+                    }
+                }
+            }
+        }
+        int count = values.size();
+        int delCount = values.size() - deleteStartRow;
+        if (delCount > 0) {
+            int delNum = 0;
+            do {
+                // 删除当前日期已存在的旧数据
+                httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+                HttpEntity<Object> deleteEntity = new HttpEntity<>(
+                        String.format("{\n" +
+                                "    \"dimension\": {\n" +
+                                "        \"sheetId\": \"%s\",\n" +
+                                "        \"majorDimension\": \"ROWS\",\n" +
+                                "        \"startIndex\": %s,\n" +
+                                "        \"endIndex\": %s\n" +
+                                "    }\n" +
+                                "}", sheetId, deleteStartRow, Math.min(startRowIndex + 4000, count - delNum + startRowIndex) - 1),
+                        httpHeaders);
+                ResponseEntity<String> response = restTemplate.exchange(String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/dimension_range", sheetToken),
+                        HttpMethod.DELETE, deleteEntity, String.class);
+                JSONObject responseJSON = JSONObject.parseObject(response.getBody());
+                if (0 != responseJSON.getInteger("code")) {
+                    log.error("doSendFeishuSheet write error :{}", responseJSON.getString("msg"));
+                }
+                delNum = Math.min(delNum + 4000, delCount);
+            } while (delNum < delCount);
+        }
+    }
+
     @XxlJob("scheduleExportIntermediateIndicators")
     public ReturnT<String> scheduleIntermediateIndicatorsExport(String param) {
         List<String> dateStrList = DateUtils.getBeforeDays(null, null, 3);
@@ -876,7 +941,7 @@ public class DataDashboardService {
                         Pair.of("BP", "0.00%")
                 );
 
-        doSendFeishuSheet(dateStrs, sheetToken, sheetId, rowNum, rows, 3, styles);
+        doSendFeishuSheet(dateStrs, sheetToken, sheetId, rowNum, rows, 3, styles, null);
     }
 
     public List<IntermediateIndicatorsExport> intermediateIndicatorsData(String dateStr) {
@@ -912,12 +977,10 @@ public class DataDashboardService {
                 .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
         String lessDateStr = DateUtils.getBeforeDayStr(dateStr, "yyyyMMdd", 7);
         List<PublishSortLog> publishSortLogList = publishSortLogRepository.findByDateStrGreaterThanEqual(lessDateStr);
-        Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap = publishSortLogList.stream()
+        Map<String, Map<String, Map<Integer, List<PublishSortLog>>>> publishSortLogMap = publishSortLogList.stream()
                 .collect(Collectors.groupingBy(PublishSortLog::getGhId,
-                        Collectors.groupingBy(PublishSortLog::getDateStr, Collectors.toMap(
-                                PublishSortLog::getIndex, o -> o, (existing, replacement) -> replacement
-                        ))));
-        List<NewSortStrategyExport> newSortStrategyExportList = getNewSortStrategyExportList(todayPublish,
+                        Collectors.groupingBy(PublishSortLog::getDateStr, Collectors.groupingBy(PublishSortLog::getIndex))));
+        List<DatastatSortStrategy> newSortStrategyExportList = getNewSortStrategyExportList(todayPublish,
                 articleDetailInfoMap, accountAvgInfoIndexMap);
 
         List<Article> futurePublishList = articleRepository.getByPublishTimestampGreaterThanAndTypeEquals(dateEnd, ArticleTypeEnum.QUNFA.getVal());
@@ -932,7 +995,7 @@ public class DataDashboardService {
         Map<String, Map<Integer, List<String>>> titleTypePoolMap = new HashMap<>();
         Map<String, Set<String>> wxsnHisDistinctSetMap = new HashMap<>();
         Map<String, List<String>> fansAccountTypeMap = new HashMap<>();
-        for (NewSortStrategyExport data : newSortStrategyExportList) {
+        for (DatastatSortStrategy data : newSortStrategyExportList) {
             String type = getArticleType(data, small);
             Article article = articleMap.get(data.getWxSn());
             if (!checkIsAigcPublish(article, publishSortLogMap)) {
@@ -965,10 +1028,10 @@ public class DataDashboardService {
             // 发布表现
             setPublishPerformance(item, data, publishSortLogMap);
             // 发布依赖表现
-            setPublishSourcePerformance(item, accountAvgInfoIndexMap, articleDetailInfoMap, publishSortLogMap,
+            setPublishSourcePerformance(item, data, article, accountAvgInfoIndexMap, articleDetailInfoMap, publishSortLogMap,
                     type, scoreHisPublishTimeMap, wxsnHisDistinctSetMap, list, poolLevel, small, hisPublishMap);
             // 发布未来表现
-            setPublishFuturePerformance(item, data, poolLevel, promotionSourceMap, futurePublishMap, publishSortLogMap, small);
+            setPublishFuturePerformance(item, data, poolLevel, promotionSourceMap, futurePublishMap, small);
 
             titles.add(data.getTitle());
             poolTitles.add(data.getTitle());
@@ -978,15 +1041,19 @@ public class DataDashboardService {
     }
 
     private boolean checkIsAigcPublish(Article article,
-                                       Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap) {
-        Map<String, Map<Integer, PublishSortLog>> dateSortMap = publishSortLogMap.get(article.getGhId());
+                                       Map<String, Map<String, Map<Integer, List<PublishSortLog>>>> publishSortLogMap) {
+        Map<String, Map<Integer, List<PublishSortLog>>> dateSortMap = publishSortLogMap.get(article.getGhId());
         if (Objects.nonNull(dateSortMap)) {
             for (String dateStr : dateSortMap.keySet()) {
-                Map<Integer, PublishSortLog> indexMap = dateSortMap.get(dateStr);
+                Map<Integer, List<PublishSortLog>> indexMap = dateSortMap.get(dateStr);
                 if (Objects.nonNull(indexMap)) {
-                    PublishSortLog log = indexMap.get(article.getItemIndex());
-                    if (Objects.nonNull(log) && log.getTitle().equals(article.getTitle())) {
-                        return true;
+                    List<PublishSortLog> logs = indexMap.get(article.getItemIndex());
+                    if (CollectionUtil.isNotEmpty(logs)) {
+                        for (PublishSortLog log : logs) {
+                            if (log.getTitle().equals(article.getTitle())) {
+                                return true;
+                            }
+                        }
                     }
                 }
             }
@@ -1010,10 +1077,24 @@ public class DataDashboardService {
         return avgInfo;
     }
 
-    private void setPublishFuturePerformance(IntermediateIndicatorsExport item, NewSortStrategyExport data, Integer poolLevel,
+    private Map<String, AccountAvgInfo> getDateAccountAvgInfo(Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap,
+                                                              String ghId, Long updateTime, Integer itemIndex) {
+        Map<String, Map<String, AccountAvgInfo>> dateAvgMap = accountAvgInfoIndexMap.get(ghId);
+        String hisPublishDate = DateUtils.timestampToYMDStr(updateTime, "yyyy-MM-dd");
+        if (Objects.nonNull(dateAvgMap)) {
+            List<String> avgMapDateList = new ArrayList<>(dateAvgMap.keySet());
+            hisPublishDate = DateUtils.findNearestDate(avgMapDateList, hisPublishDate, "yyyy-MM-dd");
+            Map<String, AccountAvgInfo> accountAvgInfoMap = dateAvgMap.get(hisPublishDate);
+            if (Objects.nonNull(accountAvgInfoMap)) {
+                return accountAvgInfoMap;
+            }
+        }
+        return new HashMap<>();
+    }
+
+    private void setPublishFuturePerformance(IntermediateIndicatorsExport item, DatastatSortStrategy data, Integer poolLevel,
                                              Map<String, ArticlePoolPromotionSource> promotionSourceMap,
                                              Map<String, List<Article>> futurePublishMap,
-                                             Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap,
                                              List<String> small) {
         int futurePoolLevel = getPromotionPoolLevel(data.getGhId(), data.getTitle(), promotionSourceMap, small);
         if (futurePoolLevel > poolLevel) {
@@ -1142,28 +1223,30 @@ public class DataDashboardService {
         }
     }
 
-    private List<NewSortStrategyExport> getNewSortStrategyExportList(List<Article> todayPublish,
+    private List<DatastatSortStrategy> getNewSortStrategyExportList(List<Article> todayPublish,
                                                                      Map<String, List<ArticleDetailInfo>> articleDetailInfoMap,
                                                                      Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap) {
-        List<NewSortStrategyExport> result = new ArrayList<>();
+        List<DatastatSortStrategy> result = new ArrayList<>();
         for (Article article : todayPublish) {
-            NewSortStrategyExport obj = new NewSortStrategyExport();
+            DatastatSortStrategy obj = new DatastatSortStrategy();
             List<ArticleDetailInfo> articleDetailInfos = articleDetailInfoMap.get(article.getWxSn());
             setObjArticleDetailInfo(article, obj, articleDetailInfos);
-            AccountAvgInfo avgInfo = getAccountAvgInfo(accountAvgInfoIndexMap, article.getGhId(),
+            Map<String, AccountAvgInfo> indexAvgInfoMap = getDateAccountAvgInfo(accountAvgInfoIndexMap, article.getGhId(),
                     article.getPublishTimestamp(), article.getItemIndex());
             String date = DateUtils.timestampToYMDStr(article.getPublishTimestamp(), "yyyyMMdd");
             setObjBaseInfo(article, obj, date);
-            setObjAvgInfo(article, obj, avgInfo);
+            setObjAvgInfo(article, obj, indexAvgInfoMap);
             result.add(obj);
         }
         return result;
     }
 
     private void setPublishSourcePerformance(IntermediateIndicatorsExport item,
+                                             DatastatSortStrategy data,
+                                             Article article,
                                              Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap,
                                              Map<String, List<ArticleDetailInfo>> articleDetailInfoMap,
-                                             Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap,
+                                             Map<String, Map<String, Map<Integer, List<PublishSortLog>>>> publishSortLogMap,
                                              String type,
                                              Map<String, Integer> scoreHisPublishTimeMap,
                                              Map<String, Set<String>> wxsnHisDistinctSetMap,
@@ -1235,36 +1318,45 @@ public class DataDashboardService {
                         (item.getActualArticleReleaseCount() - 1) + firstLayerFissionToViewBaseRatio) / item.getActualArticleReleaseCount();
             }
             item.setAverageFirstLayerFissionToViewBaseRatio(averageFirstLayerFissionToViewBaseRatio);
-            Map<String, Map<Integer, PublishSortLog>> dateSortMap = publishSortLogMap.get(publish.getGhId());
+            Map<String, Map<Integer, List<PublishSortLog>>> dateSortMap = publishSortLogMap.get(publish.getGhId());
             if (Objects.nonNull(dateSortMap)) {
                 String publishDateStr = DateUtils.timestampToYMDStr(publish.getPublishTimestamp(), "yyyyMMdd");
-                Map<Integer, PublishSortLog> indexMap = dateSortMap.get(publishDateStr);
+                Map<Integer, List<PublishSortLog>> indexMap = dateSortMap.get(publishDateStr);
                 if (Objects.nonNull(indexMap)) {
-                    PublishSortLog log = indexMap.get(publish.getItemIndex());
-                    double scoreAvg = item.getRedundantAccountArticleRelevanceAvg();
-                    double score = 0.0;
-                    if (Objects.nonNull(log) && Objects.nonNull(log.getScore())) {
-                        JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
-                        if (scoreMap.containsKey("SimilarityStrategy")) {
-                            score = scoreMap.getDoubleValue("SimilarityStrategy");
+                    List<PublishSortLog> logs = indexMap.get(publish.getItemIndex());
+                    if (CollectionUtil.isNotEmpty(logs)) {
+                        for (PublishSortLog log : logs) {
+                            if (log.getTitle().equals(data.getTitle())) {
+                                double scoreAvg = item.getRedundantAccountArticleRelevanceAvg();
+                                double score = 0.0;
+                                if (Objects.nonNull(log.getScore())) {
+                                    JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
+                                    if (scoreMap.containsKey("SimilarityStrategy")) {
+                                        score = scoreMap.getDoubleValue("SimilarityStrategy");
+                                    }
+                                }
+                                scorePublishTime++;
+                                scoreAvg = (scoreAvg * (scorePublishTime - 1) + score) / scorePublishTime;
+                                item.setRedundantAccountArticleRelevanceAvg(scoreAvg);
+                                break;
+                            }
                         }
                     }
-                    scorePublishTime++;
-                    scoreAvg = (scoreAvg * (scorePublishTime - 1) + score) / scorePublishTime;
-                    item.setRedundantAccountArticleRelevanceAvg(scoreAvg);
                 }
             }
+        }
+        if (hisMinDate > 0) {
             double firstExplorationIntervalAvg = Double.isNaN(item.getFirstExplorationIntervalAvg()) ? 0.0 : item.getFirstExplorationIntervalAvg();
-            double explorationInterval = (publish.getPublishTimestamp() - hisMinDate) / 86400.0;
-            firstExplorationIntervalAvg = (firstExplorationIntervalAvg * (item.getTotalArticleReleaseCountNonInfinite() - 1)
-                    + explorationInterval) / item.getTotalArticleReleaseCountNonInfinite();
+            double explorationInterval = (article.getPublishTimestamp() - hisMinDate) / 86400.0;
+            firstExplorationIntervalAvg = (firstExplorationIntervalAvg * (item.getActualArticleReleaseCount() - 1)
+                    + explorationInterval) / item.getActualArticleReleaseCount();
             item.setFirstExplorationIntervalAvg(firstExplorationIntervalAvg);
         }
         scoreHisPublishTimeMap.put(type, scorePublishTime);
     }
 
-    private void setPublishPerformance(IntermediateIndicatorsExport item, NewSortStrategyExport data,
-                                       Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap) {
+    private void setPublishPerformance(IntermediateIndicatorsExport item, DatastatSortStrategy data,
+                                       Map<String, Map<String, Map<Integer, List<PublishSortLog>>>> publishSortLogMap) {
         if (Objects.isNull(data.getFirstLevel())) {
             return;
         }
@@ -1296,21 +1388,28 @@ public class DataDashboardService {
         if (item.getT0FissionCount() + item.getT1FissionCount() + item.getT2FissionCount() > 0 && item.getViewBase() > 0) {
             item.setT2CumulativeFissionRate((item.getT0FissionCount() + item.getT1FissionCount() + item.getT2FissionCount()) / (double) item.getFirstLayerUV());
         }
-        Map<String, Map<Integer, PublishSortLog>> dateSortMap = publishSortLogMap.get(data.getGhId());
+        Map<String, Map<Integer, List<PublishSortLog>>> dateSortMap = publishSortLogMap.get(data.getGhId());
         if (Objects.nonNull(dateSortMap)) {
-            Map<Integer, PublishSortLog> indexMap = dateSortMap.get(item.getDateStr());
+            Map<Integer, List<PublishSortLog>> indexMap = dateSortMap.get(item.getDateStr());
             if (Objects.nonNull(indexMap)) {
-                PublishSortLog log = indexMap.get(data.getPosition());
-                double scoreAvg = item.getAccountArticleRelevanceAvg();
-                double score = 0.0;
-                if (Objects.nonNull(log) && Objects.nonNull(log.getScore())) {
-                    JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
-                    if (scoreMap.containsKey("SimilarityStrategy")) {
-                        score = scoreMap.getDoubleValue("SimilarityStrategy");
+                List<PublishSortLog> logs = indexMap.get(data.getPosition());
+                if (CollectionUtil.isNotEmpty(logs)) {
+                    for (PublishSortLog log : logs) {
+                        if (log.getTitle().equals(data.getTitle())) {
+                            double scoreAvg = item.getAccountArticleRelevanceAvg();
+                            double score = 0.0;
+                            if (Objects.nonNull(log.getScore())) {
+                                JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
+                                if (scoreMap.containsKey("SimilarityStrategy")) {
+                                    score = scoreMap.getDoubleValue("SimilarityStrategy");
+                                }
+                            }
+                            scoreAvg = (scoreAvg * (item.getActualArticleReleaseCount() - 1) + score) / item.getActualArticleReleaseCount();
+                            item.setAccountArticleRelevanceAvg(scoreAvg);
+                            break;
+                        }
                     }
                 }
-                scoreAvg = (scoreAvg * (item.getActualArticleReleaseCount() - 1) + score) / item.getActualArticleReleaseCount();
-                item.setAccountArticleRelevanceAvg(scoreAvg);
             }
         }
     }
@@ -1383,7 +1482,7 @@ public class DataDashboardService {
         }
     }
 
-    private String getArticleType(NewSortStrategyExport data, List<String> small) {
+    private String getArticleType(DatastatSortStrategy data, List<String> small) {
         if (data.getPosition() == 1) {
             if (small.contains(data.getGhId())) {
                 return "L3";
@@ -1463,7 +1562,7 @@ public class DataDashboardService {
                         Pair.of("W", "#,##0.00")
                 );
 
-        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles);
+        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles, null);
     }
 
     private List<FirstContentScoreExport> firstContentScoreData(List<String> dateStrList) {
@@ -1494,6 +1593,7 @@ public class DataDashboardService {
                                 Collectors.toMap(AccountAvgInfo::getPosition, o -> o))));
 
         String title = "";
+        List<DatastatScore> saveList = new ArrayList<>();
         for (PublishSortLog publishSortLog : sortLogList) {
             Map<String, Map<Integer, Article>> dateArticleMap = articleMap.get(publishSortLog.getGhId());
             if (Objects.isNull(dateArticleMap)) {
@@ -1559,40 +1659,45 @@ public class DataDashboardService {
                 item.setFirstExplorationIntervalAvg(explorationInterval);
             }
             result.add(item);
+            saveList.add(buildDatastatScore(publishSortLog, item, article));
         }
-        saveDatastatScore(dateStrList, result);
+        saveDatastatScore(dateStrList, saveList);
         result = result.stream().filter(o -> o.getIndex() == 1).collect(Collectors.toList());
         result.sort(Comparator.comparing(FirstContentScoreExport::getDateStr).reversed()
                 .thenComparing(FirstContentScoreExport::getGhId));
         return result;
     }
 
-    private void saveDatastatScore(List<String> dateStrList, List<FirstContentScoreExport> result) {
-        if (CollectionUtils.isNotEmpty(result)) {
+    private DatastatScore buildDatastatScore(PublishSortLog publishSortLog, FirstContentScoreExport value, Article article) {
+        DatastatScore item = new DatastatScore();
+        item.setDt(value.getDateStr());
+        item.setGhId(value.getGhId());
+        item.setAccountName(value.getAccountName());
+        item.setIndex(value.getIndex());
+        item.setTitle(value.getTitle());
+        item.setCategory(value.getCategory());
+        item.setStrategy(value.getStrategy());
+        item.setScore(value.getScore());
+        item.setHisFissionAvgReadRateRate(value.getHisFissionAvgReadRateRateStrategy());
+        item.setHisFissionAvgReadSumRate(value.getHisFissionAvgReadSumRateStrategy());
+        item.setSimilarity(value.getSimilarityStrategy());
+        item.setCategoryScore(value.getCategoryStrategy());
+        item.setViewCountRate(value.getViewCountRateStrategy());
+        item.setHisFissionDeWeightAvgReadSumRate(value.getHisFissionDeWeightAvgReadSumRateStrategy());
+        item.setReadCount(value.getReadCount());
+        item.setReadAvg(value.getReadAvg());
+        item.setReadAvgRate(value.getReadAvgRate());
+        item.setFirstPubInterval(value.getFirstExplorationIntervalAvg());
+        item.setPublishContentId(publishSortLog.getPublishContentId());
+        item.setCrawlerChannelContentId(publishSortLog.getCrawlerChannelContentId());
+        item.setSourceId(publishSortLog.getSourceId());
+        item.setPublishTimestamp(article.getPublishTimestamp());
+        return item;
+    }
+
+    private void saveDatastatScore(List<String> dateStrList, List<DatastatScore> saveList) {
+        if (CollectionUtils.isNotEmpty(saveList)) {
             longArticleBaseMapper.deleteDatastatScoreByDtIn(dateStrList);
-            List<DatastatScore> saveList = new ArrayList<>();
-            for (FirstContentScoreExport value : result) {
-                DatastatScore item = new DatastatScore();
-                item.setDt(value.getDateStr());
-                item.setGhId(value.getGhId());
-                item.setAccountName(value.getAccountName());
-                item.setIndex(value.getIndex());
-                item.setTitle(value.getTitle());
-                item.setCategory(value.getCategory());
-                item.setStrategy(value.getStrategy());
-                item.setScore(value.getScore());
-                item.setHisFissionAvgReadRateRate(value.getHisFissionAvgReadRateRateStrategy());
-                item.setHisFissionAvgReadSumRate(value.getHisFissionAvgReadSumRateStrategy());
-                item.setSimilarity(value.getSimilarityStrategy());
-                item.setCategoryScore(value.getCategoryStrategy());
-                item.setViewCountRate(value.getViewCountRateStrategy());
-                item.setHisFissionDeWeightAvgReadSumRate(value.getHisFissionDeWeightAvgReadSumRateStrategy());
-                item.setReadCount(value.getReadCount());
-                item.setReadAvg(value.getReadAvg());
-                item.setReadAvgRate(value.getReadAvgRate());
-                item.setFirstPubInterval(value.getFirstExplorationIntervalAvg());
-                saveList.add(item);
-            }
             for (List<DatastatScore> saveListPartition : Lists.partition(saveList, 1000)) {
                 longArticleBaseMapper.batchInsertDatastatScore(saveListPartition);
             }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/PublishContentFilterService.java

@@ -0,0 +1,51 @@
+package com.tzld.longarticle.recommend.server.service.recommend;
+
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ContentPoolEnum;
+import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
+import com.tzld.longarticle.recommend.server.model.param.PublishContentFilterContentItem;
+import com.tzld.longarticle.recommend.server.model.param.PublishContentFilterParam;
+import com.tzld.longarticle.recommend.server.model.vo.PublishContentFilterResultVO;
+import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.util.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+@Service
+@Slf4j
+public class PublishContentFilterService {
+
+    @Autowired
+    private LongArticleBaseMapper longArticleBaseMapper;
+
+    public List<PublishContentFilterResultVO> publishContentFilter(PublishContentFilterParam param) {
+        List<PublishContentFilterResultVO> result = new ArrayList<>();
+        if (CollectionUtils.isEmpty(param.getContentList())) {
+            return result;
+        }
+        List<String> filterTitleList = longArticleBaseMapper.getFilterColdLongArticleTitle();
+        List<Set<Character>> firstSecondTitleCache = TitleSimilarCheckUtil.makeCache(filterTitleList);
+        for (PublishContentFilterContentItem contentItem : param.getContentList()) {
+            if (!StringUtils.hasText(contentItem.getTitle())) {
+                continue;
+            }
+            boolean isDuplicate = false;
+            if (ContentPoolEnum.autoArticlePoolLevel4.getContentPool().equals(contentItem.getPoolLevel())) {
+                isDuplicate = TitleSimilarCheckUtil.isDuplicateContentByCache(contentItem.getTitle(),
+                        firstSecondTitleCache, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
+            }
+            if (isDuplicate) {
+                PublishContentFilterResultVO vo = new PublishContentFilterResultVO();
+                vo.setSourceId(contentItem.getSourceId());
+                vo.setFilterReason("退场机制过滤");
+                result.add(vo);
+            }
+        }
+        return result;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/RecommendService.java

@@ -2,14 +2,14 @@ package com.tzld.longarticle.recommend.server.service.recommend;
 
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
+import com.tzld.longarticle.recommend.server.common.CostMonitor;
+import com.tzld.longarticle.recommend.server.common.constant.SceneConstants;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PushTypeEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
-import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.ArticleUserGroupMapper;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
-import com.tzld.longarticle.recommend.server.model.entity.crawler.PublishContentSortLog;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.PublishSortLog;
 import com.tzld.longarticle.recommend.server.model.param.RecommendParam;
 import com.tzld.longarticle.recommend.server.model.param.RecommendRequest;
@@ -19,9 +19,7 @@ import com.tzld.longarticle.recommend.server.model.vo.RecommendResponse;
 import com.tzld.longarticle.recommend.server.model.vo.RecommendWithUserGroupResponse;
 import com.tzld.longarticle.recommend.server.repository.crawler.AccountAvgInfoRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
-import com.tzld.longarticle.recommend.server.repository.crawler.PublishContentSortLogRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.PublishSortLogRepository;
-import com.tzld.longarticle.recommend.server.common.constant.SceneConstants;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountIndexAvgViewCountService;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
@@ -61,8 +59,6 @@ public class RecommendService {
     @Autowired
     private RankService rankService;
     @Autowired
-    private PublishContentSortLogRepository publishContentSortLogRepository;
-    @Autowired
     private PublishSortLogRepository publishSortLogRepository;
     @Autowired
     AccountIndexAvgViewCountService accountIndexAvgViewCountService;
@@ -71,8 +67,6 @@ public class RecommendService {
     @Autowired
     private ArticleUserGroupMapper articleUserGroupMapper;
     @Autowired
-    private AigcBaseMapper aigcBaseMapper;
-    @Autowired
     private ArticleRepository articleRepository;
 
     @ApolloJsonValue("${accountStrategyConfig:{}}")
@@ -92,19 +86,26 @@ public class RecommendService {
 
         RecallResult recallResult = recallService.recall(convertToRecallParam(param));
         long t2 = System.currentTimeMillis();
-        log.info("recommendCost param:{} recall cost:{}", JSONObject.toJSONString(request), t2 - start);
+        CostMonitor.logCost("Recommend", "Recall", t2 - start);
         RankResult rankResult = rankService.rank(convertToRankParam(param, recallResult));
         long t3 = System.currentTimeMillis();
-        log.info("recommendCost param:{} rank cost:{}", JSONObject.toJSONString(request), t3 - t2);
+        CostMonitor.logCost("Recommend", "Rank", t3 - t2);
         saveSortLog(param, rankResult);
 
         RecommendResponse response = buildRecommendResponse(recallResult, rankResult, param.getPublishNum());
-        log.info("recommendCost param:{} response {} cost:{}", JSONObject.toJSONString(request), JSONObject.toJSONString(response),
-                System.currentTimeMillis() - start);
+        long t4  = System.currentTimeMillis();
+        log.info("recommendCost param:{} total cost:{} recall:{} rank:{} response: {}", JSONObject.toJSONString(request),
+                t4 - start, t2 - start, t3 - t2, JSONObject.toJSONString(response));
+        CostMonitor.logCost("Recommend", "Total", t4 - start);
         return response;
     }
 
     private void setStrategy(RecommendRequest request, RecommendParam param) {
+        int historyCount = articleRepository.countByGhIdAndTypeAndItemIndex(request.getGhId(),
+                ArticleTypeEnum.QUNFA.getVal(), 1);
+        if (historyCount < 10) {
+            param.setReplaceSimilarityAccount(true);
+        }
         // 无限发表，设置为无限发表策略
         if (Objects.equals(request.getPushType(), PushTypeEnum.AUTO_PUBLISH.getVal())
                 || Objects.equals(request.getPushType(), PushTypeEnum.ROBOPOST.getVal())) {
@@ -118,8 +119,6 @@ public class RecommendService {
             param.setStrategy(strategyConfig);
         }
         // 历史群发头条小于10条，且开启配置，则走历史表现随机策略
-        int historyCount = articleRepository.countByGhIdAndTypeAndItemIndex(request.getGhId(),
-                ArticleTypeEnum.QUNFA.getVal(), 1);
         if (historyCount < 10 && accountHisJumpStrategyList.contains(request.getGhId())) {
             param.setStrategy(RankStrategyEnum.HIS_JUMP_STRATEGY.getStrategy());
         }
@@ -262,6 +261,7 @@ public class RecommendService {
         rankParam.setScene(param.getScene());
         rankParam.setUserGroupIds(param.getUserGroupIds());
         rankParam.setType(param.getType());
+        rankParam.setReplaceSimilarityAccount(param.getReplaceSimilarityAccount());
 
         return rankParam;
     }
@@ -279,21 +279,6 @@ public class RecommendService {
         if (!ArticleTypeEnum.QUNFA.getVal().equals(param.getType())) {
             return;
         }
-        switch (param.getScene()) {
-            case FWH_COLD_START:
-                break;
-            default:
-                PublishContentSortLog log = new PublishContentSortLog();
-                log.setGhId(param.getGhId());
-                log.setAccountName(param.getAccountName());
-                log.setStrategy(param.getStrategy());
-                List<String> publishContentIds = rankResult.getContents().stream().map(Content::getId).collect(Collectors.toList());
-                log.setPublishContentId(JSONObject.toJSONString(publishContentIds));
-                log.setCreateTimestamp(System.currentTimeMillis());
-                publishContentSortLogRepository.save(log);
-                break;
-
-        }
         String dateStr = DateUtils.getCurrentDateStr("yyyyMMdd");
         List<PublishSortLog> publishSortLogSaveList = new ArrayList<>();
         List<AccountAvgInfo> avgInfoList = accountAvgInfoRepository.getAllByGhIdEqualsAndStatusEquals(param.getGhId(), 1);
@@ -310,7 +295,10 @@ public class RecommendService {
             sortLog.setDateStr(dateStr);
             sortLog.setGhId(param.getGhId());
             sortLog.setAccountName(param.getAccountName());
+            sortLog.setPublishContentId(content.getId());
             sortLog.setCrawlerChannelContentId(content.getCrawlerChannelContentId());
+            sortLog.setSourceType(content.getSourceType());
+            sortLog.setSourceId(content.getSourceId());
             sortLog.setTitle(content.getTitle());
             sortLog.setIndex(i);
             sortLog.setIndexAvgCount(accountIndexAvgViewCountService.getAvgReadCountByDB(avgInfoList, param.getGhId(), i));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/FilterService.java

@@ -98,6 +98,7 @@ public class FilterService {
         strategies.add(ServiceBeanFactory.getBean(DeDuplicationStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(KeywordStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(ArticlePromotionStrategy.class));
+        strategies.add(ServiceBeanFactory.getBean(VideoPoolBadAuditStrategy.class));
         if (param.getScene().equals(FWH_COLD_START)) {
             strategies.add(ServiceBeanFactory.getBean(HistoryTitleForFwhColdStartStrategy.class));
         } else {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/HistoryTitleStrategy.java

@@ -15,10 +15,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.StringUtils;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
@@ -43,9 +40,8 @@ public class HistoryTitleStrategy implements FilterStrategy {
 
     @Override
     public FilterResult filter(FilterParam param) {
-        long start = System.currentTimeMillis();
         FilterResult filterResult = new FilterResult();
-        List<String> result = new ArrayList<>();
+        List<String> result = new ArrayList<>(param.getContents().size());
         List<Content> filterContents = new ArrayList<>();
         List<Article> allArticleList = articleListRemoteService.articleList(param.getGhId(), allIndex, param.getType());
         List<String> allTitleList = allArticleList.stream().map(Article::getTitle).distinct().collect(Collectors.toList());
@@ -58,45 +54,22 @@ public class HistoryTitleStrategy implements FilterStrategy {
         if (Objects.nonNull(contentPoolConfig)) {
             firstSecondContentPool.addAll(Arrays.asList(contentPoolConfig[0], contentPoolConfig[1]));
         }
-        List<Future<Content>> futures = new ArrayList<>();
-        CountDownLatch cdl = new CountDownLatch(param.getContents().size());
+        List<Set<Character>> firstSecondTitleCache = TitleSimilarCheckUtil.makeCache(firstSecondTitleList);
+        List<Set<Character>> allTitleCache = TitleSimilarCheckUtil.makeCache(allTitleList);
+        // TODO: batching for parallelism
         for (Content content : param.getContents()) {
-            Future<Content> future = pool.submit(() -> {
-                try {
-                    boolean isDuplicate;
-                    if (CollectionUtils.isNotEmpty(firstSecondContentPool) && firstSecondContentPool.contains(content.getContentPoolType())) {
-                        // 四个内容池 配置 判断头条，次头条
-                        isDuplicate = TitleSimilarCheckUtil.isDuplicateContent(content.getTitle(), firstSecondTitleList, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
-                    } else {
-                        isDuplicate = TitleSimilarCheckUtil.isDuplicateContent(content.getTitle(), allTitleList, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
-                    }
-                    if (isDuplicate) {
-                        content.setFilterReason("历史已发布文章");
-                    }
-                    return content;
-                } finally {
-                    cdl.countDown();
-                }
-            });
-            futures.add(future);
-        }
-        try {
-            cdl.await();
-        } catch (InterruptedException e) {
-            log.error("filter error", e);
-            return null;
-        }
-
-        for (Future<Content> f : futures) {
-            try {
-                Content content = f.get();
-                if (StringUtils.hasText(content.getFilterReason())) {
-                    filterContents.add(content);
-                } else {
-                    result.add(content.getId());
-                }
-            } catch (Exception e) {
-                log.error("future get error ", e);
+            boolean isDuplicate;
+            if (CollectionUtils.isNotEmpty(firstSecondContentPool) && firstSecondContentPool.contains(content.getContentPoolType())) {
+                // 四个内容池 配置 判断头条，次头条
+                isDuplicate = TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), firstSecondTitleCache, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
+            } else {
+                isDuplicate = TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), allTitleCache, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
+            }
+            if (isDuplicate) {
+                content.setFilterReason("历史已发布文章");
+                filterContents.add(content);
+            } else {
+                result.add(content.getId());
             }
         }
         filterResult.setContentIds(result);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/LowScoreStrategy.java

@@ -41,6 +41,7 @@ public class LowScoreStrategy implements FilterStrategy {
             }
             long publishCount = content.getHisPublishArticleList().stream().filter(ContentHisPublishArticle::isInnerAccount).count();
             if (publishCount == 0) {
+                filterResult.getContentIds().add(content.getId());
                 continue;
             }
             int hisViewCount = content.getHisPublishArticleList().stream().filter(ContentHisPublishArticle::isInnerAccount)

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/SensitiveStrategy.java

@@ -1,26 +1,27 @@
 package com.tzld.longarticle.recommend.server.service.recommend.filter.strategy;
 
-import cn.hutool.core.collection.CollectionUtil;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
+import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
+import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleSensitive;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleUnsafeTitle;
 import com.tzld.longarticle.recommend.server.remote.ArticleSensitiveRemoteService;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleSensitiveRepository;
-import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleSensitive;
+import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleUnsafeTitleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterStrategy;
-import com.tzld.longarticle.recommend.server.util.Md5Util;
 import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import java.util.*;
-import java.util.concurrent.CountDownLatch;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.stream.Collectors;
 
 @Component
 @Slf4j
@@ -30,6 +31,8 @@ public class SensitiveStrategy implements FilterStrategy {
     private ArticleSensitiveRemoteService articleSensitiveRemoteService;
     @Autowired
     private ArticleSensitiveRepository articleSensitiveRepository;
+    @Autowired
+    private ArticleUnsafeTitleRepository articleUnsafeTitleRepository;
 
     @ApolloJsonValue("${UnSafeTitles:[]}")
     private static List<String> UnSafeTitles;
@@ -54,8 +57,14 @@ public class SensitiveStrategy implements FilterStrategy {
 //        if (CollectionUtil.isNotEmpty(articleSensitiveList)) {
 //            articleSensitiveMap = articleSensitiveList.stream().collect(Collectors.toMap(ArticleSensitive::getMd5, o -> o));
 //        }
-
-        List<Set<Character>> unsafeTitleCache = TitleSimilarCheckUtil.makeCache(UnSafeTitles);
+        List<String> allUnSafeTitles = UnSafeTitles;
+        List<ArticleUnsafeTitle> articleUnsafeTitleList = articleUnsafeTitleRepository.getByStatus(StatusEnum.ONE.getCode());
+        if (CollectionUtils.isNotEmpty(articleUnsafeTitleList)) {
+            for (ArticleUnsafeTitle articleUnsafeTitle : articleUnsafeTitleList) {
+                allUnSafeTitles.add(articleUnsafeTitle.getTitle());
+            }
+        }
+        List<Set<Character>> unsafeTitleCache = TitleSimilarCheckUtil.makeCache(allUnSafeTitles);
         for (Content content : param.getContents()) {
             try {
                 if (TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), unsafeTitleCache,

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/VideoPoolBadAuditStrategy.java

@@ -0,0 +1,59 @@
+package com.tzld.longarticle.recommend.server.service.recommend.filter.strategy;
+
+import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
+import com.tzld.longarticle.recommend.server.model.dto.Content;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource;
+import com.tzld.longarticle.recommend.server.repository.longArticle.PublishSingleVideoSourceRepository;
+import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterParam;
+import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterResult;
+import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class VideoPoolBadAuditStrategy implements FilterStrategy {
+
+    @Autowired
+    private PublishSingleVideoSourceRepository publishSingleVideoSourceRepository;
+
+    @Override
+    public FilterResult filter(FilterParam param) {
+        FilterResult filterResult = new FilterResult();
+        List<String> result = new ArrayList<>(param.getContents().size());
+        List<Content> filterContents = new ArrayList<>();
+        List<String> videoPoolContentSourceIds = param.getContents().stream().map(Content::getSourceId).collect(Collectors.toList());
+        List<PublishSingleVideoSource> videoPoolList = publishSingleVideoSourceRepository.getByContentTraceIdIn(videoPoolContentSourceIds);
+        Map<String, PublishSingleVideoSource> videoPoolMap = videoPoolList.stream().collect(Collectors.toMap(PublishSingleVideoSource::getContentTraceId, o -> o));
+        for (Content content : param.getContents()) {
+            if (!PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal().equals(content.getSourceType())) {
+                result.add(content.getId());
+                continue;
+            }
+            PublishSingleVideoSource videoPool = videoPoolMap.get(content.getSourceId());
+            if (Objects.isNull(videoPool)) {
+                content.setFilterReason("视频内容池不存在");
+                filterContents.add(content);
+            } else if (videoPool.getBadStatus() != 0) {
+                content.setFilterReason("视频内容池已退场");
+                filterContents.add(content);
+            } else if (videoPool.getAuditStatus() != 1) {
+                content.setFilterReason("视频内容池审核不通过");
+                filterContents.add(content);
+            } else {
+                result.add(content.getId());
+            }
+        }
+        filterResult.setContentIds(result);
+        filterResult.setFilterContent(filterContents);
+        return filterResult;
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/RankParam.java

@@ -19,5 +19,6 @@ public class RankParam {
     private String scene;
     private List<Integer> userGroupIds;
     private String type;
+    private Boolean replaceSimilarityAccount = false;
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/RankService.java

@@ -1,16 +1,28 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank;
 
 
+import cn.hutool.core.collection.CollectionUtil;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.model.ConfigChangeEvent;
+import com.ctrip.framework.apollo.spring.annotation.ApolloConfigChangeListener;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.service.ServiceBeanFactory;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.strategy.FwhColdStartRankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 import static com.tzld.longarticle.recommend.server.common.constant.SceneConstants.FWH_COLD_START;
 
@@ -19,7 +31,38 @@ import static com.tzld.longarticle.recommend.server.common.constant.SceneConstan
  */
 @Service
 @Slf4j
-public class RankService {
+public class RankService implements InitializingBean {
+
+    private static Map<String, Map<String, Integer>> staticStrategyPoolSourceTypeMap;
+
+    @ApolloJsonValue("${strategyPoolSourceTypeConfig:{}}")
+    private Map<String, Map<String, Integer>> strategyPoolSourceTypeMap;
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        RankService.staticStrategyPoolSourceTypeMap = strategyPoolSourceTypeMap;
+    }
+
+    @ApolloConfigChangeListener(interestedKeys = {"strategyPoolSourceTypeConfig"})
+    public void configChange(ConfigChangeEvent changeEvent) {
+        RankService.staticStrategyPoolSourceTypeMap = JSONObject.parseObject(changeEvent.getChange(
+                "strategyPoolSourceTypeConfig").getNewValue(), Map.class);
+        log.info("strategyPoolSourceTypeConfig change updateStaticValue success newValue:{}",
+                JSONObject.toJSONString(staticStrategyPoolSourceTypeMap));
+    }
+
+    public static Integer getStrategyPoolSourceType(String strategy, Integer index) {
+        Integer sourceType = 0;
+        Map<String, Integer> indexSourceTypeMap = staticStrategyPoolSourceTypeMap.get(strategy);
+        if (Objects.nonNull(indexSourceTypeMap)) {
+            sourceType = indexSourceTypeMap.get(String.valueOf(index));
+        }
+        return sourceType;
+    }
+
+    public static Map<String, Integer> getStrategyPoolSourceType(String strategy) {
+        return staticStrategyPoolSourceTypeMap.get(strategy);
+    }
 
     public RankResult rank(RankParam param) {
         RankStrategy strategy = getRankStrategy(param);
@@ -38,8 +81,11 @@ public class RankService {
 
     }
 
-    public static void printSortLog(String strategy, String accountName, List<Content> contentList) {
+    public static void printSortLog(String strategy, String accountName, String position, List<Content> contentList) {
         JSONArray jsonArray = new JSONArray();
+        if (CollectionUtil.isEmpty(contentList)) {
+            return;
+        }
         for (Content content : contentList) {
             JSONObject obj = new JSONObject();
             obj.put("id", content.getId());
@@ -48,6 +94,71 @@ public class RankService {
             obj.put("scoreMap", content.getScoreMap());
             jsonArray.add(obj);
         }
-        log.info("{} 账号名称 {} 头条评分结果 {}", strategy, accountName, JSONObject.toJSONString(jsonArray));
+        log.info("{} 账号名称 {} {}评分结果 {}", strategy, accountName, position, JSONObject.toJSONString(jsonArray));
+    }
+
+    public static void commonAddSecondContent(RankParam param, List<Content> result,
+                                              String[] publishPool, String[] contentPools,
+                                              Map<String, List<Content>> contentMap,
+                                              Map<Integer, AccountIndexReplacePoolConfig> indexReplacePoolConfigMap) {
+        List<Content> pool = contentMap.get(contentPools[1]);
+        if (CollectionUtils.isNotEmpty(pool)) {
+            pool = contentSourceTypeFilter(param.getStrategy(), pool, 2);
+        }
+        if (CollectionUtils.isNotEmpty(pool)) {
+            int i = RandomUtils.nextInt(0, Math.min(pool.size(), 5));
+            int j = RandomUtils.nextInt(0, Math.min(pool.size(), 5));
+            result.add(pool.get(i));
+            // 替补 头条内容不足使用次条内容
+            if (result.size() == 1 && pool.size() > 1) {
+                while (i == j && pool.size() > 1) {
+                    j = RandomUtils.nextInt(0, Math.min(pool.size(), 5));
+                    if (i != j) {
+                        publishPool[0] = contentPools[1];
+                        result.add(pool.get(1));
+                        break;
+                    }
+                }
+            }
+        } else {
+            // 替补 根据设置替补内容池查找内容进行替补
+            findReplacePoolContent(result, publishPool, contentMap, indexReplacePoolConfigMap);
+        }
+    }
+
+    public static void findReplacePoolContent(List<Content> result, String[] publishPool,
+                                              Map<String, List<Content>> contentMap,
+                                              Map<Integer, AccountIndexReplacePoolConfig> indexReplacePoolConfigMap) {
+        AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
+        if (Objects.nonNull(replacePoolConfig)) {
+            List<Content> poolReplace = contentMap.get(replacePoolConfig.getContentPool());
+            if (CollectionUtils.isNotEmpty(poolReplace)) {
+                publishPool[1] = replacePoolConfig.getContentPool();
+                result.add(poolReplace.get(0));
+            }
+        }
+    }
+
+    public static List<Content> contentSourceTypeFilter(String strategy, List<Content> pool, Integer index) {
+        Integer sourceType = getStrategyPoolSourceType(strategy, index);
+        Integer videoSourceType = PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal();
+        if (Objects.nonNull(sourceType) && sourceType.equals(videoSourceType)) {
+            pool = pool.stream().filter(o -> Objects.equals(o.getSourceType(), videoSourceType)).collect(Collectors.toList());
+        } else {
+            pool = pool.stream().filter(o -> !Objects.equals(o.getSourceType(), videoSourceType)).collect(Collectors.toList());
+        }
+        return pool;
+    }
+
+    public static void commonAdd38Content(RankParam param, List<Content> result, String[] contentPools,
+                                          Map<String, List<Content>> contentMap, String strategy) {
+        List<Content> pool = contentMap.get(contentPools[2]);
+        if (CollectionUtils.isNotEmpty(pool)) {
+            pool = contentSourceTypeFilter(param.getStrategy(), pool, 3);
+        }
+        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
+            RankService.printSortLog(strategy, param.getAccountName(), "3-8", pool.subList(0, Math.min(pool.size(), 200)));
+            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
+        }
     }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/RankStrategy.java

@@ -64,6 +64,7 @@ public interface RankStrategy {
         scoreParam.setContents(param.getContents());
         scoreParam.setStrategy(param.getStrategy());
         scoreParam.setScene(param.getScene());
+        scoreParam.setReplaceSimilarityAccount(param.getReplaceSimilarityAccount());
         return scoreParam;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/HisJumpRankStrategy.java

@@ -1,9 +1,7 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
-import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
-import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
 import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
@@ -28,8 +26,6 @@ public class HisJumpRankStrategy implements RankStrategy {
     @Autowired
     private AccountContentPoolConfigService accountContentPoolConfigService;
     @Autowired
-    private ArticleRepository articleRepository;
-    @Autowired
     private StrategyIndexScoreWeightService weightService;
 
     public RankResult rank(RankParam param) {
@@ -51,6 +47,9 @@ public class HisJumpRankStrategy implements RankStrategy {
             if (contentPools[0].equals(item.getContent().getContentPoolType())
                     || contentPools[1].equals(item.getContent().getContentPoolType())) {
                 score = item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value())
+                        + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value());
                 if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
                     score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
@@ -59,6 +58,9 @@ public class HisJumpRankStrategy implements RankStrategy {
                 }
             } else {
                 score = item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
+                        + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value())
                         + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value())
                         + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
@@ -84,7 +86,7 @@ public class HisJumpRankStrategy implements RankStrategy {
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
-        RankService.printSortLog(RankStrategyEnum.HIS_JUMP_STRATEGY.getStrategy(), param.getAccountName(), pool1);
+        RankService.printSortLog(param.getStrategy(), param.getAccountName(), "头条", pool1);
         if (CollectionUtils.isNotEmpty(pool1)) {
             int i = RandomUtils.nextInt(0, Math.min(pool1.size(), 20));
             result.add(pool1.get(i));
@@ -113,7 +115,10 @@ public class HisJumpRankStrategy implements RankStrategy {
         // 3-8
         List<Content> pool = contentMap.get(contentPools[2]);
         if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
+            int slotNum = param.getSize() - result.size();
+            List<Content> subPool = pool.subList(0, Math.min(pool.size(), 30));
+            Collections.shuffle(subPool);
+            result.addAll(subPool.subList(0, Math.min(subPool.size(), slotNum)));
         }
 
         RankStrategy.deduplication(result, contentMap, publishPool);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/InfiniteRankStrategy.java

@@ -1,19 +1,16 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
+import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
 import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreParam;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -76,10 +73,7 @@ public class InfiniteRankStrategy implements RankStrategy {
         }
 
         // 全部使用3-8内容池
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool)) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         return new RankResult(result);
     }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/LateRankStrategy.java

@@ -1,19 +1,16 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
+import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
 import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreParam;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -76,10 +73,7 @@ public class LateRankStrategy implements RankStrategy {
         }
 
         // 全部使用3-8内容池
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool)) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         return new RankResult(result);
     }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RandomRankStrategy.java

@@ -3,10 +3,7 @@ package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
 import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV10Strategy.java

@@ -1,23 +1,20 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
 
+import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
 import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
 import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -112,39 +109,10 @@ public class RankV10Strategy implements RankStrategy {
             return new RankResult(result);
         }
         // 次
-        List<Content> pool2 = contentMap.get(contentPools[1]);
-        if (CollectionUtils.isNotEmpty(pool2)) {
-            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            int j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            result.add(pool2.get(i));
-            // 替补 头条内容不足使用次条内容
-            if (result.size() == 1 && pool2.size() > 1) {
-                while (i == j && pool2.size() > 1) {
-                    j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-                    if (i != j) {
-                        publishPool[0] = contentPools[1];
-                        result.add(pool2.get(1));
-                        break;
-                    }
-                }
-            }
-        } else {
-            // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
-        }
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV11Strategy.java

@@ -15,7 +15,6 @@ import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreServic
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -122,7 +121,7 @@ public class RankV11Strategy implements RankStrategy {
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
-        RankService.printSortLog(RankStrategyEnum.ArticleRankV11.getStrategy(), param.getAccountName(), pool1);
+        RankService.printSortLog(param.getStrategy(), param.getAccountName(), "头条", pool1);
         if (CollectionUtils.isNotEmpty(pool1)) {
             result.add(pool1.get(0));
         } else {
@@ -130,39 +129,10 @@ public class RankV11Strategy implements RankStrategy {
             return new RankResult(result);
         }
         // 次
-        List<Content> pool2 = contentMap.get(contentPools[1]);
-        if (CollectionUtils.isNotEmpty(pool2)) {
-            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            int j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            result.add(pool2.get(i));
-            // 替补 头条内容不足使用次条内容
-            if (result.size() == 1 && pool2.size() > 1) {
-                while (i == j && pool2.size() > 1) {
-                    j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-                    if (i != j) {
-                        publishPool[0] = contentPools[1];
-                        result.add(pool2.get(1));
-                        break;
-                    }
-                }
-            }
-        } else {
-            // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
-        }
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV12Strategy.java

@@ -15,7 +15,6 @@ import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreServic
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -122,7 +121,7 @@ public class RankV12Strategy implements RankStrategy {
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
-        RankService.printSortLog(RankStrategyEnum.ArticleRankV12.getStrategy(), param.getAccountName(), pool1);
+        RankService.printSortLog(param.getStrategy(), param.getAccountName(), "头条", pool1);
         if (CollectionUtils.isNotEmpty(pool1)) {
             result.add(pool1.get(0));
         } else {
@@ -130,39 +129,10 @@ public class RankV12Strategy implements RankStrategy {
             return new RankResult(result);
         }
         // 次
-        List<Content> pool2 = contentMap.get(contentPools[1]);
-        if (CollectionUtils.isNotEmpty(pool2)) {
-            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            int j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            result.add(pool2.get(i));
-            // 替补 头条内容不足使用次条内容
-            if (result.size() == 1 && pool2.size() > 1) {
-                while (i == j && pool2.size() > 1) {
-                    j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-                    if (i != j) {
-                        publishPool[0] = contentPools[1];
-                        result.add(pool2.get(1));
-                        break;
-                    }
-                }
-            }
-        } else {
-            // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
-        }
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV13Strategy.java

@@ -15,7 +15,6 @@ import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreServic
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -123,7 +122,7 @@ public class RankV13Strategy implements RankStrategy {
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
-        RankService.printSortLog(RankStrategyEnum.ArticleRankV13.getStrategy(), param.getAccountName(), pool1);
+        RankService.printSortLog(param.getStrategy(), param.getAccountName(), "头条", pool1);
         if (CollectionUtils.isNotEmpty(pool1)) {
             result.add(pool1.get(0));
         } else {
@@ -131,39 +130,10 @@ public class RankV13Strategy implements RankStrategy {
             return new RankResult(result);
         }
         // 次
-        List<Content> pool2 = contentMap.get(contentPools[1]);
-        if (CollectionUtils.isNotEmpty(pool2)) {
-            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            int j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            result.add(pool2.get(i));
-            // 替补 头条内容不足使用次条内容
-            if (result.size() == 1 && pool2.size() > 1) {
-                while (i == j && pool2.size() > 1) {
-                    j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-                    if (i != j) {
-                        publishPool[0] = contentPools[1];
-                        result.add(pool2.get(1));
-                        break;
-                    }
-                }
-            }
-        } else {
-            // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
-        }
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV14Strategy.java

@@ -15,7 +15,6 @@ import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreServic
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -123,7 +122,7 @@ public class RankV14Strategy implements RankStrategy {
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
-        RankService.printSortLog(RankStrategyEnum.ArticleRankV14.getStrategy(), param.getAccountName(), pool1);
+        RankService.printSortLog(param.getStrategy(), param.getAccountName(), "头条", pool1);
         if (CollectionUtils.isNotEmpty(pool1)) {
             result.add(pool1.get(0));
         } else {
@@ -131,39 +130,10 @@ public class RankV14Strategy implements RankStrategy {
             return new RankResult(result);
         }
         // 次
-        List<Content> pool2 = contentMap.get(contentPools[1]);
-        if (CollectionUtils.isNotEmpty(pool2)) {
-            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            int j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            result.add(pool2.get(i));
-            // 替补 头条内容不足使用次条内容
-            if (result.size() == 1 && pool2.size() > 1) {
-                while (i == j && pool2.size() > 1) {
-                    j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-                    if (i != j) {
-                        publishPool[0] = contentPools[1];
-                        result.add(pool2.get(1));
-                        break;
-                    }
-                }
-            }
-        } else {
-            // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
-        }
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV15Strategy.java

@@ -0,0 +1,149 @@
+package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
+
+
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
+import com.tzld.longarticle.recommend.server.model.dto.Content;
+import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
+import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
+import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
+import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
+import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
+import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
+import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Service
+@Slf4j
+public class RankV15Strategy implements RankStrategy {
+
+    @Autowired
+    private ScoreService scoreService;
+    @Autowired
+    private AccountContentPoolConfigService accountContentPoolConfigService;
+    @Autowired
+    private ArticleRepository articleRepository;
+    @Autowired
+    private StrategyIndexScoreWeightService weightService;
+
+    @ApolloJsonValue("${touliu.account.ghIds:[\"gh_93e00e187787\", \"gh_ac43e43b253b\", \"gh_68e7fdc09fe4\",\"gh_77f36c109fb1\", \"gh_b181786a6c8c\", \"gh_1ee2e1b39ccf\"]}")
+    private List<String> touliuAccountGhIds;
+
+    public RankResult rank(RankParam param) {
+        List<Content> result = new ArrayList<>();
+
+        ScoreResult scoreResult = scoreService.score(RankStrategy.convertToScoreParam(param));
+
+        Map<String, Map<String, Double>> scoreMap = scoreResult.getScoreMap();
+        String[] contentPools = accountContentPoolConfigService.getContentPools(param.getAccountName());
+        Map<Integer, AccountIndexReplacePoolConfig> indexReplacePoolConfigMap = accountContentPoolConfigService.getContentReplacePools(param.getAccountName());
+
+        List<RankItem> items = CommonCollectionUtils.toList(param.getContents(), c -> {
+            RankItem item = new RankItem();
+            item.setContent(c);
+            c.setScoreMap(scoreMap.get(c.getId()));
+            item.setScoreMap(scoreMap.get(c.getId()));
+            double score;
+            int index = weightService.getIndex(item.getContent().getContentPoolType(), contentPools);
+            if (contentPools[0].equals(item.getContent().getContentPoolType())
+                    || contentPools[1].equals(item.getContent().getContentPoolType())) {
+                score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.SIMILARITY.value())
+                        + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
+                        + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value())
+                        + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value());
+                if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
+                    score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
+                            * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                            ScoreStrategyEnum.VIEW_COUNT_RATE.value());
+                }
+            } else {
+                score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.SIMILARITY.value())
+                        + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
+                        + item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
+                        + item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value())
+                        + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value())
+                        + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
+            }
+            c.setScore(score);
+            item.setScore(score);
+            return item;
+        });
+        // 相似度评分为0 报警返回
+        List<Article> hisPublishFirstArticleList = articleRepository.getByGhIdAndItemIndexAndTypeEqualsAndStatusEquals(
+                param.getGhId(), 1, param.getType(), 1);
+        if (RankStrategy.SimilarityScoreZero(items, param, hisPublishFirstArticleList)) {
+            return new RankResult(result);
+        }
+
+        // 1 排序
+        Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
+        // 2 相似去重
+        List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
+
+        // 3 文章按照内容池分组
+        Map<String, List<Content>> contentMap = new HashMap<>();
+        for (Content c : contents) {
+            List<Content> data = contentMap.computeIfAbsent(c.getContentPoolType(), k -> new ArrayList<>());
+            data.add(c);
+        }
+        // 4 选文章
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
+
+        // 头
+        List<Content> pool1 = contentMap.get(contentPools[0]);
+        if (CollectionUtils.isNotEmpty(pool1)) {
+            pool1 = RankService.contentSourceTypeFilter(param.getStrategy(), pool1, 1);
+        }
+        RankService.printSortLog(param.getStrategy(), param.getAccountName(), "头条", pool1);
+        if (CollectionUtils.isNotEmpty(pool1)) {
+            result.add(pool1.get(0));
+        } else {
+            RankStrategy.sendFeishuFirstPoolEmpty(param, contentPools[0]);
+            return new RankResult(result);
+        }
+        // 次
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
+
+        // 3-8
+        // RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
+        List<Content> pool = contentMap.get(contentPools[2]);
+        Integer videoSourceType = PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal();
+        Queue<Content> videoPoolQueue = pool.stream().filter(o -> Objects.equals(o.getSourceType(), videoSourceType))
+                .collect(Collectors.toCollection(LinkedList::new));
+        Queue<Content> otherPoolQueue = pool.stream().filter(o -> !Objects.equals(o.getSourceType(), videoSourceType))
+                .collect(Collectors.toCollection(LinkedList::new));
+        for (int i = 3; i < param.getSize() + 1; i++) {
+            Integer sourceType = RankService.getStrategyPoolSourceType(param.getStrategy(), i);
+            if (Objects.equals(sourceType, videoSourceType) && !videoPoolQueue.isEmpty()) {
+                result.add(videoPoolQueue.poll());
+            } else if (!otherPoolQueue.isEmpty()) {
+                result.add(otherPoolQueue.poll());
+            }
+        }
+
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
+        return new RankResult(result);
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV3Strategy.java

@@ -1,15 +1,13 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
 
+import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
 import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
@@ -108,21 +106,11 @@ public class RankV3Strategy implements RankStrategy {
             result.add(pool2.get(i));
         } else {
             // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
+            RankService.findReplacePoolContent(result, publishPool, contentMap, indexReplacePoolConfigMap);
         }
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV4Strategy.java

@@ -1,22 +1,19 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
 
+import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
 import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -102,39 +99,10 @@ public class RankV4Strategy implements RankStrategy {
             return new RankResult(result);
         }
         // 次
-        List<Content> pool2 = contentMap.get(contentPools[1]);
-        if (CollectionUtils.isNotEmpty(pool2)) {
-            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            int j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            result.add(pool2.get(i));
-            // 替补 头条内容不足使用次条内容
-            if (result.size() == 1 && pool2.size() > 1) {
-                while (i == j && pool2.size() > 1) {
-                    j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-                    if (i != j) {
-                        publishPool[0] = contentPools[1];
-                        result.add(pool2.get(1));
-                        break;
-                    }
-                }
-            }
-        } else {
-            // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
-        }
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV5Strategy.java

@@ -2,7 +2,6 @@ package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
@@ -16,7 +15,6 @@ import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreServic
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -111,7 +109,7 @@ public class RankV5Strategy implements RankStrategy {
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
-        RankService.printSortLog(RankStrategyEnum.ArticleRankV5.getStrategy(), param.getAccountName(), pool1);
+        RankService.printSortLog(param.getStrategy(), param.getAccountName(), "头条", pool1);
         if (CollectionUtils.isNotEmpty(pool1)) {
             result.add(pool1.get(0));
         } else {
@@ -119,39 +117,10 @@ public class RankV5Strategy implements RankStrategy {
             return new RankResult(result);
         }
         // 次
-        List<Content> pool2 = contentMap.get(contentPools[1]);
-        if (CollectionUtils.isNotEmpty(pool2)) {
-            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            int j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            result.add(pool2.get(i));
-            // 替补 头条内容不足使用次条内容
-            if (result.size() == 1 && pool2.size() > 1) {
-                while (i == j && pool2.size() > 1) {
-                    j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-                    if (i != j) {
-                        publishPool[0] = contentPools[1];
-                        result.add(pool2.get(1));
-                        break;
-                    }
-                }
-            }
-        } else {
-            // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
-        }
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV7Strategy.java

@@ -1,22 +1,19 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
 
+import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
 import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -104,39 +101,10 @@ public class RankV7Strategy implements RankStrategy {
             return new RankResult(result);
         }
         // 次
-        List<Content> pool2 = contentMap.get(contentPools[1]);
-        if (CollectionUtils.isNotEmpty(pool2)) {
-            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            int j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            result.add(pool2.get(i));
-            // 替补 头条内容不足使用次条内容
-            if (result.size() == 1 && pool2.size() > 1) {
-                while (i == j && pool2.size() > 1) {
-                    j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-                    if (i != j) {
-                        publishPool[0] = contentPools[1];
-                        result.add(pool2.get(1));
-                        break;
-                    }
-                }
-            }
-        } else {
-            // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
-        }
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV8Strategy.java

@@ -15,7 +15,6 @@ import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreServic
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -106,7 +105,7 @@ public class RankV8Strategy implements RankStrategy {
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
-        RankService.printSortLog(RankStrategyEnum.ArticleRankV8.getStrategy(), param.getAccountName(), pool1);
+        RankService.printSortLog(param.getStrategy(), param.getAccountName(), "头条", pool1);
         if (CollectionUtils.isNotEmpty(pool1)) {
             result.add(pool1.get(0));
         } else {
@@ -114,39 +113,10 @@ public class RankV8Strategy implements RankStrategy {
             return new RankResult(result);
         }
         // 次
-        List<Content> pool2 = contentMap.get(contentPools[1]);
-        if (CollectionUtils.isNotEmpty(pool2)) {
-            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            int j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            result.add(pool2.get(i));
-            // 替补 头条内容不足使用次条内容
-            if (result.size() == 1 && pool2.size() > 1) {
-                while (i == j && pool2.size() > 1) {
-                    j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-                    if (i != j) {
-                        publishPool[0] = contentPools[1];
-                        result.add(pool2.get(1));
-                        break;
-                    }
-                }
-            }
-        } else {
-            // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
-        }
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV9Strategy.java

@@ -1,22 +1,19 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
 
+import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
 import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -104,39 +101,10 @@ public class RankV9Strategy implements RankStrategy {
             return new RankResult(result);
         }
         // 次
-        List<Content> pool2 = contentMap.get(contentPools[1]);
-        if (CollectionUtils.isNotEmpty(pool2)) {
-            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            int j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-            result.add(pool2.get(i));
-            // 替补 头条内容不足使用次条内容
-            if (result.size() == 1 && pool2.size() > 1) {
-                while (i == j && pool2.size() > 1) {
-                    j = RandomUtils.nextInt(0, Math.min(pool2.size(), 5));
-                    if (i != j) {
-                        publishPool[0] = contentPools[1];
-                        result.add(pool2.get(1));
-                        break;
-                    }
-                }
-            }
-        } else {
-            // 替补 根据设置替补内容池查找内容尽心替补
-            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
-            if (Objects.nonNull(replacePoolConfig)) {
-                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
-                if (CollectionUtils.isNotEmpty(pool2Replace)) {
-                    publishPool[1] = replacePoolConfig.getContentPool();
-                    result.add(pool2Replace.get(0));
-                }
-            }
-        }
+        RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        List<Content> pool = contentMap.get(contentPools[2]);
-        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
-        }
+        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -1,12 +1,15 @@
 package com.tzld.longarticle.recommend.server.service.recommend.recall;
 
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
+import com.tzld.longarticle.recommend.server.common.CostMonitor;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
-import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
-import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
+import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.*;
 import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
+import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.dto.ContentHisPublishArticle;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.CrawlerMetaArticle;
@@ -15,9 +18,8 @@ import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountCorrelation;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleDetailInfo;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.AccountCategory;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticlePoolPromotionSource;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
+import com.tzld.longarticle.recommend.server.model.param.TitleHisCacheParam;
 import com.tzld.longarticle.recommend.server.remote.aigc.AIGCWaitingPublishContentService;
 import com.tzld.longarticle.recommend.server.repository.aigc.CrawlerMetaArticleRepository;
 import com.tzld.longarticle.recommend.server.repository.aigc.PublishContentRepository;
@@ -25,9 +27,7 @@ import com.tzld.longarticle.recommend.server.repository.crawler.AccountAvgInfoRe
 import com.tzld.longarticle.recommend.server.repository.crawler.AccountCorrelationRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleDetailInfoRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
-import com.tzld.longarticle.recommend.server.repository.longArticle.AccountCategoryRepository;
-import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleCategoryRepository;
-import com.tzld.longarticle.recommend.server.repository.longArticle.ArticlePoolPromotionSourceRepository;
+import com.tzld.longarticle.recommend.server.repository.longArticle.*;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountIndexAvgViewCountService;
 import com.tzld.longarticle.recommend.server.service.recommend.recall.strategy.DefaultRecallStrategy;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreStrategy;
@@ -87,6 +87,12 @@ public class RecallService implements ApplicationContextAware {
     PublishContentRepository publishContentRepository;
     @Autowired
     AccountCategoryRepository accountCategoryRepository;
+    @Autowired
+    PublishSingleVideoSourceRepository publishSingleVideoSourceRepository;
+    @Autowired
+    ArticleTitleHisCacheRepository articleTitleHisCacheRepository;
+    @Autowired
+    LongArticleBaseMapper longArticleBaseMapper;
 
     private final Map<String, RecallStrategy> strategyMap = new HashMap<>();
     private ApplicationContext applicationContext;
@@ -108,15 +114,12 @@ public class RecallService implements ApplicationContextAware {
     }
 
     public RecallResult recall(RecallParam param) {
-        long t1 = System.currentTimeMillis();
         List<RecallResult.RecallData> results = new ArrayList<>();
         log.info("RecallParam {}", JSONUtils.toJson(param));
         List<RecallStrategy> strategies = getRecallStrategy(param);
         log.info("RecallStrategy {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies,
                 s -> s.getClass().getSimpleName())));
         List<Content> content = getAllContent(param);
-        long t2 = System.currentTimeMillis();
-        log.info("recall account:{} get content:{}", param.getAccountName(), t2 - t1);
         if (CollectionUtils.isEmpty(content)) {
             return new RecallResult(results);
         }
@@ -164,7 +167,10 @@ public class RecallService implements ApplicationContextAware {
     }
 
     private List<Content> getAllContent(RecallParam param) {
+        long t1 = System.currentTimeMillis();
         List<Content> content = aigcWaitingPublishContentService.getAllContent(param);
+        long t2 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "GetAllContents", t2 - t1);
         if (CollectionUtils.isEmpty(content)) {
             FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
                     "内容召回失败\n"
@@ -173,48 +179,37 @@ public class RecallService implements ApplicationContextAware {
                             + "账号名称: " + param.getAccountName());
             return content;
         }
-        // category 查询
-//        setContentCategory(content);
+        long t3 = System.currentTimeMillis();
         // 标题历史均值
         setTitleAvgViewCount(content, param.getGhId(), param.getType());
+        long t4 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "SetAvgViewCount", t4 - t3);
+        // 视频内容池查询抓取时间
+        setVideoContent(content);
         return content;
     }
 
-    public void setContentCategory(List<Content> contentList) {
-        List<String> channelContentIds = contentList.stream().map(Content::getCrawlerChannelContentId).collect(Collectors.toList());
-        // 查询晋升rootProduceContentId
-        List<ArticlePoolPromotionSource> sourceList = articlePoolPromotionSourceRepository.getByChannelContentIdIn(channelContentIds);
-        Map<String, ArticlePoolPromotionSource> sourceMap = sourceList.stream().collect(Collectors.toMap(ArticlePoolPromotionSource::getChannelContentId, Function.identity()));
-        List<String> publishContentIds = sourceList.stream().map(ArticlePoolPromotionSource::getRootPublishContentId).collect(Collectors.toList());
-        List<PublishContent> publishContentList = publishContentRepository.getByIdIn(publishContentIds);
-        Map<String, PublishContent> publishContentMap = publishContentList.stream().collect(Collectors.toMap(PublishContent::getId, Function.identity()));
-        // 根据produceContentId查询category
-        List<ArticleCategory> articleCategoryList = articleCategoryRepository.findAll();
-        Map<String, ArticleCategory> categoryMap = articleCategoryList.stream().collect(Collectors.toMap(ArticleCategory::getProduceContentId, Function.identity()));
-        Map<String, ArticleCategory> coldStartCategoryMap = articleCategoryList.stream().collect(Collectors.toMap(ArticleCategory::getChannelContentId, Function.identity(), (a, b) -> a));
-        Map<String, ArticleCategory> titleCategoryMap = articleCategoryList.stream().collect(Collectors.toMap(ArticleCategory::getTitleMd5, Function.identity(), (a, b) -> a));
-        for (Content content : contentList) {
-            ArticlePoolPromotionSource source = sourceMap.get(content.getCrawlerChannelContentId());
-            ArticleCategory category = null;
-            if (Objects.nonNull(source) && Objects.nonNull(source.getRootProduceContentId())) {
-                category = categoryMap.get(source.getRootProduceContentId());
-                PublishContent publishContent = publishContentMap.get(source.getRootPublishContentId());
-                if (Objects.nonNull(publishContent)) {
-                    content.setRootPublishTimestamp(publishContent.getPublishTimestamp());
-                }
-            }
-            if (Objects.isNull(category)) {
-                category = coldStartCategoryMap.get(content.getCrawlerChannelContentId());
-            }
-            if (Objects.isNull(category)) {
-                String titleMd5 = Md5Util.encoderByMd5(content.getTitle());
-                category = titleCategoryMap.get(titleMd5);
-            }
-            if (Objects.nonNull(category)) {
-                content.setCategory(Collections.singletonList(category.getCategory()));
+    private void setVideoContent(List<Content> contentList) {
+        List<String> contentTraceIds = contentList.stream()
+                .filter(o -> PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal().equals(o.getSourceType()))
+                .map(Content::getSourceId).collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(contentTraceIds)) {
+            return;
+        }
+        Map<String, Content> contentMap = contentList.stream()
+                .collect(Collectors.toMap(Content::getSourceId, Function.identity()));
+        List<PublishSingleVideoSource> sourceList = publishSingleVideoSourceRepository.getByContentTraceIdIn(contentTraceIds);
+        Map<String, PublishSingleVideoSource> sourceMap = sourceList.stream()
+                .collect(Collectors.toMap(PublishSingleVideoSource::getContentTraceId, Function.identity()));
+        for (String contentTraceId : contentTraceIds) {
+            Content content = contentMap.get(contentTraceId);
+            PublishSingleVideoSource source = sourceMap.get(contentTraceId);
+            if (Objects.isNull(source)) {
                 continue;
             }
-            log.error("setContentCategory NullError channelContentId:{}", content.getCrawlerChannelContentId());
+            content.setCrawlerTimestamp(source.getCrawlerTimestamp());
+            ContentPoolEnum poolEnum = ContentPoolEnum.from(source.getFlowPoolLevel());
+            content.setContentPoolType(poolEnum.getContentPool());
         }
     }
 
@@ -252,20 +247,117 @@ public class RecallService implements ApplicationContextAware {
 
     public void setTitleAvgViewCount(List<Content> contentList, String ghId, String type) {
         long start = System.currentTimeMillis();
+        contentList.forEach(content -> content.setTitleMd5(Md5Util.encoderByMd5(content.getTitle())));
+        List<String> sourceIdList = contentList.stream().map(Content::getSourceId).distinct().collect(Collectors.toList());
+        Map<String, Content> sourceIdToContentMap = contentList.stream().collect(
+                Collectors.toMap(Content::getSourceId, Function.identity(), (o1, o2) -> o2));
+        // 根据sourceId查询数据库获取数据
+        List<ArticleTitleHisCache> articleTitleHisCacheList = new ArrayList<>(sourceIdList.size());
+        for (List<String> partition : Lists.partition(sourceIdList, 1000)) {
+            articleTitleHisCacheList.addAll(articleTitleHisCacheRepository.getBySourceIdInAndType(partition, type));
+        }
+        Map<String, ArticleTitleHisCache> articleTitleHisCacheMap = articleTitleHisCacheList.stream()
+                .collect(Collectors.toMap(ArticleTitleHisCache::getSourceId, Function.identity()));
+        // sourceId 进行过滤 排除缓存中数据 重新走下方查询
+        sourceIdList.removeIf(articleTitleHisCacheMap::containsKey);
         // 获取账号相关性
         List<AccountCorrelation> accountCorrelationList = accountCorrelationRepository.findByGhIdAndStatus(ghId, 1);
         Map<String, Double> accountCorrelationMap = accountCorrelationList.stream().collect(
                 Collectors.toMap(AccountCorrelation::getRelGhId, AccountCorrelation::getCorrelation));
+        List<TitleHisCacheParam> paramList = sourceIdList.stream().map(sourceId -> {
+            Content content = sourceIdToContentMap.get(sourceId);
+            TitleHisCacheParam cacheParam = new TitleHisCacheParam();
+            cacheParam.setSourceId(sourceId);
+            cacheParam.setTitleMd5(content.getTitleMd5());
+            cacheParam.setTitle(content.getTitle());
+            cacheParam.setCrawlerTitle(content.getCrawlerTitle());
+            cacheParam.setCrawlerChannelContentId(content.getCrawlerChannelContentId());
+            cacheParam.setCategory(content.getCategory());
+            return cacheParam;
+        }).collect(Collectors.toList());
+        Map<String, Content> articlesWithHistory = getArticleWithHistory(paramList, type);
+        List<Content> newCacheSaveList = new ArrayList<>();
+        Set<String> newCacheSourceIdSet = new HashSet<>();
+        for (Content content : contentList) {
+            if (articleTitleHisCacheMap.containsKey(content.getSourceId())) {
+                ArticleTitleHisCache cache = articleTitleHisCacheMap.get(content.getSourceId());
+                List<ContentHisPublishArticle> hisPublishArticleList =
+                        JSONArray.parseArray(cache.getHisPublishArticleList(), ContentHisPublishArticle.class);
+                for (ContentHisPublishArticle article : hisPublishArticleList) {
+                    article.setCorrelation(Optional.ofNullable(accountCorrelationMap.get(article.getGhId())).orElse(0.0));
+                }
+                if (StringUtils.hasText(cache.getCategory())) {
+                    content.setCategory(JSONArray.parseArray(cache.getCategory(), String.class));
+                }
+                content.setRootPublishTimestamp(cache.getRootPublishTimestamp());
+                content.setHisPublishArticleList(hisPublishArticleList);
+                setT0Data(content);
+                continue;
+            }
+            if (articlesWithHistory.containsKey(content.getSourceId())) {
+                Content articleWithHistory = articlesWithHistory.get(content.getSourceId());
+                content.setHisPublishArticleList(articleWithHistory.getHisPublishArticleList());
+                if (CollectionUtils.isNotEmpty(articleWithHistory.getCategory())) {
+                    content.setCategory(articleWithHistory.getCategory());
+                }
+                content.setRootPublishTimestamp(articleWithHistory.getRootPublishTimestamp());
+                for (ContentHisPublishArticle article : content.getHisPublishArticleList()) {
+                    article.setCorrelation(Optional.ofNullable(accountCorrelationMap.get(article.getGhId())).orElse(0.0));
+                }
+                setT0Data(content);
+                if (!newCacheSourceIdSet.contains(content.getSourceId())) {
+                    newCacheSaveList.add(content);
+                    newCacheSourceIdSet.add(content.getSourceId());
+                }
+            }
+        }
+        // 写入缓存
+        saveArticleTitleHisCache(newCacheSaveList, type);
+        log.info("setTitleAvgViewCount cost:{}", System.currentTimeMillis() - start);
+    }
+
+    private void saveArticleTitleHisCache(List<Content> saveList, String type) {
+        if (CollectionUtils.isEmpty(saveList)) {
+            return;
+        }
+        List<ArticleTitleHisCache> cacheList = new ArrayList<>();
+        try {
+            for (Content content : saveList) {
+                if (CollectionUtils.isEmpty(content.getHisPublishArticleList())) {
+                    continue;
+                }
+                ArticleTitleHisCache cache = new ArticleTitleHisCache();
+                BeanUtils.copyProperties(content, cache);
+                cache.setType(type);
+                cache.setChannelContentId(content.getCrawlerChannelContentId());
+                if (CollectionUtils.isNotEmpty(content.getCategory())) {
+                    cache.setCategory(JSONObject.toJSONString(content.getCategory()));
+                }
+                cache.setRootPublishTimestamp(content.getRootPublishTimestamp());
+                cache.setHisPublishArticleList(JSONObject.toJSONString(content.getHisPublishArticleList()));
+                cache.setCreateTimestamp(System.currentTimeMillis());
+                cacheList.add(cache);
+            }
+            if (CollectionUtils.isEmpty(cacheList)) {
+                return;
+            }
+            longArticleBaseMapper.batchInsertArticleTitleHisCache(cacheList);
+        } catch (Exception e) {
+            log.error("saveArticleTitleHisCache error:{}", e.getMessage());
+        }
+    }
 
-        Set<String> titleMd5List = contentList.stream().map(o -> Md5Util.encoderByMd5(o.getTitle())).collect(Collectors.toSet());
-//        Set<String> titleList = contentList.stream().map(Content::getTitle).collect(Collectors.toSet());
-//        Set<String> crawlerTitleList = contentList.stream().map(Content::getCrawlerTitle).collect(Collectors.toSet());
-//        titleList.addAll(crawlerTitleList);
+    public Map<String, Content> getArticleWithHistory(List<TitleHisCacheParam> paramList, String type) {
+        Map<String, Content> result = new HashMap<>();
+        List<String> titleMd5List = paramList.stream().map(TitleHisCacheParam::getTitleMd5).collect(Collectors.toList());
         // 获取历史已发布文章
         List<Article> hisArticleList = new ArrayList<>();
         List<List<String>> titleMd5Partition = Lists.partition(new ArrayList<>(titleMd5List), 1000);
         for (List<String> titleMd5s : titleMd5Partition) {
-            hisArticleList.addAll(articleRepository.getByTitleMd5InAndTypeEqualsAndStatusEquals(titleMd5s, type, 1));
+            hisArticleList.addAll(crawlerBaseMapper.getByTitleMd5InAndTypeEqualsAndStatusEquals(titleMd5s, type, 1));
+        }
+        if (CollectionUtils.isEmpty(hisArticleList)) {
+            return result;
         }
         Map<String, Map<Integer, List<Article>>> map = hisArticleList.stream()
                 .collect(Collectors.groupingBy(Article::getTitle, Collectors.groupingBy(Article::getItemIndex)));
@@ -273,19 +365,19 @@ public class RecallService implements ApplicationContextAware {
         List<ArticleDetailInfo> articleDetailInfoList = new ArrayList<>();
         List<List<String>> snPartition = Lists.partition(new ArrayList<>(snList), 1000);
         for (List<String> sns : snPartition) {
-            articleDetailInfoList.addAll(articleDetailInfoRepository.getAllByWxSnIn(sns));
+            articleDetailInfoList.addAll(crawlerBaseMapper.getAllByWxSnIn(sns));
         }
         Map<String, List<ArticleDetailInfo>> articleDetailInfoMap = articleDetailInfoList.stream()
                 .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
         // 获取历史已发布文章所属头条内容
         Set<String> ghIds = hisArticleList.stream().map(Article::getGhId).collect(Collectors.toSet());
         Set<String> appMsgIds = hisArticleList.stream().map(Article::getAppMsgId).collect(Collectors.toSet());
-        List<Article> firstIndexHisArticleList = articleRepository.getByGhIdInAndAppMsgIdInAndItemIndexAndTypeEqualsAndStatusEquals(
+        List<Article> firstIndexHisArticleList = crawlerBaseMapper.getByGhIdInAndAppMsgIdInAndItemIndexAndTypeEqualsAndStatusEquals(
                 ghIds, appMsgIds, 1, type, 1);
         Map<String, Map<String, Article>> firstIndexHisArticleMap = firstIndexHisArticleList.stream()
                 .collect(Collectors.groupingBy(Article::getGhId, Collectors.toMap(Article::getAppMsgId, o -> o)));
         // 获取发布账号 位置历史均值
-        List<AccountAvgInfo> accountAvgInfoList = accountAvgInfoRepository.getAllByGhIdIn(ghIds);
+        List<AccountAvgInfo> accountAvgInfoList = crawlerBaseMapper.getAllByGhIdIn(ghIds);
         Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap = accountAvgInfoList.stream()
                 .filter(o -> Objects.nonNull(o.getReadAvg()) && o.getReadAvg() > 0 && o.getFans() > 1000)
                 .collect(Collectors.groupingBy(AccountAvgInfo::getGhId, Collectors.groupingBy(AccountAvgInfo::getUpdateTime,
@@ -297,13 +389,62 @@ public class RecallService implements ApplicationContextAware {
         List<AccountCategory> accountCategoryList = accountCategoryRepository.getByStatus(StatusEnum.ONE.getCode());
         Map<String, JSONObject> accountCategoryMap = accountCategoryList.stream().filter(o -> StringUtils.hasText(o.getCategoryMap()))
                 .collect(Collectors.toMap(AccountCategory::getGhId, o -> JSONObject.parseObject(o.getCategoryMap())));
-        for (Content content : contentList) {
+
+        // 获取品类
+        List<String> channelContentIds = paramList.stream().map(TitleHisCacheParam::getCrawlerChannelContentId)
+                .filter(StringUtils::hasText).collect(Collectors.toList());
+        // 查询晋升rootProduceContentId
+        Map<String, ArticlePoolPromotionSource> sourceMap = new HashMap<>();
+        Map<String, PublishContent> publishContentMap = new HashMap<>();
+        if (CollectionUtils.isNotEmpty(channelContentIds)) {
+            List<ArticlePoolPromotionSource> sourceList = articlePoolPromotionSourceRepository
+                    .getByChannelContentIdInAndStatusAndDeleted(channelContentIds,
+                            ArticlePoolPromotionSourceStatusEnum.FINISH.getCode(), 0);
+            sourceMap = sourceList.stream()
+                    .collect(Collectors.toMap(ArticlePoolPromotionSource::getChannelContentId, Function.identity()));
+            List<String> publishContentIds = sourceList.stream().
+                    map(ArticlePoolPromotionSource::getRootPublishContentId).collect(Collectors.toList());
+            List<PublishContent> publishContentList = publishContentRepository.getByIdIn(publishContentIds);
+            publishContentMap = publishContentList.stream()
+                    .collect(Collectors.toMap(PublishContent::getId, Function.identity()));
+        }
+        // 根据produceContentId查询category
+        List<ArticleCategory> articleCategoryList = articleCategoryRepository.getByStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
+        Map<String, ArticleCategory> categoryMap = articleCategoryList.stream()
+                .collect(Collectors.toMap(ArticleCategory::getProduceContentId, Function.identity()));
+        Map<String, ArticleCategory> coldStartCategoryMap = articleCategoryList.stream()
+                .collect(Collectors.toMap(ArticleCategory::getChannelContentId, Function.identity(), (a, b) -> a));
+        Map<String, ArticleCategory> titleCategoryMap = articleCategoryList.stream()
+                .collect(Collectors.toMap(ArticleCategory::getTitleMd5, Function.identity(), (a, b) -> a));
+
+        for (TitleHisCacheParam cacheParam : paramList) {
+            Content res = new Content();
+            // 设置品类
+            ArticleCategory category = categoryMap.get(cacheParam.getSourceId());
+            if (Objects.isNull(category)) {
+                category = coldStartCategoryMap.get(cacheParam.getCrawlerChannelContentId());
+            }
+            if (Objects.isNull(category)) {
+                category = titleCategoryMap.get(cacheParam.getTitleMd5());
+            }
+            if (Objects.nonNull(category)) {
+                res.setCategory(Collections.singletonList(category.getCategory()));
+            }
+            // 溯源查找源发布时间
+            ArticlePoolPromotionSource source = sourceMap.get(cacheParam.getCrawlerChannelContentId());
+            if (Objects.nonNull(source) && Objects.nonNull(source.getRootProduceContentId())) {
+                PublishContent publishContent = publishContentMap.get(source.getRootPublishContentId());
+                if (Objects.nonNull(publishContent)) {
+                    res.setRootPublishTimestamp(publishContent.getPublishTimestamp());
+                }
+            }
+            // 设置历史表现
             List<Article> hisArticles = new ArrayList<>();
-            Map<Integer, List<Article>> indexArticleMap = map.get(content.getTitle());
+            Map<Integer, List<Article>> indexArticleMap = map.get(cacheParam.getTitle());
             if (Objects.isNull(indexArticleMap)) {
-                indexArticleMap = map.get(content.getCrawlerTitle());
-            } else if (!content.getTitle().equals(content.getCrawlerTitle())) {
-                Map<Integer, List<Article>> crawlerTitleIndexArticleMap = map.get(content.getCrawlerTitle());
+                indexArticleMap = map.get(cacheParam.getCrawlerTitle());
+            } else if (!cacheParam.getTitle().equals(cacheParam.getCrawlerTitle())) {
+                Map<Integer, List<Article>> crawlerTitleIndexArticleMap = map.get(cacheParam.getCrawlerTitle());
                 if (Objects.nonNull(crawlerTitleIndexArticleMap)) {
                     for (Map.Entry<Integer, List<Article>> entry : crawlerTitleIndexArticleMap.entrySet()) {
                         if (indexArticleMap.containsKey(entry.getKey())) {
@@ -322,17 +463,17 @@ public class RecallService implements ApplicationContextAware {
                     hisArticles.addAll(indexArticleList);
                 }
             }
-            content.setHisPublishArticleList(new ArrayList<>());
+            res.setHisPublishArticleList(new ArrayList<>());
             for (Article hisArticle : hisArticles) {
                 if (ScoreStrategy.hisContentLateFilter(hisArticle.getPublishTimestamp())) {
                     continue;
                 }
                 // 历史表现 文章品类如果与历史发布账号负相关 则过滤，不计算该历史发布表现
                 JSONObject categoryWeightMap = accountCategoryMap.get(hisArticle.getGhId());
-                if (Objects.nonNull(categoryWeightMap) && CollectionUtils.isNotEmpty(content.getCategory())) {
-                    String category = content.getCategory().get(0);
-                    if (categoryWeightMap.containsKey(category)) {
-                        double weight = categoryWeightMap.getDoubleValue(category);
+                if (Objects.nonNull(categoryWeightMap) && CollectionUtils.isNotEmpty(cacheParam.getCategory())) {
+                    String hisCategory = cacheParam.getCategory().get(0);
+                    if (categoryWeightMap.containsKey(hisCategory)) {
+                        double weight = categoryWeightMap.getDoubleValue(hisCategory);
                         if (weight < 0) {
                             continue;
                         }
@@ -357,12 +498,15 @@ public class RecallService implements ApplicationContextAware {
                                 .orElse(0.0).intValue();
                     } else {
                         if (ArticleTypeEnum.QUNFA.getVal().equals(type)) {
-                            FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
-                                    "历史表现阅读均值获取失败\n"
-                                            + "ghId: " + hisArticle.getGhId() + "\n"
-                                            + "账号名称: " + hisArticle.getAccountName() + "\n"
-                                            + "日期: " + hisPublishDate + "\n"
-                                            + "位置: " + hisArticle.getItemIndex());
+                            log.error("历史表现阅读均值获取失败 ghId:{} accountName:{} date:{} index:{}",
+                                    hisArticle.getGhId(), hisArticle.getAccountName(), hisPublishDate,
+                                    hisArticle.getItemIndex());
+//                            FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
+//                                    "历史表现阅读均值获取失败\n"
+//                                            + "ghId: " + hisArticle.getGhId() + "\n"
+//                                            + "账号名称: " + hisArticle.getAccountName() + "\n"
+//                                            + "日期: " + hisPublishDate + "\n"
+//                                            + "位置: " + hisArticle.getItemIndex());
                         }
                     }
                 }
@@ -389,13 +533,12 @@ public class RecallService implements ApplicationContextAware {
                         article.setFirstViewCountRate((firstArticle.getShowViewCount() * 1.0) / firstIndexAvgInfo.getReadAvg());
                     }
                 }
-                article.setCorrelation(Optional.ofNullable(accountCorrelationMap.get(article.getGhId())).orElse(0.0));
-                content.getHisPublishArticleList().add(article);
+                res.getHisPublishArticleList().add(article);
             }
             // 设置头条阅读均值
-            setT0Data(content);
+            result.put(cacheParam.getSourceId(), res);
         }
-        log.info("setTitleAvgViewCount cost:{}", System.currentTimeMillis() - start);
+        return result;
     }
 
     private void setT0Data(Content content) {
@@ -417,19 +560,23 @@ public class RecallService implements ApplicationContextAware {
             if (CollectionUtils.isEmpty(article.getArticleDetailInfoList())) {
                 // 仅判断7.12以后发布文章
                 if (article.getPublishTimestamp() > 1720713600 && contentHisFeishuEnable) {
-                    FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
-                            "历史表现裂变特征获取失败\n"
-                                    + "ghId: " + article.getGhId() + "\n"
-                                    + "账号名称: " + article.getAccountName() + "\n"
-                                    + "位置: " + article.getItemIndex() + "\n"
-                                    + "标题: " + article.getTitle() + "\n"
-                                    + "发布时间: " + DateUtils.timestampToYMDStr(article.getPublishTimestamp(), "yyyyMMdd") + "\n"
-                                    + "wxsn: " + article.getWxSn());
+                    log.error("历史表现裂变特征获取失败 ghId:{} accountName:{} itemIndex:{} title:{} date:{} wxsn:{}",
+                            article.getGhId(), article.getAccountName(), article.getItemIndex(), article.getTitle(),
+                            DateUtils.timestampToYMDStr(article.getPublishTimestamp(), "yyyyMMdd"), article.getWxSn());
+//                    FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
+//                            "历史表现裂变特征获取失败\n"
+//                                    + "ghId: " + article.getGhId() + "\n"
+//                                    + "账号名称: " + article.getAccountName() + "\n"
+//                                    + "位置: " + article.getItemIndex() + "\n"
+//                                    + "标题: " + article.getTitle() + "\n"
+//                                    + "发布时间: " + DateUtils.timestampToYMDStr(article.getPublishTimestamp(), "yyyyMMdd") + "\n"
+//                                    + "wxsn: " + article.getWxSn());
                 }
                 continue;
             }
             int sumFission0 = 0;
-            Date minDate = article.getArticleDetailInfoList().stream().map(ArticleDetailInfo::getRecallDt).min(Date::compareTo).orElse(new Date());
+            Date minDate = article.getArticleDetailInfoList().stream().map(ArticleDetailInfo::getRecallDt)
+                    .min(Date::compareTo).orElse(new Date());
             for (ArticleDetailInfo articleDetailInfo : article.getArticleDetailInfoList()) {
                 if (articleDetailInfo.getRecallDt().equals(minDate) && Objects.nonNull(articleDetailInfo.getFission0())) {
                     sumFission0 += articleDetailInfo.getFission0();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/strategy/DefaultRecallStrategy.java

@@ -1,5 +1,6 @@
 package com.tzld.longarticle.recommend.server.service.recommend.recall.strategy;
 
+import com.tzld.longarticle.recommend.server.common.CostMonitor;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterResult;
@@ -24,15 +25,20 @@ public class DefaultRecallStrategy implements RecallStrategy {
 
     @Override
     public RecallResult.RecallData recall(RecallParam param) {
+        long t1 = System.currentTimeMillis();
         List<Content> content = param.getContent();
         // 处理 content
         FilterParam filterParam = FilterParamFactory.create(param, content);
         FilterResult filterResult = filterService.filter(filterParam);
+        long t2 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "ProcessFilter", t2 - t1);
         // 处理 content
         RecallResult.RecallData result = new RecallResult.RecallData();
         result.setContents(content.stream().filter(o -> filterResult.getContentIds().contains(o.getId()))
                 .collect(Collectors.toList()));
         result.setFilterContents(filterResult.getFilterContent());
+        long t3 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "SetResults", t3 - t2);
         return result;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/ScoreParam.java

@@ -21,4 +21,5 @@ public class ScoreParam {
     private List<Content> contents;
     private String strategy;
     private String scene;
+    private Boolean replaceSimilarityAccount = false;
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/ScoreService.java

@@ -59,9 +59,12 @@ public class ScoreService implements ApplicationContextAware {
                 try {
                     List<Score> result = strategy.score(param);
                     return result;
+                } catch (Exception e) {
+                    log.error("scoreService error:{}", e.getMessage(), e);
                 } finally {
                     cdl.countDown();
                 }
+                return new ArrayList<>();
             });
             futures.add(future);
         }
@@ -119,7 +122,8 @@ public class ScoreService implements ApplicationContextAware {
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV11.getStrategy())
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV12.getStrategy())
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV13.getStrategy())
-                || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV14.getStrategy())) {
+                || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV14.getStrategy())
+                || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV15.getStrategy())) {
             strategies.add(strategyMap.get(ScoreStrategyEnum.CATEGORY.value()));
             strategies.add(strategyMap.get(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value()));
             strategies.add(strategyMap.get(ScoreStrategyEnum.FLOW_CTL_DECREASE.value()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/strategy/CategoryStrategy.java

@@ -42,8 +42,14 @@ public class CategoryStrategy implements ScoreStrategy {
         if (CollectionUtils.isEmpty(param.getContents())) {
             return scores;
         }
-        AccountCategory accountCategory = accountCategoryRepository.getByGhIdAndStatus(param.getGhId(), StatusEnum.ONE.getCode());
-        if (Objects.isNull(accountCategory) || !StringUtils.hasText(accountCategory.getCategoryMap())) {
+        List<AccountCategory> accountCategoryList = accountCategoryRepository.getByGhIdAndStatus(param.getGhId(), StatusEnum.ONE.getCode());
+        if (CollectionUtils.isEmpty(accountCategoryList)) {
+            return scores;
+        }
+        AccountCategory accountCategory = accountCategoryList.stream()
+                .sorted(Comparator.comparing(AccountCategory::getDt, Comparator.reverseOrder()))
+                .findFirst().get();
+        if (!StringUtils.hasText(accountCategory.getCategoryMap())) {
             return scores;
         }
         JSONObject categoryWeightMap = JSONObject.parseObject(accountCategory.getCategoryMap());

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/strategy/FlowCtlDecreaseStrategy.java

@@ -12,9 +12,9 @@ import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreParam;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreStrategy;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -95,10 +95,13 @@ public class FlowCtlDecreaseStrategy implements ScoreStrategy {
                 sumViewCount = publishContents.stream().filter(o -> !filterCalGhId.contains(o.getGhId()))
                         .mapToDouble(PublishSortLog::getIndexAvgCount).sum();
             }
-            int hisViewCountSum = content.getHisPublishArticleList().stream()
-                    .filter(ContentHisPublishArticle::isInnerAccount)
-                    .filter(o -> !filterCalGhId.contains(o.getGhId()))
-                    .mapToInt(ContentHisPublishArticle::getAvgViewCount).sum();
+            int hisViewCountSum = 0;
+            if (CollectionUtils.isNotEmpty(content.getHisPublishArticleList())) {
+                hisViewCountSum = content.getHisPublishArticleList().stream()
+                        .filter(ContentHisPublishArticle::isInnerAccount)
+                        .filter(o -> !filterCalGhId.contains(o.getGhId()))
+                        .mapToInt(ContentHisPublishArticle::getAvgViewCount).sum();
+            }
             if ((sumViewCount + hisViewCountSum) > indexTotalAvgReadCountBase.get(String.valueOf(index))) {
                 return indexWeight.get(String.valueOf(index));
             }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/strategy/SimilarityStrategy.java

@@ -1,18 +1,27 @@
 package com.tzld.longarticle.recommend.server.service.recommend.score.strategy;
 
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.model.dto.kimi.KimiResult;
+import com.tzld.longarticle.recommend.server.remote.KimiApiService;
 import com.tzld.longarticle.recommend.server.remote.NLPRemoteService;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.score.Score;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreParam;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreStrategy;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
 
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author dyp
@@ -23,6 +32,23 @@ public class SimilarityStrategy implements ScoreStrategy {
 
     @Autowired
     private NLPRemoteService nlpRemoteService;
+    @Autowired
+    private ArticleRepository articleRepository;
+    @Autowired
+    private KimiApiService kimiApiService;
+
+    @Autowired
+    private RedisTemplate<String, String> redisTemplate;
+
+    @ApolloJsonValue("${account.score.sim.replace:{}}")
+    private Map<String, String> accountSimScoreReplaceMap;
+    @Value("${kimiSimilarityTypePrompt:}")
+    private String kimiSimilarityTypePrompt;
+    @ApolloJsonValue("${similarityTypeAccountMap:}")
+    private Map<String, List<String>> similarityTypeAccountMap;
+
+    @Value("${spring.profiles.active:prod}")
+    private String env;
 
     @Override
     public List<Score> score(ScoreParam param) {
@@ -31,20 +57,49 @@ public class SimilarityStrategy implements ScoreStrategy {
         if (CollectionUtils.isEmpty(param.getContents())) {
             return Collections.emptyList();
         }
-        Map<String, Double> scoreMap = nlpRemoteService.score(param.getGhId(), param.getAccountName(), param.getContents());
-
-//        double min = scoreMap.values().stream()
-//                .min(Double::compareTo)
-//                .orElse(0.0);
-//        double max = scoreMap.values().stream()
-//                .max(Double::compareTo)
-//                .orElse(0.0);
+        String ghId = param.getGhId();
+        if (param.getReplaceSimilarityAccount()) {
+            if (accountSimScoreReplaceMap.containsKey(ghId)) {
+                ghId = accountSimScoreReplaceMap.get(ghId);
+            } else {
+                String redisKey = "AccountSimType_" + param.getAccountName();
+                String type = redisTemplate.opsForValue().get(redisKey);
+                if (!StringUtils.hasText(type)) {
+                    String prompt = kimiSimilarityTypePrompt.replace("accountName", param.getAccountName());
+                    // 调用kimi判断账号类型
+                    KimiResult kimiResult = kimiApiService.requestOfficialApi(prompt, null, null);
+                    if (kimiResult.isSuccess()) {
+                        try {
+                            type = kimiResult.getResponse().getChoices().get(0).getMessage().getContent();
+                            log.info("SimilarityStrategy kimiResultType:{}", type);
+                            redisTemplate.opsForValue().set(redisKey, type, 10L, TimeUnit.DAYS);
+                        } catch (Exception e) {
+                            log.error(kimiResult.getResponse().getChoices().get(0).getMessage().getContent());
+                        }
+                    }
+                }
+                // 根据类型随机再类型池中选取账号
+                List<String> ghIds = similarityTypeAccountMap.get(type);
+                if (CollectionUtils.isNotEmpty(ghIds)) {
+                    ghId = ghIds.get(RandomUtils.nextInt(0, ghIds.size()));
+//                    try {
+//                        Map<String, String> value = accountSimScoreReplaceMap;
+//                        value.put(param.getGhId(), ghId);
+//                        ApolloConfigModifier.modifyConfig("account.score.sim.replace", JSONObject.toJSONString(value), env.toUpperCase());
+//                        log.info("SimilarityStrategy 更新apollo配置 account.score.sim.replace 成功 {}：{}", param.getGhId(), ghId);
+//                    } catch (Exception e) {
+//                        log.error("SimilarityStrategy 更新apollo配置失败", e);
+//                    }
+                }
+                log.info("SimilarityStrategy 排序账号：{} 替换账号：{}", param.getGhId(), ghId);
+            }
+        }
+        Map<String, Double> scoreMap = nlpRemoteService.score(ghId, param.getAccountName(), param.getContents());
 
         List<Score> scores = CommonCollectionUtils.toList(param.getContents(), c -> {
             Score score = new Score();
             score.setContentId(c.getId());
             double val = scoreMap.get(c.getId()) == null ? 0.0 : scoreMap.get(c.getId());
-            // score.setScore(NormalizationUtils.minMax(val, min, max));
             score.setScore(val);
             score.setStrategy(this);
             return score;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/strategy/ViewCountRateCorrelationStrategy.java

@@ -11,6 +11,7 @@ import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreParam;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreStrategy;
 import com.tzld.longarticle.recommend.server.util.MathUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -67,6 +68,9 @@ public class ViewCountRateCorrelationStrategy implements ScoreStrategy {
                 double showViewCountSumSecond = 0D;
                 double avgViewCountSumSecond = 0D;
                 double maxAvgViewCount = 0D;
+                if (CollectionUtils.isEmpty(content.getHisPublishArticleList())) {
+                    continue;
+                }
                 for (ContentHisPublishArticle hisItem : content.getHisPublishArticleList()) {
                     // 过滤掉发布时间晚于19点数据
                     if (ScoreStrategy.hisContentLateFilter(hisItem.getPublishTimestamp())) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/strategy/ViewCountRateStrategy.java

@@ -11,6 +11,7 @@ import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreParam;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreStrategy;
 import com.tzld.longarticle.recommend.server.util.MathUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -67,6 +68,9 @@ public class ViewCountRateStrategy implements ScoreStrategy {
                 double showViewCountSumSecond = 0D;
                 double avgViewCountSumSecond = 0D;
                 double maxAvgViewCount = 0D;
+                if (CollectionUtils.isEmpty(content.getHisPublishArticleList())) {
+                    continue;
+                }
                 for (ContentHisPublishArticle hisItem : content.getHisPublishArticleList()) {
                     // 过滤掉发布时间晚于19点数据
                     if (ScoreStrategy.hisContentLateFilter(hisItem.getPublishTimestamp())) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/util/DateUtils.java

@@ -148,7 +148,7 @@ public final class DateUtils {
         // 存储所有日期的列表
         List<String> datesList = new ArrayList<>();
         // 从startDate到today遍历日期
-        while (!startDate.isAfter(today) && !startDate.isAfter(end)) {
+        while (startDate.isBefore(today) && !startDate.isAfter(end)) {
             // 将当前日期格式化为"yyyyMMdd"并添加到列表中
             datesList.add(startDate.format(formatter));
             // 日期加1天

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/util/TitleSimilarCheckUtil.java

@@ -9,14 +9,19 @@ public class TitleSimilarCheckUtil {
     public static final double SIMILARITY_THRESHOLD = 0.8;
     public static final double ARTICLE_PROMOTION_THRESHOLD = 0.75;
 
+    public static Set<Character> makeCache(String title) {
+        title = title.trim().replace("\u200b", "");
+        Set<Character> cacheSet = new HashSet<>(title.length());
+        for (char c : title.toCharArray()) {
+            cacheSet.add(c);
+        }
+        return cacheSet;
+    }
+
     public static List<Set<Character>> makeCache(List<String> titles) {
         List<Set<Character>> cache = new ArrayList<>(titles.size());
         for (String title : titles) {
-            Set<Character> currentSet = new HashSet<>(title.length());
-            for (char c : title.toCharArray()) {
-                currentSet.add(c);
-            }
-            cache.add(currentSet);
+            cache.add(makeCache(title));
         }
         return cache;
     }
@@ -25,9 +30,9 @@ public class TitleSimilarCheckUtil {
         if (CollectionUtils.isEmpty(existsContentCache)) {
             return false;
         }
-        title = title.trim().replace("\u200b", "");
+        Set<Character> titleCache = makeCache(title);
         for (Set<Character> existTitleCache : existsContentCache) {
-            if (isSimilar(title, existTitleCache, threshold)) {
+            if (isSimilar(titleCache, existTitleCache, threshold)) {
                 return true;
             }
         }
@@ -49,29 +54,27 @@ public class TitleSimilarCheckUtil {
         return result;
     }
 
-    public static boolean isSimilar(String titleA, Set<Character> titleB, double threshold) {
+    public static boolean isSimilar(Set<Character> titleA, Set<Character> titleB, double threshold) {
         if (titleA.isEmpty() || titleB.isEmpty()) {
             return false;
         }
-        Set<Character> setA = new HashSet<>(titleA.length());
-        for (char c : titleA.toCharArray()) {
-            setA.add(c);
-        }
-        int minLen = Math.max(Math.min(setA.size(), titleB.size()), 1);
-        setA.retainAll(titleB);
-        double rate = setA.size() / (double) minLen;
+        int minLen = Math.min(titleA.size(), titleB.size());
+        // since retainAll is an in-place operation, copy it first
+        titleA = new HashSet<>(titleA);
+        titleA.retainAll(titleB);
+        double rate = titleA.size() / (double) minLen;
         return rate >= threshold;
     }
 
+    public static boolean isSimilar(String titleA, Set<Character> titleB, double threshold) {
+        Set<Character> setA = makeCache(titleA);
+        return isSimilar(setA, titleB, threshold);
+    }
+
     public static boolean isSimilar(String titleA, String titleB, double threshold) {
-        if (titleA.isEmpty() || titleB.isEmpty()) {
-            return false;
-        }
-        Set<Character> setB = new HashSet<>(titleB.length());
-        for (char c : titleB.toCharArray()) {
-            setB.add(c);
-        }
-        return isSimilar(titleA, setB, threshold);
+        Set<Character> setA = makeCache(titleA);
+        Set<Character> setB = makeCache(titleB);
+        return isSimilar(setA, setB, threshold);
     }
 
     public static void main(String[] args) {
@@ -86,5 +89,9 @@ public class TitleSimilarCheckUtil {
         List<Set<Character>> titlesCache = makeCache(existsContentTitle);
         result = isDuplicateContentByCache(title, titlesCache, SIMILARITY_THRESHOLD);
         System.out.println(result);
+
+        title = "江苏高考文科女状元，遭多所985名校拒绝录取，成为“最惨状元”";
+        result = isDuplicateContentByCache(title, titlesCache, SIMILARITY_THRESHOLD);
+        System.out.println(result);
     }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/util/feishu/FeishuMessageSender.java

@@ -24,6 +24,8 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.util.EntityUtils;
 import org.apache.http.util.TextUtils;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.util.Pair;
 import org.springframework.stereotype.Component;
 
@@ -32,7 +34,7 @@ import java.util.*;
 
 @Component
 @Slf4j
-public class FeishuMessageSender {
+public class FeishuMessageSender implements InitializingBean {
 
     private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(FeishuMessageSender.class);
 
@@ -40,6 +42,16 @@ public class FeishuMessageSender {
 
     private static final CloseableHttpClient client = HttpPoolFactory.defaultPool();
 
+    private static String staticEnv;
+
+    @Value("${spring.profiles.active:prod}")
+    private String env;
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        FeishuMessageSender.staticEnv = env;
+    }
+
     public void sendChat(String chatId, String title, Map<String, String> infos) {
         sendChat(chatId, null, title, infos);
     }
@@ -73,6 +85,9 @@ public class FeishuMessageSender {
 
     public static void sendWebHookMessage(String robotId, String msg) {
         // 使用自定义群机器人webhook方式，支持外部群
+        if (!"prod".equals(staticEnv)) {
+            return;
+        }
         String webhookUrl = webHookUrl + robotId;
         String hrc = request(msg, webhookUrl);
         if (hrc == null) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/ControllerAspect.java

@@ -20,7 +20,8 @@ import java.util.concurrent.TimeUnit;
 @Slf4j
 public class ControllerAspect {
 
-    @Around("execution(* com.tzld.longarticle.recommend.server.web.*Controller.*(..))")
+    @Around("execution(* com.tzld.longarticle.recommend.server.web..*Controller.*(..)) " +
+            "&& !execution(* com.tzld.longarticle.recommend.server.web.HealthCheckController.ok(..))")
     public Object around(ProceedingJoinPoint pjp) throws Throwable {
         TraceUtils.setMDC();
 
@@ -31,9 +32,13 @@ public class ControllerAspect {
                 JSONUtils.toJson(pjp.getArgs()));
         Object result = pjp.proceed();
         if (result != null && result instanceof String) {
-            log.info("response result=[{}] cost=[{}]", result, stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+            log.info("response className=[{}], method=[{}], param=[{}] result=[{}] cost=[{}]", className,
+                    signature.getName(), JSONUtils.toJson(pjp.getArgs()), result,
+                    stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
         } else {
-            log.info("response result=[{}] cost=[{}]", JSONUtils.toJson(result), stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+            log.info("response className=[{}], method=[{}], param=[{}] result=[{}] cost=[{}]", className,
+                    signature.getName(), JSONUtils.toJson(pjp.getArgs()), JSONUtils.toJson(result),
+                    stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
         }
 
         TraceUtils.removeMDC();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/XxlJobController.java

@@ -24,6 +24,16 @@ public class XxlJobController {
         service.checkPublishPlanAccount(null);
     }
 
+    @GetMapping("/checkColdCrawlerPlan")
+    public void checkColdCrawlerPlan(String dateStr) {
+        service.checkColdCrawlerPlan(dateStr);
+    }
+
+    @GetMapping("/checkColdProducePlanAudit")
+    public void checkColdProducePlanAudit(String dateStr) {
+        service.checkColdProducePlanAudit(dateStr);
+    }
+
     @GetMapping("/migrateCrawlerRootSourceId")
     public void migrateCrawlerRootSourceId(String dateStr) {
         service.migrateCrawlerRootSourceId(dateStr);
@@ -69,5 +79,9 @@ public class XxlJobController {
         service.articleCategoryJobRetry(null);
     }
 
+    @GetMapping("/refreshArticleHisCache")
+    public void refreshArticleHisCache() {
+        service.refreshArticleHisCache(null);
+    }
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/recommend/ArticleAuditController.java

@@ -1,6 +1,7 @@
 package com.tzld.longarticle.recommend.server.web.recommend;
 
 import com.tzld.longarticle.recommend.server.common.response.CommonResponse;
+import com.tzld.longarticle.recommend.server.model.param.ArticleDangerFindDeleteParam;
 import com.tzld.longarticle.recommend.server.model.param.ArticleVideoAuditResultParam;
 import com.tzld.longarticle.recommend.server.service.recommend.ArticleAuditService;
 import lombok.extern.slf4j.Slf4j;
@@ -27,6 +28,12 @@ public class ArticleAuditController {
         return CommonResponse.success();
     }
 
+    @PostMapping("/titleDangerFindDelete")
+    public CommonResponse<Void> titleDangerFindDelete(@RequestBody ArticleDangerFindDeleteParam param) {
+        service.titleDangerFindDelete(param);
+        return CommonResponse.success();
+    }
+
     @GetMapping("/articleVideoDelete")
     public void articleVideoDelete() {
         service.articleVideoDelete(null);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/recommend/ArticleVideoPoolController.java

@@ -0,0 +1,28 @@
+package com.tzld.longarticle.recommend.server.web.recommend;
+
+import com.tzld.longarticle.recommend.server.common.response.CommonResponse;
+import com.tzld.longarticle.recommend.server.model.param.ArticleVideoPoolSourceParam;
+import com.tzld.longarticle.recommend.server.model.vo.ArticleVideoPoolSourceVO;
+import com.tzld.longarticle.recommend.server.service.ArticleVideoPoolService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+@RestController
+@RequestMapping("/videoPool")
+@Slf4j
+public class ArticleVideoPoolController {
+
+    @Autowired
+    private ArticleVideoPoolService service;
+    @PostMapping("/getSource")
+    public CommonResponse<List<ArticleVideoPoolSourceVO>> getSource(@RequestBody ArticleVideoPoolSourceParam param) {
+        return CommonResponse.success(service.getSource(param));
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/DataFlushController.java → long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/recommend/DataFlushController.java

@@ -1,4 +1,4 @@
-package com.tzld.longarticle.recommend.server.web;
+package com.tzld.longarticle.recommend.server.web.recommend;
 
 import com.tzld.longarticle.recommend.server.service.DataFlushService;
 import lombok.extern.slf4j.Slf4j;