Merge branch 'master' into wyp/1209-articleDelete

# Conflicts:
#	long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/FeishuRobotIdEnum.java

long-article-recommend-service/pom.xml

@@ -60,6 +60,11 @@
             <groupId>com.ctrip.framework.apollo</groupId>
             <artifactId>apollo-client</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.ctrip.framework.apollo</groupId>
+            <artifactId>apollo-openapi</artifactId>
+            <version>1.9.1</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/ApolloConfigModifier.java

@@ -0,0 +1,39 @@
+package com.tzld.longarticle.recommend.server.common;
+
+import com.ctrip.framework.apollo.openapi.client.ApolloOpenApiClient;
+import com.ctrip.framework.apollo.openapi.dto.NamespaceReleaseDTO;
+import com.ctrip.framework.apollo.openapi.dto.OpenItemDTO;
+import com.tzld.longarticle.recommend.server.util.DateUtils;
+
+public class ApolloConfigModifier {
+
+    public static void modifyConfig(String key, String value, String env) {
+        // 1. Apollo OpenAPI 客户端配置
+        ApolloOpenApiClient client = ApolloOpenApiClient.newBuilder()
+                .withPortalUrl("http://apolloportal-internal.piaoquantv.com/")
+                .withToken("a3234bae4cd0eab36be8107a13106d4a8c04f895")
+                .build();
+
+        // 2. 定义修改参数
+        String appId = "longarticle-recommend";
+        String clusterName = "default";
+        String namespace = "application";
+
+        // 3. 创建配置项
+        OpenItemDTO item = new OpenItemDTO();
+        item.setKey(key);
+        item.setValue(value);
+        item.setDataChangeCreatedBy("wangyunpeng");
+
+        // 4. 调用 API 更新配置
+        client.createOrUpdateItem(appId, env, clusterName, namespace, item);
+
+        // 5. 发布配置
+        NamespaceReleaseDTO releaseDTO = new NamespaceReleaseDTO();
+        releaseDTO.setReleaseTitle(DateUtils.getCurrentDateStr("yyyyMMddHHmmss") + "-release");
+        releaseDTO.setReleaseComment("Release Comment");
+        releaseDTO.setReleasedBy("wangyunpeng");
+        client.publishNamespace(appId, env, clusterName, namespace, releaseDTO);
+        System.out.println("配置已修改并发布成功！");
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/CostMonitor.java

@@ -0,0 +1,14 @@
+package com.tzld.longarticle.recommend.server.common;
+
+import com.tzld.longarticle.recommend.server.common.constant.LogConstants;
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
+
+@Slf4j
+public class CostMonitor {
+    public static void logCost(String type, String name, long cost) {
+        Marker marker = MarkerFactory.getMarker(LogConstants.MARKER_COST);
+        log.info(marker, "{}.{} cost: {}", type, name, cost);
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/HttpPoolFactory.java

@@ -14,10 +14,10 @@ public final class HttpPoolFactory {
             HttpClientFactory.create(60000, 60000, 200, 200, 0, 60000);
 
     private static CloseableHttpClient NLP =
-            HttpClientFactory.create(15000, 15000, 200, 200, 0, 15000);
+            HttpClientFactory.create(1000, 30000, 200, 200, 0, 5000);
 
     private static CloseableHttpClient ThirtySecond =
-            HttpClientFactory.create(30000, 30000, 200, 200, 0, 30000);
+            HttpClientFactory.create(1000, 60000, 200, 200, 0, 5000);
 
 
     public static CloseableHttpClient defaultPool() {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/RecommendLoghubAppender.java

@@ -13,6 +13,7 @@ import com.aliyun.openservices.aliyun.log.producer.ProducerConfig;
 import com.aliyun.openservices.aliyun.log.producer.ProjectConfig;
 import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException;
 import com.aliyun.openservices.log.common.LogItem;
+import com.tzld.longarticle.recommend.server.common.constant.LogConstants;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
@@ -103,62 +104,105 @@ public class RecommendLoghubAppender<E> extends UnsynchronizedAppenderBase<E> {
         } catch (Exception var3) {
             this.addError("Failed to append event.", var3);
         }
-
     }
 
     private void appendEvent(E eventObject) {
-        if (eventObject instanceof LoggingEvent) {
-            LoggingEvent event = (LoggingEvent) eventObject;
-            List<LogItem> logItems = new ArrayList();
-            LogItem item = new LogItem();
-            logItems.add(item);
-            item.SetTime((int) (event.getTimeStamp() / 1000L));
-            if (this.formatter != null) {
-                DateTime dateTime = new DateTime(event.getTimeStamp());
-                item.PushBack("time", dateTime.toString(this.formatter));
-            } else {
-                Instant instant = Instant.ofEpochMilli(event.getTimeStamp());
-                item.PushBack("time", this.formatter1.format(instant));
-            }
-
-            item.PushBack("level", event.getLevel().toString());
-            item.PushBack("thread", event.getThreadName());
-            StackTraceElement[] caller = event.getCallerData();
-            if (caller != null && caller.length > 0) {
-                item.PushBack("location", caller[0].toString());
-            }
-
-            String message = event.getFormattedMessage();
-            item.PushBack("message", message);
-            IThrowableProxy iThrowableProxy = event.getThrowableProxy();
-            if (iThrowableProxy != null) {
-                String throwable = this.getExceptionInfo(iThrowableProxy);
-                throwable = throwable + this.fullDump(event.getThrowableProxy().getStackTraceElementProxyArray());
-                item.PushBack("throwable", throwable);
-            }
-
-            if (this.encoder != null) {
-                item.PushBack("log", new String(this.encoder.encode(eventObject)));
-            }
-
-            Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
-                event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
-                    return Arrays.stream(f.split(",")).anyMatch((i) -> {
-                        return i.equals(v.getKey());
-                    });
-                }).forEach((map) -> {
-                    item.PushBack((String) map.getKey(), (String) map.getValue());
+        if (!(eventObject instanceof LoggingEvent)) {
+            return;
+        }
+        LoggingEvent event = (LoggingEvent) eventObject;
+        if (event.getMarker() == null) {
+            appendPlainEvent(eventObject);
+        } else if (LogConstants.MARKER_COST.equals(event.getMarker().getName())) {
+            appendCostEvent(event);
+        }
+    }
+
+    private void appendCostEvent(LoggingEvent event) {
+        List<LogItem> logItems = new ArrayList();
+        LogItem item = new LogItem();
+        logItems.add(item);
+        item.SetTime((int) (event.getTimeStamp() / 1000L));
+
+        item.PushBack("thread", event.getThreadName());
+        Object[] args = event.getArgumentArray();
+        try {
+            item.PushBack("type", (String) args[0]);
+            item.PushBack("name", (String) args[1]);
+            item.PushBack("cost", String.valueOf(args[2]));
+        } catch (Exception e) {
+            this.addError("invalid event argument ", e);
+        }
+
+        Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
+            event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
+                return Arrays.stream(f.split(",")).anyMatch((i) -> {
+                    return i.equals(v.getKey());
                 });
+            }).forEach((map) -> {
+                item.PushBack((String) map.getKey(), (String) map.getValue());
             });
+        });
+
+        try {
+            this.producer.send(this.projectConfig.getProject(), this.logStore, "CostMonitor", this.source, logItems,
+                    new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
+                            this.topic, this.source, logItems));
+        } catch (Exception e) {
+            this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, e);
+        }
+    }
+
+    private void appendPlainEvent(E eventObject) {
+        LoggingEvent event = (LoggingEvent) eventObject;
+        List<LogItem> logItems = new ArrayList();
+        LogItem item = new LogItem();
+        logItems.add(item);
+        item.SetTime((int) (event.getTimeStamp() / 1000L));
+        if (this.formatter != null) {
+            DateTime dateTime = new DateTime(event.getTimeStamp());
+            item.PushBack("time", dateTime.toString(this.formatter));
+        } else {
+            Instant instant = Instant.ofEpochMilli(event.getTimeStamp());
+            item.PushBack("time", this.formatter1.format(instant));
+        }
+
+        item.PushBack("level", event.getLevel().toString());
+        item.PushBack("thread", event.getThreadName());
+        StackTraceElement[] caller = event.getCallerData();
+        if (caller != null && caller.length > 0) {
+            item.PushBack("location", caller[0].toString());
+        }
+
+        String message = event.getFormattedMessage();
+        item.PushBack("message", message);
+        IThrowableProxy iThrowableProxy = event.getThrowableProxy();
+        if (iThrowableProxy != null) {
+            String throwable = this.getExceptionInfo(iThrowableProxy);
+            throwable = throwable + this.fullDump(event.getThrowableProxy().getStackTraceElementProxyArray());
+            item.PushBack("throwable", throwable);
+        }
+
+        if (this.encoder != null) {
+            item.PushBack("log", new String(this.encoder.encode(eventObject)));
+        }
 
-            try {
-                this.producer.send(this.projectConfig.getProject(), this.logStore, this.topic, this.source, logItems,
-                        new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
-                                this.topic, this.source, logItems));
-            } catch (Exception var9) {
-                this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, var9);
-            }
+        Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
+            event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
+                return Arrays.stream(f.split(",")).anyMatch((i) -> {
+                    return i.equals(v.getKey());
+                });
+            }).forEach((map) -> {
+                item.PushBack((String) map.getKey(), (String) map.getValue());
+            });
+        });
 
+        try {
+            this.producer.send(this.projectConfig.getProject(), this.logStore, this.topic, this.source, logItems,
+                    new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
+                            this.topic, this.source, logItems));
+        } catch (Exception var9) {
+            this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, var9);
         }
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/constant/LogConstants.java

@@ -0,0 +1,5 @@
+package com.tzld.longarticle.recommend.server.common.constant;
+
+public class LogConstants {
+    public static final String MARKER_COST = "COST";
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/SecretEnum.java

@@ -8,7 +8,8 @@ public enum SecretEnum {
 
     SECRET_ENUM_1("3b83574b477d4c5b8508a6e33f6e35ec", "魅力", "ml"),
     SECRET_ENUM_2("70d342bf11a84ac7aca6b3e99541e085", "老来福", "llf"),
-    SECRET_ENUM_3("595db67618174499b2bed23d8be6a3c1", "微小盟", "wxm");
+    SECRET_ENUM_3("595db67618174499b2bed23d8be6a3c1", "微小盟", "wxm"),
+    SECRET_ENUM_4("9a48498757774ddf8a3878b9c02d0fef", "福州像素", "fzxs");
 
     SecretEnum(String secret, String desc, String channel) {
         this.secret = secret;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/FeishuRobotIdEnum.java

@@ -8,6 +8,7 @@ import java.util.Objects;
 public enum FeishuRobotIdEnum {
     RECOMMEND("长文排序报警群", "07026a9f-43f5-448b-ba40-a8d71bd6e634"),
     JOB("定时任务报警群", "186c9798-5b6a-4ff8-b7fc-4ce4b6ea5076"),
+    DAILY("长文每日更新报警群", "b44333f2-16c0-4cb1-af01-d135f8704410"),
     ARTICLE_DELETE("文章删除报警群", "6e43785b-19f6-4d28-a369-829ba8d7bf5d"),
 
     ;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -33,6 +33,8 @@ public interface AigcBaseMapper {
 
     List<PublishContent> getNearestPublishContent(String publishAccountId, Long publishTimestamp, Integer size);
 
+    List<PublishContent> getLateNearestPublishContent(String publishAccountId, Long publishTimestamp);
+
     CrawlerContent getCrawlerContentByChannelContentId(String channelContentId);
 
     List<CrawlerContent> getCrawlerContentByChannelContentIdIn(List<String> channelContentIds);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/crawler/CrawlerBaseMapper.java

@@ -7,9 +7,11 @@ import com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideoDTO;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountCorrelation;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
+import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleDetailInfo;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesRootSourceId;
 
 import java.util.List;
+import java.util.Set;
 
 public interface CrawlerBaseMapper {
 
@@ -46,4 +48,13 @@ public interface CrawlerBaseMapper {
     List<LongArticlesText> getLongArticlesText();
 
     List<LongArticlesVideoDTO> getLongArticlesVideo(List<String> traceIds);
+
+    List<Article> getByTitleMd5InAndTypeEqualsAndStatusEquals(List<String> titleMd5s, String type, Integer status);
+
+    List<ArticleDetailInfo> getAllByWxSnIn(List<String> wxSnList);
+
+    List<AccountAvgInfo> getAllByGhIdIn(Set<String> ghIdList);
+
+    List<Article> getByGhIdInAndAppMsgIdInAndItemIndexAndTypeEqualsAndStatusEquals(
+            Set<String> ghIdList, Set<String> appMsgIdList, Integer itemIndex, String type, Integer status);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -1,11 +1,11 @@
 package com.tzld.longarticle.recommend.server.mapper.longArticle;
 
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesRootSourceId;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatScore;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticlePoolPromotionSource;
-import com.tzld.longarticle.recommend.server.model.dto.*;
-import com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatSortStrategy;
+import com.tzld.longarticle.recommend.server.model.dto.GetOffVideos;
+import com.tzld.longarticle.recommend.server.model.dto.LongArticlesCrawlerVideos;
+import com.tzld.longarticle.recommend.server.model.dto.LongArticlesMatchVideos;
+import com.tzld.longarticle.recommend.server.model.dto.LongArticlesText;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
+import com.tzld.longarticle.recommend.server.model.param.ArticleVideoPoolSourceParam;
 import org.apache.ibatis.annotations.Mapper;
 
 import java.util.List;
@@ -61,4 +61,6 @@ public interface LongArticleBaseMapper {
     void updateLongArticleMatchVideosResponse(LongArticlesMatchVideos longArticlesMatchVideos);
 
     int countNeedMatchVideos(Long id);
+
+    List<PublishSingleVideoSource> getPublishSingleVideoSource(ArticleVideoPoolSourceParam param);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/crawler/PublishContentSortLog.java

@@ -1,30 +0,0 @@
-package com.tzld.longarticle.recommend.server.model.entity.crawler;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import javax.persistence.*;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-@Entity
-@Table(name = "publish_content_sort_log")
-public class PublishContentSortLog {
-
-    @Id
-    @GeneratedValue(strategy = GenerationType.IDENTITY)
-    private Long id;
-    @Column(name = "gh_id")
-    private String ghId;
-    @Column(name = "account_name")
-    private String accountName;
-    @Column(name = "strategy")
-    private String strategy;
-    @Column(name = "publish_content_id")
-    private String publishContentId;
-    @Column(name = "create_timestamp")
-    private Long createTimestamp;
-
-}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/crawler/PublishSortLog.java

@@ -22,8 +22,12 @@ public class PublishSortLog {
     private String ghId;
     @Column(name = "account_name")
     private String accountName;
+    @Column(name = "publish_content_id")
+    private String publishContentId;
     @Column(name = "crawler_channel_content_id")
     private String crawlerChannelContentId;
+    @Column(name = "source_id")
+    private String sourceId;
     @Column(name = "title")
     private String title;
     @Column(name = "`index`")

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/DatastatScore.java

@@ -74,4 +74,13 @@ public class DatastatScore {
     @Column(name = "first_pub_interval")
     private Integer firstPubInterval;
 
+    @Column(name = "publish_content_id")
+    private String publishContentId;
+
+    @Column(name = "crawler_channel_content_id")
+    private String crawlerChannelContentId;
+
+    @Column(name = "source_id")
+    private String sourceId;
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/PublishSingleVideoSource.java

@@ -0,0 +1,79 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.*;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "publish_single_video_source")
+public class PublishSingleVideoSource {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @Column(name = "content_trace_id")
+    private String contentTraceId;
+
+    @Column(name = "article_title")
+    private String articleTitle;
+
+    @Column(name = "out_account_id")
+    private String outAccountId;
+
+    @Column(name = "out_account_name")
+    private String outAccountName;
+
+    @Column(name = "read_cnt")
+    private Integer readCount;
+
+    @Column(name = "like_cnt")
+    private Integer likeCount;
+
+    @Column(name = "article_index")
+    private Byte articleIndex;
+
+    @Column(name = "article_publish_type")
+    private String articlePublishType;
+
+    @Column(name = "article_url")
+    private String articleUrl;
+
+    @Column(name = "cover_url")
+    private String coverUrl;
+
+    @Column(name = "video_oss_path")
+    private String videoOssPath;
+
+    @Column(name = "flow_pool_level")
+    private Integer flowPoolLevel;
+
+    @Column(name = "bad_status")
+    private Byte badStatus;
+
+    @Column(name = "publish_timestamp")
+    private Long publishTimestamp;
+
+    @Column(name = "crawler_timestamp")
+    private Long crawlerTimestamp;
+
+    @Column(name = "url_unique_md5")
+    private String urlUniqueMd5;
+
+    @Column(name = "up_level_timestamp")
+    private Long upLevelTimestamp;
+
+    @Column(name = "exit_timestamp")
+    private Long exitTimestamp;
+
+    @Column(name = "source_account")
+    private Integer sourceAccount;
+
+    @Column(name = "audit_status")
+    private Integer auditStatus;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/ArticleVideoPoolSourceParam.java

@@ -0,0 +1,8 @@
+package com.tzld.longarticle.recommend.server.model.param;
+
+import lombok.Data;
+
+@Data
+public class ArticleVideoPoolSourceParam {
+    private Integer flowPoolLevel;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/RecommendParam.java

@@ -27,5 +27,6 @@ public class RecommendParam {
     private boolean excludeLog = false;
     private String scene;
     private List<Integer> userGroupIds;
+    private Boolean replaceSimilarityAccount = false;
 }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/ArticleVideoPoolSourceVO.java

@@ -0,0 +1,26 @@
+package com.tzld.longarticle.recommend.server.model.vo;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource;
+import lombok.Data;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Data
+public class ArticleVideoPoolSourceVO {
+    private Integer poolId;
+    private Integer flowPoolLevel;
+    private String sourceId;
+    private String title;
+
+    public static List<ArticleVideoPoolSourceVO> convertFromPublishSingleVideoSource(List<PublishSingleVideoSource> list) {
+        return list.stream().map(publishSingleVideoSource -> {
+            ArticleVideoPoolSourceVO articleVideoPoolSourceVO = new ArticleVideoPoolSourceVO();
+            articleVideoPoolSourceVO.setPoolId(publishSingleVideoSource.getFlowPoolLevel());
+            articleVideoPoolSourceVO.setFlowPoolLevel(publishSingleVideoSource.getFlowPoolLevel());
+            articleVideoPoolSourceVO.setSourceId(publishSingleVideoSource.getContentTraceId());
+            articleVideoPoolSourceVO.setTitle(publishSingleVideoSource.getArticleTitle());
+            return articleVideoPoolSourceVO;
+        }).collect(Collectors.toList());
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/NLPRemoteService.java

@@ -57,7 +57,6 @@ public class NLPRemoteService {
         String url = scoreListUrl;
         JSONObject bodyParam = new JSONObject();
         bodyParam.put("gh_id_list", Collections.singletonList(ghId));
-        bodyParam.put("account_nickname_list", Collections.singletonList(accountName));
         bodyParam.put("text_list", titleList);
         bodyParam.put("interest_type", accountScoreInterestTypeMap.getOrDefault(accountName, "avg"));
         bodyParam.put("sim_type", accountScoreSimTypeMap.getOrDefault(accountName, "mean"));
@@ -69,7 +68,7 @@ public class NLPRemoteService {
             bodyParam.put("min_time", accountScoreMinTimeMap.get(accountName));
         }
         int retry = 0;
-        while (retry < 3) {
+        while (retry < 2) {
             retry++;
             try {
                 HttpPost httpPost = new HttpPost(url);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/aigc/AIGCWaitingPublishContentService.java

@@ -29,7 +29,6 @@ public class AIGCWaitingPublishContentService {
 
 
     public List<Content> getAllContent(RecallParam param) {
-        long start = System.currentTimeMillis();
         List<Content> result = new ArrayList<>();
         JSONObject bodyParam = new JSONObject();
         JSONObject bodyParamParams = new JSONObject();
@@ -58,7 +57,6 @@ public class AIGCWaitingPublishContentService {
         } catch (Exception e) {
             log.error("getAllContent error", e);
         }
-        log.info("getAllContent耗时：{}", System.currentTimeMillis() - start);
         return result;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/PublishContentSortLogRepository.java

@@ -1,10 +0,0 @@
-package com.tzld.longarticle.recommend.server.repository.crawler;
-
-import com.tzld.longarticle.recommend.server.model.entity.crawler.PublishContentSortLog;
-import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.stereotype.Repository;
-
-@Repository
-public interface PublishContentSortLogRepository extends JpaRepository<PublishContentSortLog, Long> {
-
-}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/AccountCategoryRepository.java

@@ -9,7 +9,7 @@ import java.util.List;
 @Repository
 public interface AccountCategoryRepository extends JpaRepository<AccountCategory, AccountCategory.PK> {
 
-    AccountCategory getByGhIdAndStatus(String ghId, Integer status);
+    List<AccountCategory> getByGhIdAndStatus(String ghId, Integer status);
 
     List<AccountCategory> getByStatus(Integer status);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/PublishSingleVideoSourceRepository.java

@@ -0,0 +1,9 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface PublishSingleVideoSourceRepository extends JpaRepository<PublishSingleVideoSource, Long> {
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/ArticleVideoPoolService.java

@@ -0,0 +1,22 @@
+package com.tzld.longarticle.recommend.server.service;
+
+import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource;
+import com.tzld.longarticle.recommend.server.model.param.ArticleVideoPoolSourceParam;
+import com.tzld.longarticle.recommend.server.model.vo.ArticleVideoPoolSourceVO;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service
+public class ArticleVideoPoolService {
+
+    @Autowired
+    private LongArticleBaseMapper longArticleBaseMapper;
+
+    public List<ArticleVideoPoolSourceVO> getSource(ArticleVideoPoolSourceParam param) {
+        List<PublishSingleVideoSource> list = longArticleBaseMapper.getPublishSingleVideoSource(param);
+        return ArticleVideoPoolSourceVO.convertFromPublishSingleVideoSource(list);
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/XxlJobService.java

@@ -161,7 +161,8 @@ public class XxlJobService {
                 accountNames.remove(errorMsgDetail.getAccountName());
             }
         }
-        if (CollectionUtil.isEmpty(planErrorMsg.getErrorMsgDetails())) {
+        if (CollectionUtil.isEmpty(planErrorMsg.getErrorMsgDetails())
+                || CollectionUtil.isEmpty(accountNames)) {
             return;
         }
         FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
@@ -512,7 +513,7 @@ public class XxlJobService {
         articlePromotionService.articlePromotion("【1】", "direct", "头条利用池_v2",
                 "头条利用池_v4", 250, 1.21, Lists.newArrayList(2));
         articlePromotionService.articlePromotion("【2】", "direct", "次条利用池_v1",
-                "头条利用池_v4", 100, 1.33, Lists.newArrayList(3, 4, 5, 6, 7, 8));
+                "次条利用池_v4", 100, 1.33, Lists.newArrayList(3, 4, 5, 6, 7, 8));
         return ReturnT.SUCCESS;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/exterior/impl/GhDetailServiceImpl.java

@@ -124,9 +124,7 @@ public class GhDetailServiceImpl implements GhDetailService {
             }
             ghDetail.setGhId(ghDetailVo.getAccountId());
             ghDetail.setGhName(ghDetailVo.getAccountName());
-            if (!CollectionUtils.isEmpty(ghDetailVo.getVideoIds())) {
-                ghDetail.setVideoIds(JSONObject.toJSONString(ghDetailVo.getVideoIds()));
-            }
+            ghDetail.setVideoIds(JSONObject.toJSONString(ghDetailVo.getVideoIds()));
             ghDetailMapper.updateByPrimaryKeySelective(ghDetail);
             return CommonResponse.success();
         } catch (Exception e) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/exterior/impl/ThirdPartyServiceImpl.java

@@ -86,24 +86,24 @@ public class ThirdPartyServiceImpl implements ThirdPartyService {
         example.createCriteria().andTypeEqualTo(GhTypeEnum.THIRD_PARTY_GH.type).andGhIdEqualTo(param.getGhId());
         List<GhDetail> ghDetails = ghDetailMapper.selectByExample(example);
         if (CollectionUtils.isEmpty(ghDetails)) {
-            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "ghId不存在，请联系管理员配置");
+            return CommonResponse.create(404, "ghId不存在，请联系管理员配置");
         }
         String channel = ghDetails.get(0).getChannel();
         if (channel == null) {
             LarkRobotUtil.sendMessage("channel不存在，请查看详情 ghId=", param.getGhId());
-            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "ghId异常，请联系管理员检查");
+            return CommonResponse.create(404, "ghId异常，请联系管理员检查");
         }
         if (!Objects.equals(secretEnum.channel, channel)) {
             LarkRobotUtil.sendMessage(String.format("channel异常 secretEnum.channel=%s ghDetail.channel=%s ghId=%s",
                     secretEnum.channel, channel, param.getGhId()));
-            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "ghId异常，请联系管理员检查");
+            return CommonResponse.create(404, "ghId异常，请联系管理员检查");
         }
         List<PushMessageVo> pushMessageVoList = new ArrayList<>();
         ReplyBucketData replyBucketData = getPushMessageData(param, channel);
         log.info("replyBucketData={}", JSON.toJSONString(replyBucketData));
         if (replyBucketData == null) {
             log.error("获取推送策略数据失败");
-            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "系统异常，获取失败");
+            return CommonResponse.create(500, "系统异常，获取失败");
         }
         List<GroupData> groupList = replyBucketData.getGroupList();
         for (GroupData groupData : groupList) {
@@ -139,9 +139,13 @@ public class ThirdPartyServiceImpl implements ThirdPartyService {
         if (!DateUtils.isValidDate(canViewReportDate)) {
             return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "系统异常");
         }
-        //llf暂时不返回数据
+        //llf只返回2024-11-30后的数据
         if (secretEnum == SecretEnum.SECRET_ENUM_2) {
-            return CommonResponse.create(500, "数据不存在");
+            long targetTime = DateUtils.dateStrToTimestamp(date, "yyyy-MM-dd");
+            long limitTime = DateUtils.dateStrToTimestamp("2024-11-30", "yyyy-MM-dd");
+            if (targetTime < limitTime) {
+                return CommonResponse.create(500, "数据不存在");
+            }
         }
 
         //10点后可查询前一天数据

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticlePromotionService.java

@@ -1,7 +1,9 @@
 package com.tzld.longarticle.recommend.server.service.recommend;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.CrawlerModeEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticlePoolPromotionSourceStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
@@ -118,6 +120,8 @@ public class ArticlePromotionService {
             aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, urlList, tag, CrawlerModeEnum.ContentIDs.getVal());
             return;
         }
+        List<String> publishContentIds = new ArrayList<>();
+        List<String> filterUrlList = new ArrayList<>();
         try {
             String level = pos.equals("【1】") ? contentPoolType.get(0) : contentPoolType.get(1);
             String produceId = produceConfig.get(accountNickName).get(pos).get(way).trim();
@@ -129,8 +133,6 @@ public class ArticlePromotionService {
             Set<String> visitedUrlIdList = contentList.stream().map(content -> getUrlId(content.getReferContentLink()))
                     .collect(Collectors.toSet());
             // 筛选URL和标题
-            List<String> publishContentIds = new ArrayList<>();
-            List<String> filterUrlList = new ArrayList<>();
             for (DatastatSortStrategy item : list) {
                 String url = item.getLink();
                 String urlId = getUrlId(item.getLink());
@@ -155,32 +157,41 @@ public class ArticlePromotionService {
                     }
                 }
             }
-            if (filterUrlList.isEmpty()) {
-                log.info("url_list empty: " + accountNickName + ", " + pos + ", " + way);
-                return;
-            }
-            String planName = String.format("%d_%s_%s_%s【%s】_%s", filterUrlList.size(), today, accountNickName, pos, way, today);
-            log.info("url_len: " + list.size() + ", " + filterUrlList.size());
-            IdNameVO<String> planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, filterUrlList, tag, CrawlerModeEnum.ContentIDs.getVal());
-            if (StringUtils.hasText(produceId)) {
-                articleAddDependPlan(produceId, planInfo.getId(), planInfo.getName());
+            if (CollectionUtils.isNotEmpty(filterUrlList)) {
+                String planName = String.format("%d_%s_%s_%s【%s】_%s", filterUrlList.size(), today, accountNickName, pos, way, today);
+                log.info("url_len: " + list.size() + ", " + filterUrlList.size());
+                IdNameVO<String> planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, filterUrlList, tag, CrawlerModeEnum.ContentIDs.getVal());
+                if (StringUtils.hasText(produceId)) {
+                    articleAddDependPlan(produceId, planInfo.getId(), planInfo.getName());
+                }
+                log.info("{}, {}, produce plan not exist: {}, {}, {}", planInfo.getName(), planInfo.getId(), accountNickName, pos, way);
             }
-            log.info("{}, {}, produce plan not exist: {}, {}, {}", planInfo.getName(), planInfo.getId(), accountNickName, pos, way);
             if (CollectionUtils.isNotEmpty(publishContentIds)) {
-                planName = String.format("%d_%s_%s_%s【%s】_%s", publishContentIds.size(), today, accountNickName, pos, way, today);
-                planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, publishContentIds, tag, CrawlerModeEnum.PublishContentIds.getVal());
+                String planName = String.format("%d_%s_%s_%s【%s】_%s", publishContentIds.size(), today, accountNickName, pos, way, today);
+                IdNameVO<String> planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, publishContentIds, tag, CrawlerModeEnum.PublishContentIds.getVal());
                 if (StringUtils.hasText(produceId)) {
                     articleAddDependPlan(produceId, planInfo.getId(), planInfo.getName());
                 }
             }
+            sendFeishuJobFinishMessage(accountNickName, filterUrlList.size(), publishContentIds.size());
         } catch (Exception e) {
             log.error("articlePromotion error: ", e);
-            FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.JOB.getRobotId(),
-                    "文章晋升ERROR:\n" +
-                            "articlePromotion error: " + e.getMessage());
+            FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.DAILY.getRobotId(),
+                    "【文章晋升ERROR】\n" +
+                            "晋级任务：" + accountNickName + "\n" +
+                            "articlePromotion error: " + e.getMessage() + "\n"
+                            + "<at user_id=\"all\">所有人</at> ");
         }
     }
 
+    private void sendFeishuJobFinishMessage(String accountNickName, Integer urlListSize, Integer contentListSize) {
+        FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.DAILY.getRobotId(),
+                "【文章晋级job完成】\n" +
+                        "晋级任务：" + accountNickName + "\n" +
+                        "url晋级数量：" + urlListSize + "\n" +
+                        "id晋级数量：" + contentListSize + "\n");
+    }
+
     private List<ProduceContentListItemVO> getProduceContentList(String accountNickName, String pos, String way) {
         List<String> planIdList = getProducePlanIdList(accountNickName, pos, way);
         CommonListDataVO<ProduceContentListItemVO> contentData = getProduceContentListByPlanIdList(planIdList);
@@ -303,6 +314,8 @@ public class ArticlePromotionService {
             articlePromotion.setTitle(title);
             articlePromotion.setTitleMd5(Md5Util.encoderByMd5(title));
             articlePromotion.setCreateTimestamp(System.currentTimeMillis());
+            articlePromotion.setStatus(ArticlePoolPromotionSourceStatusEnum.WAITING.getCode());
+            articlePromotion.setDeleted(StatusEnum.ZERO.getCode());
         }
         articlePromotion.setWxSn(wxSn);
         articlePromotion.setLevel(level);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleService.java

@@ -42,6 +42,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
@@ -88,6 +89,8 @@ public class ArticleService {
             "\"20240805154433785506170\", \"20240805154359027876170\", \"20241024100016206421084\", " +
             "\"20241030070010871546586\"]}")
     private static List<String> producePlanIds;
+    @Value("${kimiCategoryPrompt:}")
+    private String kimiCategoryPrompt;
 
     private final static ExecutorService pool = new CommonThreadPoolExecutor(
             32,
@@ -150,6 +153,13 @@ public class ArticleService {
         if (Objects.nonNull(publishContent)) {
             publishContentId = publishContent.getId();
             channelContentId = publishContent.getCrawlerChannelContentId();
+        } else {
+            publishContentList = aigcBaseMapper.getLateNearestPublishContent(publishAccount.getId(), publishTimestamp);
+            publishContent = findPublishContent(publishContentList, article.getTitle(), publishTimestamp);
+            if (Objects.nonNull(publishContent)) {
+                publishContentId = publishContent.getId();
+                channelContentId = publishContent.getCrawlerChannelContentId();
+            }
         }
         log.info("syncAigcIdByWxSn titleMatch finish");
         if (Objects.isNull(channelContentId)) {
@@ -169,6 +179,9 @@ public class ArticleService {
     private PublishContent findPublishContent(List<PublishContent> publishContentList,
                                               String title,
                                               Long publishTimestamp) {
+        if (CollectionUtils.isEmpty(publishContentList)) {
+            return null;
+        }
         Map<String, PublishContent> publishContentMap = publishContentList.stream().collect(
                 Collectors.toMap(PublishContent::getId, publishContent -> publishContent));
         List<String> publishContentIds = publishContentList.stream().map(PublishContent::getId).collect(Collectors.toList());
@@ -259,6 +272,17 @@ public class ArticleService {
             result.setRootPublishContentId(publishContent.getId());
             result.setRootProduceContentId(publishContent.getSourceId());
             channelContentId = publishContent.getCrawlerChannelContentId();
+        } else {
+            publishContentList = aigcBaseMapper.getLateNearestPublishContent(publishAccount.getId(), publishTimestamp);
+            publishContent = findPublishContent(publishContentList, title, publishTimestamp);
+            if (Objects.nonNull(publishContent)) {
+                if (!StringUtils.hasText(sourcePublishContentId)) {
+                    result.setSourcePublishContentId(publishContent.getId());
+                }
+                result.setRootPublishContentId(publishContent.getId());
+                result.setRootProduceContentId(publishContent.getSourceId());
+                channelContentId = publishContent.getCrawlerChannelContentId();
+            }
         }
         // channelContentId未被修改，说明未找到
         if (channelContentId.equals(crawlerContent.getChannelContentId())) {
@@ -284,64 +308,72 @@ public class ArticleService {
         }
         long now = System.currentTimeMillis();
         for (ArticlePoolPromotionSource task : tasks) {
-            // 判断文章是否被抓回来，如果没有抓回来，不进行处理 临时方案
-            CrawlerContent crawlerContent = aigcBaseMapper.getCrawlerContentByChannelContentId(task.getChannelContentId());
-            if (Objects.isNull(crawlerContent)) {
-                continue;
-            }
-            // 溯源
-            Article article = articleRepository.getByWxSn(task.getWxSn());
-            if (Objects.isNull(article)) {
-                task.setDeleted(StatusEnum.ONE.getCode());
-                articlePoolPromotionSourceRepository.save(task);
-                continue;
-            }
-            PublishAccount publishAccount = publishAccountRepository.getByGhId(article.getGhId());
-            if (Objects.isNull(publishAccount)) {
-                continue;
-            }
-            long publishTimestamp = article.getPublishTimestamp() > 0 ? article.getPublishTimestamp() * 1000 : article.getUpdateTime() * 1000;
-            List<PublishContent> publishContentList = aigcBaseMapper.getNearestPublishContent(publishAccount.getId(), publishTimestamp, 100);
-            PublishContent publishContent = findPublishContent(publishContentList, task.getTitle(), publishTimestamp);
-            if (Objects.isNull(publishContent)) {
-                task.setDeleted(StatusEnum.ONE.getCode());
-                articlePoolPromotionSourceRepository.save(task);
-                continue;
-            }
-            RootPublishContentVO source = getRootPublishContent(publishContent.getCrawlerChannelContentId(), null, publishContent.getId(), publishContent.getSourceId(), 0);
-            // 更新
-            if (StringUtils.hasText(source.getRootProduceContentId())) {
-                task.setStatus(ArticlePoolPromotionSourceStatusEnum.FINISH.getCode());
-                task.setSourcePublishContentId(source.getSourcePublishContentId());
-                task.setRootProduceContentId(source.getRootProduceContentId());
-                task.setRootPublishContentId(source.getRootPublishContentId());
-                task.setUpdateTimestamp(now);
-                articlePoolPromotionSourceRepository.save(task);
-                // 保存中间环节 晋级溯源
-                for (String midChannelContentId : source.getMidChannelContentIds()) {
-                    ArticlePoolPromotionSource item = new ArticlePoolPromotionSource();
-                    BeanUtils.copyProperties(task, item);
-                    ArticlePoolPromotionSource dto = articlePoolPromotionSourceRepository.getByChannelContentId(midChannelContentId);
-                    if (Objects.nonNull(dto)) {
-                        // 以dto为基础
-                        dto.setRootProduceContentId(task.getRootProduceContentId());
-                        dto.setRootPublishContentId(task.getRootPublishContentId());
-                        dto.setStatus(task.getStatus());
-                        dto.setLevel(task.getLevel());
-                        dto.setUpdateTimestamp(task.getUpdateTimestamp());
-                        item = dto;
-                    } else {
-                        // 以新item为基础
-                        item.setChannelContentId(midChannelContentId);
-                        item.setCreateTimestamp(item.getUpdateTimestamp());
+            try {
+                // 判断文章是否被抓回来，如果没有抓回来，不进行处理 临时方案
+                CrawlerContent crawlerContent = aigcBaseMapper.getCrawlerContentByChannelContentId(task.getChannelContentId());
+                if (Objects.isNull(crawlerContent)) {
+                    continue;
+                }
+                // 溯源
+                Article article = articleRepository.getByWxSn(task.getWxSn());
+                if (Objects.isNull(article)) {
+                    task.setDeleted(StatusEnum.ONE.getCode());
+                    articlePoolPromotionSourceRepository.save(task);
+                    continue;
+                }
+                PublishAccount publishAccount = publishAccountRepository.getByGhId(article.getGhId());
+                if (Objects.isNull(publishAccount)) {
+                    continue;
+                }
+                long publishTimestamp = article.getPublishTimestamp() > 0 ? article.getPublishTimestamp() * 1000 : article.getUpdateTime() * 1000;
+                List<PublishContent> publishContentList = aigcBaseMapper.getNearestPublishContent(publishAccount.getId(), publishTimestamp, 100);
+                PublishContent publishContent = findPublishContent(publishContentList, task.getTitle(), publishTimestamp);
+                if (Objects.isNull(publishContent)) {
+                    publishContentList = aigcBaseMapper.getLateNearestPublishContent(publishAccount.getId(), publishTimestamp);
+                    publishContent = findPublishContent(publishContentList, task.getTitle(), publishTimestamp);
+                    if (Objects.isNull(publishContent)) {
+                        task.setDeleted(StatusEnum.ONE.getCode());
+                        articlePoolPromotionSourceRepository.save(task);
+                        continue;
                     }
-                    item.setDeleted(0);
-                    articlePoolPromotionSourceRepository.save(item);
                 }
-                longArticleBaseMapper.updateRootProduceContentLevel(task.getRootProduceContentId(), task.getLevel());
-            } else {
-                task.setDeleted(StatusEnum.ONE.getCode());
-                articlePoolPromotionSourceRepository.save(task);
+                RootPublishContentVO source = getRootPublishContent(publishContent.getCrawlerChannelContentId(), null, publishContent.getId(), publishContent.getSourceId(), 0);
+                // 更新
+                if (StringUtils.hasText(source.getRootProduceContentId())) {
+                    task.setStatus(ArticlePoolPromotionSourceStatusEnum.FINISH.getCode());
+                    task.setSourcePublishContentId(source.getSourcePublishContentId());
+                    task.setRootProduceContentId(source.getRootProduceContentId());
+                    task.setRootPublishContentId(source.getRootPublishContentId());
+                    task.setUpdateTimestamp(now);
+                    articlePoolPromotionSourceRepository.save(task);
+                    // 保存中间环节 晋级溯源
+                    for (String midChannelContentId : source.getMidChannelContentIds()) {
+                        ArticlePoolPromotionSource item = new ArticlePoolPromotionSource();
+                        BeanUtils.copyProperties(task, item);
+                        ArticlePoolPromotionSource dto = articlePoolPromotionSourceRepository.getByChannelContentId(midChannelContentId);
+                        if (Objects.nonNull(dto)) {
+                            // 以dto为基础
+                            dto.setRootProduceContentId(task.getRootProduceContentId());
+                            dto.setRootPublishContentId(task.getRootPublishContentId());
+                            dto.setStatus(task.getStatus());
+                            dto.setLevel(task.getLevel());
+                            dto.setUpdateTimestamp(task.getUpdateTimestamp());
+                            item = dto;
+                        } else {
+                            // 以新item为基础
+                            item.setChannelContentId(midChannelContentId);
+                            item.setCreateTimestamp(item.getUpdateTimestamp());
+                        }
+                        item.setDeleted(StatusEnum.ZERO.getCode());
+                        articlePoolPromotionSourceRepository.save(item);
+                    }
+                    longArticleBaseMapper.updateRootProduceContentLevel(task.getRootProduceContentId(), task.getLevel());
+                } else {
+                    task.setDeleted(StatusEnum.ONE.getCode());
+                    articlePoolPromotionSourceRepository.save(task);
+                }
+            } catch (Exception e) {
+                log.error("articlePromotionTraceability error channelContentId:{}", task.getChannelContentId(), e);
             }
         }
     }
@@ -421,7 +453,7 @@ public class ArticleService {
         List<ProducePlanExeRecord> produceContentList = aigcBaseMapper.getAllByProducePlanId(producePlanIds);
         List<String> channelContentIds = produceContentList.stream().map(ProducePlanExeRecord::getChannelContentId).distinct().collect(Collectors.toList());
         List<ArticleCategory> articleCategoryList = articleCategoryRepository.getAllByChannelContentIdIn(channelContentIds);
-        List<String>  articleCategoryIds = articleCategoryList.stream().map(ArticleCategory::getChannelContentId).collect(Collectors.toList());
+        List<String> articleCategoryIds = articleCategoryList.stream().map(ArticleCategory::getChannelContentId).collect(Collectors.toList());
         List<ProduceContentCrawlerVO> list = produceContentList.stream().filter(o -> !articleCategoryIds.contains(o.getChannelContentId())).map(o -> {
             ProduceContentCrawlerVO item = new ProduceContentCrawlerVO();
             item.setChannelContentId(o.getChannelContentId());
@@ -453,42 +485,8 @@ public class ArticleService {
     }
 
     private String buildKimiPrompt(List<String> titleList) {
-        StringBuilder prompt = new StringBuilder(
-                "请帮我完成以下任务：输入为文章的标题，根据标题判断其内容所属的类目，输出为文章标题及其对应的类目。\n" +
-                "类目需从以下范围内选择：\n" +
-                "奇闻趣事\n" +
-                "历史人物\n" +
-                "家长里短\n" +
-                "温情故事\n" +
-                "健康养生\n" +
-                "生活知识\n" +
-                "名人八卦\n" +
-                "政治新闻\n" +
-                "军事新闻\n" +
-                "影视解读\n" +
-                "为了更好地完成任务，可参考下列对文章标题的分类：\n" +
-                "{" +
-                "\"大舅病了，我取了三万元送过去，病房门口听到舅妈的话我改了主意\": \"家长里短\",\n" +
-                "\"能活到90岁的老人，基本上在70岁的时候，就不再做这些事了！\": \"健康养生\",\n" +
-                "\"去医院看望病人时，切忌带这4样东西，再亲近也不行，这是做人的根本\": \"生活知识\",\n" +
-                "\"上海一女子去饭店吃生煎包，戳了个洞想凉一凉，往里一看，瞬间惊呆了\": \"奇闻趣事\",\n" +
-                "\"卫生间放一把食盐，一年能省下好几百，涨新知识\": \"生活知识\",\n" +
-                "\"中国有一古寺，庙不大，却有武警24小时站岗，到底有何“过人”之处\": \"奇闻趣事\",\n" +
-                "\"1974年，苏联外长故意拿邓小平身高“取笑”，邓小平一句话轻松反击\": \"历史人物\",\n" +
-                "\"中国最美的女将军：上世纪曾家喻户晓，如今仍然健在\": \"历史人物\",\n" +
-                "\"北大才女蒙曼48岁仍未婚，被问最想嫁给谁，一个名字让全场笑喷\": \"名人八卦\",\n" +
-                "\"广东一老人去世，家人把老人的旧床垫扔了，环卫工人看到后，竟发现里面藏了15万元现金！家人傻眼了\": \"奇闻趣事\",\n" +
-                "\"《孤舟》结局：要不是海沫2年不生娃，周知非至死不知，顾易中要3000元的报销费的真实目的！\": \"影视解读\"\n" +
-                "}" +
-                "最后输出结果请用JSON格式输出，key为title，value为类目，仅输出JSON，不要markdown格式，不要任何其他内容，" +
-                "并且内容可以被 fastJSON 的JSONObject.parseObject转换为JSON对象\n" +
-                "当标题的开头或结尾为以下字符时“”“，则在标题的开头或结尾增加\" " +
-                "输出结果格式如下：\n" +
-                "{" +
-                "\"浙江老人用“假钱”吃霸王餐9年，离世后，老板却崩溃大哭：“每天都在等他！”\": \"奇闻趣事\"," +
-                "\"“最美婴儿”迅速走红，像在娘胎里整过容，网友：看到第一眼就想抱回家\": \"奇闻趣事\"" +
-                "}" +
-                "以下是需要分析的文章标题列表，每一行是一个标题：\n");
+        StringBuilder prompt = new StringBuilder(kimiCategoryPrompt);
+        prompt.append("\n");
         for (String title : titleList) {
             prompt.append(title).append("\n");
         }
@@ -538,6 +536,12 @@ public class ArticleService {
         PublishContent publishContent = findPublishContent(publishContentList, article.getTitle(), publishTimestamp);
         if (Objects.nonNull(publishContent)) {
             return publishContent.getId();
+        } else {
+            publishContentList = aigcBaseMapper.getLateNearestPublishContent(publishAccount.getId(), publishTimestamp);
+            publishContent = findPublishContent(publishContentList, article.getTitle(), publishTimestamp);
+            if (Objects.nonNull(publishContent)) {
+                return publishContent.getId();
+            }
         }
         return null;
     }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/DataDashboardService.java

@@ -912,11 +912,9 @@ public class DataDashboardService {
                 .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
         String lessDateStr = DateUtils.getBeforeDayStr(dateStr, "yyyyMMdd", 7);
         List<PublishSortLog> publishSortLogList = publishSortLogRepository.findByDateStrGreaterThanEqual(lessDateStr);
-        Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap = publishSortLogList.stream()
+        Map<String, Map<String, Map<Integer, List<PublishSortLog>>>> publishSortLogMap = publishSortLogList.stream()
                 .collect(Collectors.groupingBy(PublishSortLog::getGhId,
-                        Collectors.groupingBy(PublishSortLog::getDateStr, Collectors.toMap(
-                                PublishSortLog::getIndex, o -> o, (existing, replacement) -> replacement
-                        ))));
+                        Collectors.groupingBy(PublishSortLog::getDateStr, Collectors.groupingBy(PublishSortLog::getIndex))));
         List<NewSortStrategyExport> newSortStrategyExportList = getNewSortStrategyExportList(todayPublish,
                 articleDetailInfoMap, accountAvgInfoIndexMap);
 
@@ -965,10 +963,10 @@ public class DataDashboardService {
             // 发布表现
             setPublishPerformance(item, data, publishSortLogMap);
             // 发布依赖表现
-            setPublishSourcePerformance(item, accountAvgInfoIndexMap, articleDetailInfoMap, publishSortLogMap,
+            setPublishSourcePerformance(item, data, article, accountAvgInfoIndexMap, articleDetailInfoMap, publishSortLogMap,
                     type, scoreHisPublishTimeMap, wxsnHisDistinctSetMap, list, poolLevel, small, hisPublishMap);
             // 发布未来表现
-            setPublishFuturePerformance(item, data, poolLevel, promotionSourceMap, futurePublishMap, publishSortLogMap, small);
+            setPublishFuturePerformance(item, data, poolLevel, promotionSourceMap, futurePublishMap, small);
 
             titles.add(data.getTitle());
             poolTitles.add(data.getTitle());
@@ -978,15 +976,19 @@ public class DataDashboardService {
     }
 
     private boolean checkIsAigcPublish(Article article,
-                                       Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap) {
-        Map<String, Map<Integer, PublishSortLog>> dateSortMap = publishSortLogMap.get(article.getGhId());
+                                       Map<String, Map<String, Map<Integer, List<PublishSortLog>>>> publishSortLogMap) {
+        Map<String, Map<Integer, List<PublishSortLog>>> dateSortMap = publishSortLogMap.get(article.getGhId());
         if (Objects.nonNull(dateSortMap)) {
             for (String dateStr : dateSortMap.keySet()) {
-                Map<Integer, PublishSortLog> indexMap = dateSortMap.get(dateStr);
+                Map<Integer, List<PublishSortLog>> indexMap = dateSortMap.get(dateStr);
                 if (Objects.nonNull(indexMap)) {
-                    PublishSortLog log = indexMap.get(article.getItemIndex());
-                    if (Objects.nonNull(log) && log.getTitle().equals(article.getTitle())) {
-                        return true;
+                    List<PublishSortLog> logs = indexMap.get(article.getItemIndex());
+                    if (CollectionUtil.isNotEmpty(logs)) {
+                        for (PublishSortLog log : logs) {
+                            if (log.getTitle().equals(article.getTitle())) {
+                                return true;
+                            }
+                        }
                     }
                 }
             }
@@ -1013,7 +1015,6 @@ public class DataDashboardService {
     private void setPublishFuturePerformance(IntermediateIndicatorsExport item, NewSortStrategyExport data, Integer poolLevel,
                                              Map<String, ArticlePoolPromotionSource> promotionSourceMap,
                                              Map<String, List<Article>> futurePublishMap,
-                                             Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap,
                                              List<String> small) {
         int futurePoolLevel = getPromotionPoolLevel(data.getGhId(), data.getTitle(), promotionSourceMap, small);
         if (futurePoolLevel > poolLevel) {
@@ -1161,9 +1162,11 @@ public class DataDashboardService {
     }
 
     private void setPublishSourcePerformance(IntermediateIndicatorsExport item,
+                                             NewSortStrategyExport data,
+                                             Article article,
                                              Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap,
                                              Map<String, List<ArticleDetailInfo>> articleDetailInfoMap,
-                                             Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap,
+                                             Map<String, Map<String, Map<Integer, List<PublishSortLog>>>> publishSortLogMap,
                                              String type,
                                              Map<String, Integer> scoreHisPublishTimeMap,
                                              Map<String, Set<String>> wxsnHisDistinctSetMap,
@@ -1235,36 +1238,45 @@ public class DataDashboardService {
                         (item.getActualArticleReleaseCount() - 1) + firstLayerFissionToViewBaseRatio) / item.getActualArticleReleaseCount();
             }
             item.setAverageFirstLayerFissionToViewBaseRatio(averageFirstLayerFissionToViewBaseRatio);
-            Map<String, Map<Integer, PublishSortLog>> dateSortMap = publishSortLogMap.get(publish.getGhId());
+            Map<String, Map<Integer, List<PublishSortLog>>> dateSortMap = publishSortLogMap.get(publish.getGhId());
             if (Objects.nonNull(dateSortMap)) {
                 String publishDateStr = DateUtils.timestampToYMDStr(publish.getPublishTimestamp(), "yyyyMMdd");
-                Map<Integer, PublishSortLog> indexMap = dateSortMap.get(publishDateStr);
+                Map<Integer, List<PublishSortLog>> indexMap = dateSortMap.get(publishDateStr);
                 if (Objects.nonNull(indexMap)) {
-                    PublishSortLog log = indexMap.get(publish.getItemIndex());
-                    double scoreAvg = item.getRedundantAccountArticleRelevanceAvg();
-                    double score = 0.0;
-                    if (Objects.nonNull(log) && Objects.nonNull(log.getScore())) {
-                        JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
-                        if (scoreMap.containsKey("SimilarityStrategy")) {
-                            score = scoreMap.getDoubleValue("SimilarityStrategy");
+                    List<PublishSortLog> logs = indexMap.get(publish.getItemIndex());
+                    if (CollectionUtil.isNotEmpty(logs)) {
+                        for (PublishSortLog log : logs) {
+                            if (log.getTitle().equals(data.getTitle())) {
+                                double scoreAvg = item.getRedundantAccountArticleRelevanceAvg();
+                                double score = 0.0;
+                                if (Objects.nonNull(log.getScore())) {
+                                    JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
+                                    if (scoreMap.containsKey("SimilarityStrategy")) {
+                                        score = scoreMap.getDoubleValue("SimilarityStrategy");
+                                    }
+                                }
+                                scorePublishTime++;
+                                scoreAvg = (scoreAvg * (scorePublishTime - 1) + score) / scorePublishTime;
+                                item.setRedundantAccountArticleRelevanceAvg(scoreAvg);
+                                break;
+                            }
                         }
                     }
-                    scorePublishTime++;
-                    scoreAvg = (scoreAvg * (scorePublishTime - 1) + score) / scorePublishTime;
-                    item.setRedundantAccountArticleRelevanceAvg(scoreAvg);
                 }
             }
+        }
+        if (hisMinDate > 0) {
             double firstExplorationIntervalAvg = Double.isNaN(item.getFirstExplorationIntervalAvg()) ? 0.0 : item.getFirstExplorationIntervalAvg();
-            double explorationInterval = (publish.getPublishTimestamp() - hisMinDate) / 86400.0;
-            firstExplorationIntervalAvg = (firstExplorationIntervalAvg * (item.getTotalArticleReleaseCountNonInfinite() - 1)
-                    + explorationInterval) / item.getTotalArticleReleaseCountNonInfinite();
+            double explorationInterval = (article.getPublishTimestamp() - hisMinDate) / 86400.0;
+            firstExplorationIntervalAvg = (firstExplorationIntervalAvg * (item.getActualArticleReleaseCount() - 1)
+                    + explorationInterval) / item.getActualArticleReleaseCount();
             item.setFirstExplorationIntervalAvg(firstExplorationIntervalAvg);
         }
         scoreHisPublishTimeMap.put(type, scorePublishTime);
     }
 
     private void setPublishPerformance(IntermediateIndicatorsExport item, NewSortStrategyExport data,
-                                       Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap) {
+                                       Map<String, Map<String, Map<Integer, List<PublishSortLog>>>> publishSortLogMap) {
         if (Objects.isNull(data.getFirstLevel())) {
             return;
         }
@@ -1296,21 +1308,28 @@ public class DataDashboardService {
         if (item.getT0FissionCount() + item.getT1FissionCount() + item.getT2FissionCount() > 0 && item.getViewBase() > 0) {
             item.setT2CumulativeFissionRate((item.getT0FissionCount() + item.getT1FissionCount() + item.getT2FissionCount()) / (double) item.getFirstLayerUV());
         }
-        Map<String, Map<Integer, PublishSortLog>> dateSortMap = publishSortLogMap.get(data.getGhId());
+        Map<String, Map<Integer, List<PublishSortLog>>> dateSortMap = publishSortLogMap.get(data.getGhId());
         if (Objects.nonNull(dateSortMap)) {
-            Map<Integer, PublishSortLog> indexMap = dateSortMap.get(item.getDateStr());
+            Map<Integer, List<PublishSortLog>> indexMap = dateSortMap.get(item.getDateStr());
             if (Objects.nonNull(indexMap)) {
-                PublishSortLog log = indexMap.get(data.getPosition());
-                double scoreAvg = item.getAccountArticleRelevanceAvg();
-                double score = 0.0;
-                if (Objects.nonNull(log) && Objects.nonNull(log.getScore())) {
-                    JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
-                    if (scoreMap.containsKey("SimilarityStrategy")) {
-                        score = scoreMap.getDoubleValue("SimilarityStrategy");
+                List<PublishSortLog> logs = indexMap.get(data.getPosition());
+                if (CollectionUtil.isNotEmpty(logs)) {
+                    for (PublishSortLog log : logs) {
+                        if (log.getTitle().equals(data.getTitle())) {
+                            double scoreAvg = item.getAccountArticleRelevanceAvg();
+                            double score = 0.0;
+                            if (Objects.nonNull(log.getScore())) {
+                                JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
+                                if (scoreMap.containsKey("SimilarityStrategy")) {
+                                    score = scoreMap.getDoubleValue("SimilarityStrategy");
+                                }
+                            }
+                            scoreAvg = (scoreAvg * (item.getActualArticleReleaseCount() - 1) + score) / item.getActualArticleReleaseCount();
+                            item.setAccountArticleRelevanceAvg(scoreAvg);
+                            break;
+                        }
                     }
                 }
-                scoreAvg = (scoreAvg * (item.getActualArticleReleaseCount() - 1) + score) / item.getActualArticleReleaseCount();
-                item.setAccountArticleRelevanceAvg(scoreAvg);
             }
         }
     }
@@ -1494,6 +1513,7 @@ public class DataDashboardService {
                                 Collectors.toMap(AccountAvgInfo::getPosition, o -> o))));
 
         String title = "";
+        List<DatastatScore> saveList = new ArrayList<>();
         for (PublishSortLog publishSortLog : sortLogList) {
             Map<String, Map<Integer, Article>> dateArticleMap = articleMap.get(publishSortLog.getGhId());
             if (Objects.isNull(dateArticleMap)) {
@@ -1559,40 +1579,44 @@ public class DataDashboardService {
                 item.setFirstExplorationIntervalAvg(explorationInterval);
             }
             result.add(item);
+            saveList.add(buildDatastatScore(publishSortLog, item));
         }
-        saveDatastatScore(dateStrList, result);
+        saveDatastatScore(dateStrList, saveList);
         result = result.stream().filter(o -> o.getIndex() == 1).collect(Collectors.toList());
         result.sort(Comparator.comparing(FirstContentScoreExport::getDateStr).reversed()
                 .thenComparing(FirstContentScoreExport::getGhId));
         return result;
     }
 
-    private void saveDatastatScore(List<String> dateStrList, List<FirstContentScoreExport> result) {
-        if (CollectionUtils.isNotEmpty(result)) {
+    private DatastatScore buildDatastatScore(PublishSortLog publishSortLog, FirstContentScoreExport value) {
+        DatastatScore item = new DatastatScore();
+        item.setDt(value.getDateStr());
+        item.setGhId(value.getGhId());
+        item.setAccountName(value.getAccountName());
+        item.setIndex(value.getIndex());
+        item.setTitle(value.getTitle());
+        item.setCategory(value.getCategory());
+        item.setStrategy(value.getStrategy());
+        item.setScore(value.getScore());
+        item.setHisFissionAvgReadRateRate(value.getHisFissionAvgReadRateRateStrategy());
+        item.setHisFissionAvgReadSumRate(value.getHisFissionAvgReadSumRateStrategy());
+        item.setSimilarity(value.getSimilarityStrategy());
+        item.setCategoryScore(value.getCategoryStrategy());
+        item.setViewCountRate(value.getViewCountRateStrategy());
+        item.setHisFissionDeWeightAvgReadSumRate(value.getHisFissionDeWeightAvgReadSumRateStrategy());
+        item.setReadCount(value.getReadCount());
+        item.setReadAvg(value.getReadAvg());
+        item.setReadAvgRate(value.getReadAvgRate());
+        item.setFirstPubInterval(value.getFirstExplorationIntervalAvg());
+        item.setPublishContentId(publishSortLog.getPublishContentId());
+        item.setCrawlerChannelContentId(publishSortLog.getCrawlerChannelContentId());
+        item.setSourceId(publishSortLog.getSourceId());
+        return item;
+    }
+
+    private void saveDatastatScore(List<String> dateStrList, List<DatastatScore> saveList) {
+        if (CollectionUtils.isNotEmpty(saveList)) {
             longArticleBaseMapper.deleteDatastatScoreByDtIn(dateStrList);
-            List<DatastatScore> saveList = new ArrayList<>();
-            for (FirstContentScoreExport value : result) {
-                DatastatScore item = new DatastatScore();
-                item.setDt(value.getDateStr());
-                item.setGhId(value.getGhId());
-                item.setAccountName(value.getAccountName());
-                item.setIndex(value.getIndex());
-                item.setTitle(value.getTitle());
-                item.setCategory(value.getCategory());
-                item.setStrategy(value.getStrategy());
-                item.setScore(value.getScore());
-                item.setHisFissionAvgReadRateRate(value.getHisFissionAvgReadRateRateStrategy());
-                item.setHisFissionAvgReadSumRate(value.getHisFissionAvgReadSumRateStrategy());
-                item.setSimilarity(value.getSimilarityStrategy());
-                item.setCategoryScore(value.getCategoryStrategy());
-                item.setViewCountRate(value.getViewCountRateStrategy());
-                item.setHisFissionDeWeightAvgReadSumRate(value.getHisFissionDeWeightAvgReadSumRateStrategy());
-                item.setReadCount(value.getReadCount());
-                item.setReadAvg(value.getReadAvg());
-                item.setReadAvgRate(value.getReadAvgRate());
-                item.setFirstPubInterval(value.getFirstExplorationIntervalAvg());
-                saveList.add(item);
-            }
             for (List<DatastatScore> saveListPartition : Lists.partition(saveList, 1000)) {
                 longArticleBaseMapper.batchInsertDatastatScore(saveListPartition);
             }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/RecommendService.java

@@ -2,14 +2,14 @@ package com.tzld.longarticle.recommend.server.service.recommend;
 
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
+import com.tzld.longarticle.recommend.server.common.CostMonitor;
+import com.tzld.longarticle.recommend.server.common.constant.SceneConstants;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PushTypeEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
-import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.ArticleUserGroupMapper;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
-import com.tzld.longarticle.recommend.server.model.entity.crawler.PublishContentSortLog;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.PublishSortLog;
 import com.tzld.longarticle.recommend.server.model.param.RecommendParam;
 import com.tzld.longarticle.recommend.server.model.param.RecommendRequest;
@@ -19,9 +19,7 @@ import com.tzld.longarticle.recommend.server.model.vo.RecommendResponse;
 import com.tzld.longarticle.recommend.server.model.vo.RecommendWithUserGroupResponse;
 import com.tzld.longarticle.recommend.server.repository.crawler.AccountAvgInfoRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
-import com.tzld.longarticle.recommend.server.repository.crawler.PublishContentSortLogRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.PublishSortLogRepository;
-import com.tzld.longarticle.recommend.server.common.constant.SceneConstants;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountIndexAvgViewCountService;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
@@ -61,8 +59,6 @@ public class RecommendService {
     @Autowired
     private RankService rankService;
     @Autowired
-    private PublishContentSortLogRepository publishContentSortLogRepository;
-    @Autowired
     private PublishSortLogRepository publishSortLogRepository;
     @Autowired
     AccountIndexAvgViewCountService accountIndexAvgViewCountService;
@@ -71,8 +67,6 @@ public class RecommendService {
     @Autowired
     private ArticleUserGroupMapper articleUserGroupMapper;
     @Autowired
-    private AigcBaseMapper aigcBaseMapper;
-    @Autowired
     private ArticleRepository articleRepository;
 
     @ApolloJsonValue("${accountStrategyConfig:{}}")
@@ -92,19 +86,26 @@ public class RecommendService {
 
         RecallResult recallResult = recallService.recall(convertToRecallParam(param));
         long t2 = System.currentTimeMillis();
-        log.info("recommendCost param:{} recall cost:{}", JSONObject.toJSONString(request), t2 - start);
+        CostMonitor.logCost("Recommend", "Recall", t2 - start);
         RankResult rankResult = rankService.rank(convertToRankParam(param, recallResult));
         long t3 = System.currentTimeMillis();
-        log.info("recommendCost param:{} rank cost:{}", JSONObject.toJSONString(request), t3 - t2);
+        CostMonitor.logCost("Recommend", "Rank", t3 - t2);
         saveSortLog(param, rankResult);
 
         RecommendResponse response = buildRecommendResponse(recallResult, rankResult, param.getPublishNum());
-        log.info("recommendCost param:{} response {} cost:{}", JSONObject.toJSONString(request), JSONObject.toJSONString(response),
-                System.currentTimeMillis() - start);
+        long t4  = System.currentTimeMillis();
+        log.info("recommendCost param:{} total cost:{} recall:{} rank:{} response: {}", JSONObject.toJSONString(request),
+                t4 - start, t2 - start, t3 - t2, JSONObject.toJSONString(response));
+        CostMonitor.logCost("Recommend", "Total", t4 - start);
         return response;
     }
 
     private void setStrategy(RecommendRequest request, RecommendParam param) {
+        int historyCount = articleRepository.countByGhIdAndTypeAndItemIndex(request.getGhId(),
+                ArticleTypeEnum.QUNFA.getVal(), 1);
+        if (historyCount < 10) {
+            param.setReplaceSimilarityAccount(true);
+        }
         // 无限发表，设置为无限发表策略
         if (Objects.equals(request.getPushType(), PushTypeEnum.AUTO_PUBLISH.getVal())
                 || Objects.equals(request.getPushType(), PushTypeEnum.ROBOPOST.getVal())) {
@@ -118,8 +119,6 @@ public class RecommendService {
             param.setStrategy(strategyConfig);
         }
         // 历史群发头条小于10条，且开启配置，则走历史表现随机策略
-        int historyCount = articleRepository.countByGhIdAndTypeAndItemIndex(request.getGhId(),
-                ArticleTypeEnum.QUNFA.getVal(), 1);
         if (historyCount < 10 && accountHisJumpStrategyList.contains(request.getGhId())) {
             param.setStrategy(RankStrategyEnum.HIS_JUMP_STRATEGY.getStrategy());
         }
@@ -262,6 +261,7 @@ public class RecommendService {
         rankParam.setScene(param.getScene());
         rankParam.setUserGroupIds(param.getUserGroupIds());
         rankParam.setType(param.getType());
+        rankParam.setReplaceSimilarityAccount(param.getReplaceSimilarityAccount());
 
         return rankParam;
     }
@@ -279,21 +279,6 @@ public class RecommendService {
         if (!ArticleTypeEnum.QUNFA.getVal().equals(param.getType())) {
             return;
         }
-        switch (param.getScene()) {
-            case FWH_COLD_START:
-                break;
-            default:
-                PublishContentSortLog log = new PublishContentSortLog();
-                log.setGhId(param.getGhId());
-                log.setAccountName(param.getAccountName());
-                log.setStrategy(param.getStrategy());
-                List<String> publishContentIds = rankResult.getContents().stream().map(Content::getId).collect(Collectors.toList());
-                log.setPublishContentId(JSONObject.toJSONString(publishContentIds));
-                log.setCreateTimestamp(System.currentTimeMillis());
-                publishContentSortLogRepository.save(log);
-                break;
-
-        }
         String dateStr = DateUtils.getCurrentDateStr("yyyyMMdd");
         List<PublishSortLog> publishSortLogSaveList = new ArrayList<>();
         List<AccountAvgInfo> avgInfoList = accountAvgInfoRepository.getAllByGhIdEqualsAndStatusEquals(param.getGhId(), 1);
@@ -310,7 +295,9 @@ public class RecommendService {
             sortLog.setDateStr(dateStr);
             sortLog.setGhId(param.getGhId());
             sortLog.setAccountName(param.getAccountName());
+            sortLog.setPublishContentId(content.getId());
             sortLog.setCrawlerChannelContentId(content.getCrawlerChannelContentId());
+            sortLog.setSourceId(content.getSourceId());
             sortLog.setTitle(content.getTitle());
             sortLog.setIndex(i);
             sortLog.setIndexAvgCount(accountIndexAvgViewCountService.getAvgReadCountByDB(avgInfoList, param.getGhId(), i));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/HistoryTitleStrategy.java

@@ -15,10 +15,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.StringUtils;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
@@ -43,9 +40,8 @@ public class HistoryTitleStrategy implements FilterStrategy {
 
     @Override
     public FilterResult filter(FilterParam param) {
-        long start = System.currentTimeMillis();
         FilterResult filterResult = new FilterResult();
-        List<String> result = new ArrayList<>();
+        List<String> result = new ArrayList<>(param.getContents().size());
         List<Content> filterContents = new ArrayList<>();
         List<Article> allArticleList = articleListRemoteService.articleList(param.getGhId(), allIndex, param.getType());
         List<String> allTitleList = allArticleList.stream().map(Article::getTitle).distinct().collect(Collectors.toList());
@@ -58,45 +54,22 @@ public class HistoryTitleStrategy implements FilterStrategy {
         if (Objects.nonNull(contentPoolConfig)) {
             firstSecondContentPool.addAll(Arrays.asList(contentPoolConfig[0], contentPoolConfig[1]));
         }
-        List<Future<Content>> futures = new ArrayList<>();
-        CountDownLatch cdl = new CountDownLatch(param.getContents().size());
+        List<Set<Character>> firstSecondTitleCache = TitleSimilarCheckUtil.makeCache(firstSecondTitleList);
+        List<Set<Character>> allTitleCache = TitleSimilarCheckUtil.makeCache(allTitleList);
+        // TODO: batching for parallelism
         for (Content content : param.getContents()) {
-            Future<Content> future = pool.submit(() -> {
-                try {
-                    boolean isDuplicate;
-                    if (CollectionUtils.isNotEmpty(firstSecondContentPool) && firstSecondContentPool.contains(content.getContentPoolType())) {
-                        // 四个内容池 配置 判断头条，次头条
-                        isDuplicate = TitleSimilarCheckUtil.isDuplicateContent(content.getTitle(), firstSecondTitleList, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
-                    } else {
-                        isDuplicate = TitleSimilarCheckUtil.isDuplicateContent(content.getTitle(), allTitleList, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
-                    }
-                    if (isDuplicate) {
-                        content.setFilterReason("历史已发布文章");
-                    }
-                    return content;
-                } finally {
-                    cdl.countDown();
-                }
-            });
-            futures.add(future);
-        }
-        try {
-            cdl.await();
-        } catch (InterruptedException e) {
-            log.error("filter error", e);
-            return null;
-        }
-
-        for (Future<Content> f : futures) {
-            try {
-                Content content = f.get();
-                if (StringUtils.hasText(content.getFilterReason())) {
-                    filterContents.add(content);
-                } else {
-                    result.add(content.getId());
-                }
-            } catch (Exception e) {
-                log.error("future get error ", e);
+            boolean isDuplicate;
+            if (CollectionUtils.isNotEmpty(firstSecondContentPool) && firstSecondContentPool.contains(content.getContentPoolType())) {
+                // 四个内容池 配置 判断头条，次头条
+                isDuplicate = TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), firstSecondTitleCache, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
+            } else {
+                isDuplicate = TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), allTitleCache, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
+            }
+            if (isDuplicate) {
+                content.setFilterReason("历史已发布文章");
+                filterContents.add(content);
+            } else {
+                result.add(content.getId());
             }
         }
         filterResult.setContentIds(result);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/RankParam.java

@@ -19,5 +19,6 @@ public class RankParam {
     private String scene;
     private List<Integer> userGroupIds;
     private String type;
+    private Boolean replaceSimilarityAccount = false;
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/RankService.java

@@ -1,6 +1,7 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank;
 
 
+import cn.hutool.core.collection.CollectionUtil;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
@@ -40,6 +41,9 @@ public class RankService {
 
     public static void printSortLog(String strategy, String accountName, List<Content> contentList) {
         JSONArray jsonArray = new JSONArray();
+        if (CollectionUtil.isEmpty(contentList)) {
+            return;
+        }
         for (Content content : contentList) {
             JSONObject obj = new JSONObject();
             obj.put("id", content.getId());

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/RankStrategy.java

@@ -64,6 +64,7 @@ public interface RankStrategy {
         scoreParam.setContents(param.getContents());
         scoreParam.setStrategy(param.getStrategy());
         scoreParam.setScene(param.getScene());
+        scoreParam.setReplaceSimilarityAccount(param.getReplaceSimilarityAccount());
         return scoreParam;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/HisJumpRankStrategy.java

@@ -51,6 +51,9 @@ public class HisJumpRankStrategy implements RankStrategy {
             if (contentPools[0].equals(item.getContent().getContentPoolType())
                     || contentPools[1].equals(item.getContent().getContentPoolType())) {
                 score = item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value())
+                        + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value());
                 if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
                     score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
@@ -59,6 +62,9 @@ public class HisJumpRankStrategy implements RankStrategy {
                 }
             } else {
                 score = item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
+                        + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value())
                         + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value())
                         + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
@@ -113,7 +119,10 @@ public class HisJumpRankStrategy implements RankStrategy {
         // 3-8
         List<Content> pool = contentMap.get(contentPools[2]);
         if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
-            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
+            int slotNum = param.getSize() - result.size();
+            List<Content> subPool = pool.subList(0, Math.min(pool.size(), 30));
+            Collections.shuffle(subPool);
+            result.addAll(subPool.subList(0, Math.min(subPool.size(), slotNum)));
         }
 
         RankStrategy.deduplication(result, contentMap, publishPool);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -2,8 +2,11 @@ package com.tzld.longarticle.recommend.server.service.recommend.recall;
 
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
+import com.tzld.longarticle.recommend.server.common.CostMonitor;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleCategoryStatusEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticlePoolPromotionSourceStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
 import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
@@ -108,15 +111,12 @@ public class RecallService implements ApplicationContextAware {
     }
 
     public RecallResult recall(RecallParam param) {
-        long t1 = System.currentTimeMillis();
         List<RecallResult.RecallData> results = new ArrayList<>();
         log.info("RecallParam {}", JSONUtils.toJson(param));
         List<RecallStrategy> strategies = getRecallStrategy(param);
         log.info("RecallStrategy {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies,
                 s -> s.getClass().getSimpleName())));
         List<Content> content = getAllContent(param);
-        long t2 = System.currentTimeMillis();
-        log.info("recall account:{} get content:{}", param.getAccountName(), t2 - t1);
         if (CollectionUtils.isEmpty(content)) {
             return new RecallResult(results);
         }
@@ -164,7 +164,10 @@ public class RecallService implements ApplicationContextAware {
     }
 
     private List<Content> getAllContent(RecallParam param) {
+        long t1 = System.currentTimeMillis();
         List<Content> content = aigcWaitingPublishContentService.getAllContent(param);
+        long t2 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "GetAllContents", t2 - t1);
         if (CollectionUtils.isEmpty(content)) {
             FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
                     "内容召回失败\n"
@@ -174,25 +177,38 @@ public class RecallService implements ApplicationContextAware {
             return content;
         }
         // category 查询
-//        setContentCategory(content);
+        setContentCategory(content);
+        long t3 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "GetCategory", t3 - t2);
         // 标题历史均值
         setTitleAvgViewCount(content, param.getGhId(), param.getType());
+        long t4 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "SetAvgViewCount", t4 - t3);
         return content;
     }
 
     public void setContentCategory(List<Content> contentList) {
-        List<String> channelContentIds = contentList.stream().map(Content::getCrawlerChannelContentId).collect(Collectors.toList());
+        List<String> channelContentIds = contentList.stream().map(Content::getCrawlerChannelContentId)
+                .collect(Collectors.toList());
         // 查询晋升rootProduceContentId
-        List<ArticlePoolPromotionSource> sourceList = articlePoolPromotionSourceRepository.getByChannelContentIdIn(channelContentIds);
-        Map<String, ArticlePoolPromotionSource> sourceMap = sourceList.stream().collect(Collectors.toMap(ArticlePoolPromotionSource::getChannelContentId, Function.identity()));
-        List<String> publishContentIds = sourceList.stream().map(ArticlePoolPromotionSource::getRootPublishContentId).collect(Collectors.toList());
+        List<ArticlePoolPromotionSource> sourceList = articlePoolPromotionSourceRepository
+                .getByChannelContentIdInAndStatusAndDeleted(channelContentIds,
+                        ArticlePoolPromotionSourceStatusEnum.FINISH.getCode(), 0);
+        Map<String, ArticlePoolPromotionSource> sourceMap = sourceList.stream()
+                .collect(Collectors.toMap(ArticlePoolPromotionSource::getChannelContentId, Function.identity()));
+        List<String> publishContentIds = sourceList.stream().
+                map(ArticlePoolPromotionSource::getRootPublishContentId).collect(Collectors.toList());
         List<PublishContent> publishContentList = publishContentRepository.getByIdIn(publishContentIds);
-        Map<String, PublishContent> publishContentMap = publishContentList.stream().collect(Collectors.toMap(PublishContent::getId, Function.identity()));
+        Map<String, PublishContent> publishContentMap = publishContentList.stream()
+                .collect(Collectors.toMap(PublishContent::getId, Function.identity()));
         // 根据produceContentId查询category
-        List<ArticleCategory> articleCategoryList = articleCategoryRepository.findAll();
-        Map<String, ArticleCategory> categoryMap = articleCategoryList.stream().collect(Collectors.toMap(ArticleCategory::getProduceContentId, Function.identity()));
-        Map<String, ArticleCategory> coldStartCategoryMap = articleCategoryList.stream().collect(Collectors.toMap(ArticleCategory::getChannelContentId, Function.identity(), (a, b) -> a));
-        Map<String, ArticleCategory> titleCategoryMap = articleCategoryList.stream().collect(Collectors.toMap(ArticleCategory::getTitleMd5, Function.identity(), (a, b) -> a));
+        List<ArticleCategory> articleCategoryList = articleCategoryRepository.getByStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
+        Map<String, ArticleCategory> categoryMap = articleCategoryList.stream()
+                .collect(Collectors.toMap(ArticleCategory::getProduceContentId, Function.identity()));
+        Map<String, ArticleCategory> coldStartCategoryMap = articleCategoryList.stream()
+                .collect(Collectors.toMap(ArticleCategory::getChannelContentId, Function.identity(), (a, b) -> a));
+        Map<String, ArticleCategory> titleCategoryMap = articleCategoryList.stream()
+                .collect(Collectors.toMap(ArticleCategory::getTitleMd5, Function.identity(), (a, b) -> a));
         for (Content content : contentList) {
             ArticlePoolPromotionSource source = sourceMap.get(content.getCrawlerChannelContentId());
             ArticleCategory category = null;
@@ -214,7 +230,7 @@ public class RecallService implements ApplicationContextAware {
                 content.setCategory(Collections.singletonList(category.getCategory()));
                 continue;
             }
-            log.error("setContentCategory NullError channelContentId:{}", content.getCrawlerChannelContentId());
+//            log.error("setContentCategory NullError channelContentId:{}", content.getCrawlerChannelContentId());
         }
     }
 
@@ -265,7 +281,7 @@ public class RecallService implements ApplicationContextAware {
         List<Article> hisArticleList = new ArrayList<>();
         List<List<String>> titleMd5Partition = Lists.partition(new ArrayList<>(titleMd5List), 1000);
         for (List<String> titleMd5s : titleMd5Partition) {
-            hisArticleList.addAll(articleRepository.getByTitleMd5InAndTypeEqualsAndStatusEquals(titleMd5s, type, 1));
+            hisArticleList.addAll(crawlerBaseMapper.getByTitleMd5InAndTypeEqualsAndStatusEquals(titleMd5s, type, 1));
         }
         Map<String, Map<Integer, List<Article>>> map = hisArticleList.stream()
                 .collect(Collectors.groupingBy(Article::getTitle, Collectors.groupingBy(Article::getItemIndex)));
@@ -273,19 +289,19 @@ public class RecallService implements ApplicationContextAware {
         List<ArticleDetailInfo> articleDetailInfoList = new ArrayList<>();
         List<List<String>> snPartition = Lists.partition(new ArrayList<>(snList), 1000);
         for (List<String> sns : snPartition) {
-            articleDetailInfoList.addAll(articleDetailInfoRepository.getAllByWxSnIn(sns));
+            articleDetailInfoList.addAll(crawlerBaseMapper.getAllByWxSnIn(sns));
         }
         Map<String, List<ArticleDetailInfo>> articleDetailInfoMap = articleDetailInfoList.stream()
                 .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
         // 获取历史已发布文章所属头条内容
         Set<String> ghIds = hisArticleList.stream().map(Article::getGhId).collect(Collectors.toSet());
         Set<String> appMsgIds = hisArticleList.stream().map(Article::getAppMsgId).collect(Collectors.toSet());
-        List<Article> firstIndexHisArticleList = articleRepository.getByGhIdInAndAppMsgIdInAndItemIndexAndTypeEqualsAndStatusEquals(
+        List<Article> firstIndexHisArticleList = crawlerBaseMapper.getByGhIdInAndAppMsgIdInAndItemIndexAndTypeEqualsAndStatusEquals(
                 ghIds, appMsgIds, 1, type, 1);
         Map<String, Map<String, Article>> firstIndexHisArticleMap = firstIndexHisArticleList.stream()
                 .collect(Collectors.groupingBy(Article::getGhId, Collectors.toMap(Article::getAppMsgId, o -> o)));
         // 获取发布账号 位置历史均值
-        List<AccountAvgInfo> accountAvgInfoList = accountAvgInfoRepository.getAllByGhIdIn(ghIds);
+        List<AccountAvgInfo> accountAvgInfoList = crawlerBaseMapper.getAllByGhIdIn(ghIds);
         Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap = accountAvgInfoList.stream()
                 .filter(o -> Objects.nonNull(o.getReadAvg()) && o.getReadAvg() > 0 && o.getFans() > 1000)
                 .collect(Collectors.groupingBy(AccountAvgInfo::getGhId, Collectors.groupingBy(AccountAvgInfo::getUpdateTime,
@@ -429,7 +445,8 @@ public class RecallService implements ApplicationContextAware {
                 continue;
             }
             int sumFission0 = 0;
-            Date minDate = article.getArticleDetailInfoList().stream().map(ArticleDetailInfo::getRecallDt).min(Date::compareTo).orElse(new Date());
+            Date minDate = article.getArticleDetailInfoList().stream().map(ArticleDetailInfo::getRecallDt)
+                    .min(Date::compareTo).orElse(new Date());
             for (ArticleDetailInfo articleDetailInfo : article.getArticleDetailInfoList()) {
                 if (articleDetailInfo.getRecallDt().equals(minDate) && Objects.nonNull(articleDetailInfo.getFission0())) {
                     sumFission0 += articleDetailInfo.getFission0();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/strategy/DefaultRecallStrategy.java

@@ -1,5 +1,6 @@
 package com.tzld.longarticle.recommend.server.service.recommend.recall.strategy;
 
+import com.tzld.longarticle.recommend.server.common.CostMonitor;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterResult;
@@ -24,15 +25,20 @@ public class DefaultRecallStrategy implements RecallStrategy {
 
     @Override
     public RecallResult.RecallData recall(RecallParam param) {
+        long t1 = System.currentTimeMillis();
         List<Content> content = param.getContent();
         // 处理 content
         FilterParam filterParam = FilterParamFactory.create(param, content);
         FilterResult filterResult = filterService.filter(filterParam);
+        long t2 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "ProcessFilter", t2 - t1);
         // 处理 content
         RecallResult.RecallData result = new RecallResult.RecallData();
         result.setContents(content.stream().filter(o -> filterResult.getContentIds().contains(o.getId()))
                 .collect(Collectors.toList()));
         result.setFilterContents(filterResult.getFilterContent());
+        long t3 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "SetResults", t3 - t2);
         return result;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/ScoreParam.java

@@ -21,4 +21,5 @@ public class ScoreParam {
     private List<Content> contents;
     private String strategy;
     private String scene;
+    private Boolean replaceSimilarityAccount = false;
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/ScoreService.java

@@ -59,9 +59,12 @@ public class ScoreService implements ApplicationContextAware {
                 try {
                     List<Score> result = strategy.score(param);
                     return result;
+                } catch (Exception e) {
+                    log.error("scoreService error:{}", e.getMessage(), e);
                 } finally {
                     cdl.countDown();
                 }
+                return new ArrayList<>();
             });
             futures.add(future);
         }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/strategy/CategoryStrategy.java

@@ -42,8 +42,14 @@ public class CategoryStrategy implements ScoreStrategy {
         if (CollectionUtils.isEmpty(param.getContents())) {
             return scores;
         }
-        AccountCategory accountCategory = accountCategoryRepository.getByGhIdAndStatus(param.getGhId(), StatusEnum.ONE.getCode());
-        if (Objects.isNull(accountCategory) || !StringUtils.hasText(accountCategory.getCategoryMap())) {
+        List<AccountCategory> accountCategoryList = accountCategoryRepository.getByGhIdAndStatus(param.getGhId(), StatusEnum.ONE.getCode());
+        if (CollectionUtils.isEmpty(accountCategoryList)) {
+            return scores;
+        }
+        AccountCategory accountCategory = accountCategoryList.stream()
+                .sorted(Comparator.comparing(AccountCategory::getDt, Comparator.reverseOrder()))
+                .findFirst().get();
+        if (!StringUtils.hasText(accountCategory.getCategoryMap())) {
             return scores;
         }
         JSONObject categoryWeightMap = JSONObject.parseObject(accountCategory.getCategoryMap());

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/strategy/SimilarityStrategy.java

@@ -1,18 +1,27 @@
 package com.tzld.longarticle.recommend.server.service.recommend.score.strategy;
 
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.model.dto.kimi.KimiResult;
+import com.tzld.longarticle.recommend.server.remote.KimiApiService;
 import com.tzld.longarticle.recommend.server.remote.NLPRemoteService;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.score.Score;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreParam;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreStrategy;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
 
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author dyp
@@ -23,6 +32,23 @@ public class SimilarityStrategy implements ScoreStrategy {
 
     @Autowired
     private NLPRemoteService nlpRemoteService;
+    @Autowired
+    private ArticleRepository articleRepository;
+    @Autowired
+    private KimiApiService kimiApiService;
+
+    @Autowired
+    private RedisTemplate<String, String> redisTemplate;
+
+    @ApolloJsonValue("${account.score.sim.replace:{}}")
+    private Map<String, String> accountSimScoreReplaceMap;
+    @Value("${kimiSimilarityTypePrompt:}")
+    private String kimiSimilarityTypePrompt;
+    @ApolloJsonValue("${similarityTypeAccountMap:}")
+    private Map<String, List<String>> similarityTypeAccountMap;
+
+    @Value("${spring.profiles.active:prod}")
+    private String env;
 
     @Override
     public List<Score> score(ScoreParam param) {
@@ -31,20 +57,49 @@ public class SimilarityStrategy implements ScoreStrategy {
         if (CollectionUtils.isEmpty(param.getContents())) {
             return Collections.emptyList();
         }
-        Map<String, Double> scoreMap = nlpRemoteService.score(param.getGhId(), param.getAccountName(), param.getContents());
-
-//        double min = scoreMap.values().stream()
-//                .min(Double::compareTo)
-//                .orElse(0.0);
-//        double max = scoreMap.values().stream()
-//                .max(Double::compareTo)
-//                .orElse(0.0);
+        String ghId = param.getGhId();
+        if (param.getReplaceSimilarityAccount()) {
+            if (accountSimScoreReplaceMap.containsKey(ghId)) {
+                ghId = accountSimScoreReplaceMap.get(ghId);
+            } else {
+                String redisKey = "AccountSimType_" + param.getAccountName();
+                String type = redisTemplate.opsForValue().get(redisKey);
+                if (!StringUtils.hasText(type)) {
+                    String prompt = kimiSimilarityTypePrompt.replace("accountName", param.getAccountName());
+                    // 调用kimi判断账号类型
+                    KimiResult kimiResult = kimiApiService.requestOfficialApi(prompt, null, null);
+                    if (kimiResult.isSuccess()) {
+                        try {
+                            type = kimiResult.getResponse().getChoices().get(0).getMessage().getContent();
+                            log.info("SimilarityStrategy kimiResultType:{}", type);
+                            redisTemplate.opsForValue().set(redisKey, type, 10L, TimeUnit.DAYS);
+                        } catch (Exception e) {
+                            log.error(kimiResult.getResponse().getChoices().get(0).getMessage().getContent());
+                        }
+                    }
+                }
+                // 根据类型随机再类型池中选取账号
+                List<String> ghIds = similarityTypeAccountMap.get(type);
+                if (CollectionUtils.isNotEmpty(ghIds)) {
+                    ghId = ghIds.get(RandomUtils.nextInt(0, ghIds.size()));
+//                    try {
+//                        Map<String, String> value = accountSimScoreReplaceMap;
+//                        value.put(param.getGhId(), ghId);
+//                        ApolloConfigModifier.modifyConfig("account.score.sim.replace", JSONObject.toJSONString(value), env.toUpperCase());
+//                        log.info("SimilarityStrategy 更新apollo配置 account.score.sim.replace 成功 {}：{}", param.getGhId(), ghId);
+//                    } catch (Exception e) {
+//                        log.error("SimilarityStrategy 更新apollo配置失败", e);
+//                    }
+                }
+                log.info("SimilarityStrategy 排序账号：{} 替换账号：{}", param.getGhId(), ghId);
+            }
+        }
+        Map<String, Double> scoreMap = nlpRemoteService.score(ghId, param.getAccountName(), param.getContents());
 
         List<Score> scores = CommonCollectionUtils.toList(param.getContents(), c -> {
             Score score = new Score();
             score.setContentId(c.getId());
             double val = scoreMap.get(c.getId()) == null ? 0.0 : scoreMap.get(c.getId());
-            // score.setScore(NormalizationUtils.minMax(val, min, max));
             score.setScore(val);
             score.setStrategy(this);
             return score;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/util/feishu/FeishuMessageSender.java

@@ -24,6 +24,8 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.util.EntityUtils;
 import org.apache.http.util.TextUtils;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.util.Pair;
 import org.springframework.stereotype.Component;
 
@@ -32,7 +34,7 @@ import java.util.*;
 
 @Component
 @Slf4j
-public class FeishuMessageSender {
+public class FeishuMessageSender implements InitializingBean {
 
     private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(FeishuMessageSender.class);
 
@@ -40,6 +42,16 @@ public class FeishuMessageSender {
 
     private static final CloseableHttpClient client = HttpPoolFactory.defaultPool();
 
+    private static String staticEnv;
+
+    @Value("${spring.profiles.active:prod}")
+    private String env;
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        FeishuMessageSender.staticEnv = env;
+    }
+
     public void sendChat(String chatId, String title, Map<String, String> infos) {
         sendChat(chatId, null, title, infos);
     }
@@ -73,6 +85,9 @@ public class FeishuMessageSender {
 
     public static void sendWebHookMessage(String robotId, String msg) {
         // 使用自定义群机器人webhook方式，支持外部群
+        if (!"prod".equals(staticEnv)) {
+            return;
+        }
         String webhookUrl = webHookUrl + robotId;
         String hrc = request(msg, webhookUrl);
         if (hrc == null) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/recommend/ArticleVideoPoolController.java

@@ -0,0 +1,28 @@
+package com.tzld.longarticle.recommend.server.web.recommend;
+
+import com.tzld.longarticle.recommend.server.common.response.CommonResponse;
+import com.tzld.longarticle.recommend.server.model.param.ArticleVideoPoolSourceParam;
+import com.tzld.longarticle.recommend.server.model.vo.ArticleVideoPoolSourceVO;
+import com.tzld.longarticle.recommend.server.service.ArticleVideoPoolService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+@RestController
+@RequestMapping("/videoPool")
+@Slf4j
+public class ArticleVideoPoolController {
+
+    @Autowired
+    private ArticleVideoPoolService service;
+    @PostMapping("/getSource")
+    public CommonResponse<List<ArticleVideoPoolSourceVO>> getSource(@RequestBody ArticleVideoPoolSourceParam param) {
+        return CommonResponse.success(service.getSource(param));
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/DataFlushController.java → long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/recommend/DataFlushController.java

@@ -1,4 +1,4 @@
-package com.tzld.longarticle.recommend.server.web;
+package com.tzld.longarticle.recommend.server.web.recommend;
 
 import com.tzld.longarticle.recommend.server.service.DataFlushService;
 import lombok.extern.slf4j.Slf4j;

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -160,6 +160,19 @@
         </if>
     </select>
 
+    <select id="getLateNearestPublishContent"
+            resultType="com.tzld.longarticle.recommend.server.model.entity.aigc.PublishContent">
+        select *
+        from publish_content
+        where publish_account_id = #{publishAccountId}
+        and channel = 5
+        and status = 2
+        <if test="publishTimestamp != null">
+            and publish_timestamp > (#{publishTimestamp})
+        </if>
+        order by publish_timestamp
+    </select>
+
     <select id="getCrawlerContentByChannelContentId"
             resultType="com.tzld.longarticle.recommend.server.model.dto.CrawlerContent">
         select cc.channel_content_id, ca.wx_gh as ghId, cc.title, cc.publish_timestamp

long-article-recommend-service/src/main/resources/mapper/crawler/CrawlerBaseMapper.xml

@@ -86,4 +86,51 @@
         select * from official_articles_v2 where publish_timestamp > #{timestamp} order by publish_timestamp limit 100
     </select>
 
+    <select id="getByTitleMd5InAndTypeEqualsAndStatusEquals"
+            resultType="com.tzld.longarticle.recommend.server.model.entity.crawler.Article">
+        select wx_sn, ghId, appMsgId, title, ItemIndex, publish_timestamp, show_view_count
+        from official_articles_v2
+        where title_md5 in
+        <foreach collection="titleMd5s" item="item" separator="," open="(" close=")">
+            #{item}
+        </foreach>
+        and Type = #{type} and status = #{status}
+    </select>
+
+    <select id="getAllByWxSnIn"
+            resultType="com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleDetailInfo">
+        select wx_sn, recall_dt, first_level, fission_0
+        from long_articles_detail_info
+        where wx_sn in
+        <foreach collection="wxSnList" item="item" separator="," open="(" close=")">
+            #{item}
+        </foreach>
+    </select>
+
+    <select id="getAllByGhIdIn"
+            resultType="com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo">
+        select gh_id, position, update_time, fans, read_avg
+        from account_avg_info_v3
+        where gh_id in
+        <foreach collection="ghIdList" item="item" separator="," open="(" close=")">
+            #{item}
+        </foreach>
+    </select>
+    <select id="getByGhIdInAndAppMsgIdInAndItemIndexAndTypeEqualsAndStatusEquals"
+            resultType="com.tzld.longarticle.recommend.server.model.entity.crawler.Article">
+        select ghId, appMsgId, show_view_count
+        from official_articles_v2
+        where ghId in
+        <foreach collection="ghIdList" item="item" separator="," open="(" close=")">
+            #{item}
+        </foreach>
+        and appMsgId in
+        <foreach collection="appMsgIdList" item="item" separator="," open="(" close=")">
+            #{item}
+        </foreach>
+        and itemIndex = #{itemIndex}
+        and Type = #{type}
+        and status = #{status}
+    </select>
+
 </mapper>

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -72,14 +72,16 @@
     <insert id="batchInsertDatastatScore">
         INSERT INTO datastat_score
         (dt, gh_id, account_name, `index`, title, strategy, score, similarity, view_count_rate,
-         his_fission_avg_read_rate_rate, his_fission_avg_read_sum_rate, his_fission_de_weight_avg_read_sum_rate,
-        read_count, read_avg, read_avg_rate, category, category_score, first_pub_interval)
+        his_fission_avg_read_rate_rate, his_fission_avg_read_sum_rate, his_fission_de_weight_avg_read_sum_rate,
+        read_count, read_avg, read_avg_rate, category, category_score, first_pub_interval, publish_content_id,
+        crawler_channel_content_id, source_id)
         VALUES
         <foreach collection="list" item="item" separator=",">
             (#{item.dt}, #{item.ghId}, #{item.accountName}, #{item.index}, #{item.title}, #{item.strategy},
-             #{item.score}, #{item.similarity}, #{item.viewCountRate}, #{item.hisFissionAvgReadRateRate},
+            #{item.score}, #{item.similarity}, #{item.viewCountRate}, #{item.hisFissionAvgReadRateRate},
             #{item.hisFissionAvgReadSumRate}, #{item.hisFissionDeWeightAvgReadSumRate}, #{item.readCount},
-             #{item.readAvg}, #{item.readAvgRate}, #{item.category}, #{item.categoryScore}, #{item.firstPubInterval})
+            #{item.readAvg}, #{item.readAvgRate}, #{item.category}, #{item.categoryScore}, #{item.firstPubInterval},
+            #{item.publishContentId}, #{item.crawlerChannelContentId}, #{item.sourceId})
         </foreach>
     </insert>
 
@@ -227,4 +229,14 @@
         </foreach>
     </insert>
 
+    <select id="getPublishSingleVideoSource"
+            resultType="com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishSingleVideoSource">
+        select content_trace_id, article_title, flow_pool_level
+        from publish_single_video_source
+        where bad_status = 0 and audit_status = 1
+        <if test="flowPoolLevel!= null ">
+            and flow_pool_level = #{flowPoolLevel}
+        </if>
+    </select>
+
 </mapper>

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/DataDashboardTest.java

@@ -19,7 +19,7 @@ public class DataDashboardTest {
 
     @Test
     public void test() {
-        List<IntermediateIndicatorsExport> result = dataDashboardService.intermediateIndicatorsData("20241122");
+        List<IntermediateIndicatorsExport> result = dataDashboardService.intermediateIndicatorsData("20241129");
         log.info(JSONObject.toJSONString(result));
     }
 }