历史均值获取

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/Content.java

@@ -39,5 +39,7 @@ public class Content {
 //    private Long crawlerPublishTimestamp;
 //    private String crawlerAccountName;
     private String filterReason;
+
+    private List<ContentHisPublishArticle> hisPublishArticleList;
 }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/ContentHisPublishArticle.java

@@ -0,0 +1,19 @@
+package com.tzld.longarticle.recommend.server.model;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class ContentHisPublishArticle {
+
+    private String ghId;
+    private String accountName;
+    private String title;
+    private Integer itemIndex;
+    private Integer showViewCount;
+    private Long updateTime;
+    private Integer avgViewCount;
+    private Double viewCountRate;
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/AIGCRemoteService.java

@@ -28,7 +28,6 @@ public class AIGCRemoteService {
     private static final String aigcGetAllContentUrl = "http://aigc-api.cybertogether.net/aigc/publish/content/gzhWaitingPublishContent";
 
 
-    // TODO 如果过滤失败，那么认为所有视频都被过滤掉
     public List<Content> getAllContent(RecallParam param) {
         long start = System.currentTimeMillis();
         List<Content> result = new ArrayList<>();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/ArticleSensitiveRemoteService.java

@@ -9,7 +9,6 @@ import com.tencentcloudapi.tms.v20201229.models.TextModerationRequest;
 import com.tencentcloudapi.tms.v20201229.models.TextModerationResponse;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleSensitiveRepository;
 import com.tzld.longarticle.recommend.server.repository.entity.crawler.ArticleSensitive;
-import com.tzld.longarticle.recommend.server.util.Md5Util;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -17,6 +16,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
 import java.util.Base64;
+import java.util.Map;
 import java.util.Objects;
 
 @Service
@@ -31,11 +31,11 @@ public class ArticleSensitiveRemoteService {
     @Value("${tecent.cloud.secret.key:CFArLAnCONHLany1qRrLyW2Rmw3ITXi1}")
     private String tecentCloudSecretKey;
 
-    public Boolean articleSensitive(String title) {
+    public Boolean articleSensitive(String title, Map<String, String> titleMd5Map, Map<String, ArticleSensitive> articleSensitiveMap) {
         long start = System.currentTimeMillis();
         boolean isSensitive = false;
-        String md5 = Md5Util.encoderByMd5(title);
-        ArticleSensitive articleSensitive = getArticleSensitive(md5);
+        String md5 = titleMd5Map.get(title);
+        ArticleSensitive articleSensitive = articleSensitiveMap.get(md5);
         if (Objects.nonNull(articleSensitive) && StringUtils.hasText(articleSensitive.getSensitiveSuggestion())) {
             if ("Pass".equals(articleSensitive.getSensitiveSuggestion())
                     || ("Polity".equals(articleSensitive.getLabel()) && "ForeignLeader".equals(articleSensitive.getSubLabel()))) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/ArticleRepository.java

@@ -11,4 +11,6 @@ public interface ArticleRepository extends JpaRepository<Article, String> {
 
     List<Article> getByAccountNameAndItemIndexInAndTypeEquals(String accountName, List<Integer> indexList, String type);
 
+    List<Article> getByTitleIn(List<String> titleList);
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/strategy/AccountPreDistributeStrategy.java

@@ -29,6 +29,7 @@ public class AccountPreDistributeStrategy implements FilterStrategy {
 
     @Override
     public FilterResult filter(FilterParam param) {
+        long start = System.currentTimeMillis();
         FilterResult filterResult = new FilterResult();
         List<String> result = new ArrayList<>();
         List<Content> contents = param.getContents();
@@ -61,6 +62,7 @@ public class AccountPreDistributeStrategy implements FilterStrategy {
         }
         filterResult.setContentIds(result);
         filterResult.setFilterContent(filterContents);
+        log.info("AccountPreDistributeStrategy cost:{}", System.currentTimeMillis() - start);
         return filterResult;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/strategy/BadStrategy.java

@@ -26,6 +26,7 @@ public class BadStrategy implements FilterStrategy {
 
     @Override
     public FilterResult filter(FilterParam param) {
+        long start = System.currentTimeMillis();
         FilterResult filterResult = new FilterResult();
         List<String> result = new ArrayList<>();
         List<Content> contents = param.getContents();
@@ -40,6 +41,7 @@ public class BadStrategy implements FilterStrategy {
         }
         filterResult.setContentIds(result);
         filterResult.setFilterContent(filterContents);
+        log.info("BadStrategy cost:{}", System.currentTimeMillis() - start);
         return filterResult;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/strategy/ColdStartBackupFilterStrategy.java

@@ -24,6 +24,7 @@ public class ColdStartBackupFilterStrategy implements FilterStrategy {
 
     @Override
     public FilterResult filter(FilterParam param) {
+        long start = System.currentTimeMillis();
         FilterResult filterResult = new FilterResult();
         List<String> result = new ArrayList<>();
         List<Content> contents = param.getContents();
@@ -34,6 +35,7 @@ public class ColdStartBackupFilterStrategy implements FilterStrategy {
             }
         }
         filterResult.setContentIds(result);
+        log.info("ColdStartBackupFilterStrategy cost:{}", System.currentTimeMillis() - start);
         return filterResult;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/strategy/HistoryTitleStrategy.java

@@ -37,6 +37,7 @@ public class HistoryTitleStrategy implements FilterStrategy {
 
     @Override
     public FilterResult filter(FilterParam param) {
+        long start = System.currentTimeMillis();
         FilterResult filterResult = new FilterResult();
         List<String> result = new ArrayList<>();
         List<Content> filterContents = new ArrayList<>();
@@ -67,6 +68,7 @@ public class HistoryTitleStrategy implements FilterStrategy {
         }
         filterResult.setContentIds(result);
         filterResult.setFilterContent(filterContents);
+        log.info("HistoryTitleStrategy cost:{}", System.currentTimeMillis() - start);
         return filterResult;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/strategy/SensitiveStrategy.java

@@ -1,12 +1,16 @@
 package com.tzld.longarticle.recommend.server.service.filter.strategy;
 
+import cn.hutool.core.collection.CollectionUtil;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.model.Content;
 import com.tzld.longarticle.recommend.server.remote.ArticleSensitiveRemoteService;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleSensitiveRepository;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.ArticleSensitive;
 import com.tzld.longarticle.recommend.server.service.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.filter.FilterStrategy;
+import com.tzld.longarticle.recommend.server.util.Md5Util;
 import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -14,11 +18,14 @@ import org.springframework.stereotype.Component;
 import org.springframework.util.StringUtils;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 @Component
 @Slf4j
@@ -26,6 +33,8 @@ public class SensitiveStrategy implements FilterStrategy {
 
     @Autowired
     private ArticleSensitiveRemoteService articleSensitiveRemoteService;
+    @Autowired
+    private ArticleSensitiveRepository articleSensitiveRepository;
 
     @ApolloJsonValue("${UnSafeTitles:[]}")
     private static List<String> UnSafeTitles;
@@ -34,17 +43,33 @@ public class SensitiveStrategy implements FilterStrategy {
 
     @Override
     public FilterResult filter(FilterParam param) {
+        long start = System.currentTimeMillis();
         FilterResult filterResult = new FilterResult();
         List<String> result = new ArrayList<>();
         List<Content> filterContents = new ArrayList<>();
 
         CountDownLatch cdl = new CountDownLatch(param.getContents().size());
         List<Future<Content>> futures = new ArrayList<>();
+        Map<String, String> titleMd5Map = new HashMap<>();
+        Map<String, ArticleSensitive> articleSensitiveMap = new HashMap<>();
+        List<String> md5List = new ArrayList<>();
+        for (Content content : param.getContents()) {
+            String md5 = Md5Util.encoderByMd5(content.getTitle());
+            md5List.add(md5);
+            titleMd5Map.put(content.getTitle(), md5);
+        }
+        List<ArticleSensitive> articleSensitiveList = getArticleSensitive(md5List);
+        if (CollectionUtil.isNotEmpty(articleSensitiveList)) {
+            articleSensitiveMap = articleSensitiveList.stream().collect(Collectors.toMap(ArticleSensitive::getMd5, o -> o));
+        }
 
         for (Content content : param.getContents()) {
+            Map<String, ArticleSensitive> finalArticleSensitiveMap = articleSensitiveMap;
             Future<Content> future = pool.submit(() -> {
                 try {
-                    boolean isSensitive = articleSensitiveRemoteService.articleSensitive(content.getTitle());
+                    boolean isSensitive = articleSensitiveRemoteService.articleSensitive(content.getTitle(),
+                            titleMd5Map,
+                            finalArticleSensitiveMap);
                     if (isSensitive) {
                         content.setFilterReason("安全违规");
                     } else if (TitleSimilarCheckUtil.isDuplicateContent(content.getTitle(), UnSafeTitles)) {
@@ -78,7 +103,12 @@ public class SensitiveStrategy implements FilterStrategy {
         }
         filterResult.setContentIds(result);
         filterResult.setFilterContent(filterContents);
+        log.info("SensitiveStrategy cost:{}", System.currentTimeMillis() - start);
         return filterResult;
     }
 
+    public List<ArticleSensitive> getArticleSensitive(List<String> md5List) {
+        return articleSensitiveRepository.findAllById(md5List);
+    }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recall/RecallService.java

@@ -1,25 +1,32 @@
 package com.tzld.longarticle.recommend.server.service.recall;
 
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
+import com.tzld.longarticle.recommend.server.model.Content;
+import com.tzld.longarticle.recommend.server.model.ContentHisPublishArticle;
+import com.tzld.longarticle.recommend.server.repository.aigc.CrawlerMetaArticleRepository;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
+import com.tzld.longarticle.recommend.server.repository.entity.aigc.CrawlerMetaArticle;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.service.recall.strategy.ColdStartBackupRecallStrategy;
 import com.tzld.longarticle.recommend.server.service.recall.strategy.DefaultRecallStrategy;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import com.tzld.longarticle.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.PostConstruct;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 /**
  * @author dyp
@@ -28,6 +35,11 @@ import java.util.concurrent.TimeUnit;
 @Slf4j
 public class RecallService implements ApplicationContextAware {
 
+    @Autowired
+    ArticleRepository articleRepository;
+    @Autowired
+    CrawlerMetaArticleRepository crawlerMetaArticleRepository;
+
     private final Map<String, RecallStrategy> strategyMap = new HashMap<>();
     private ApplicationContext applicationContext;
     private final ExecutorService pool = ThreadPoolFactory.recallPool();
@@ -90,4 +102,58 @@ public class RecallService implements ApplicationContextAware {
         this.applicationContext = applicationContext;
     }
 
+    public void setContentCategory(List<Content> contentList) {
+        long start = System.currentTimeMillis();
+        List<String> channelContentIds = contentList.stream().map(Content::getCrawlerChannelContentId).collect(Collectors.toList());
+        List<CrawlerMetaArticle> categoryList = getContentCategoryByChannelContentId(channelContentIds);
+        if (CollectionUtils.isEmpty(categoryList)) {
+            return;
+        }
+        Map<String, List<String>> categoryMap = categoryList.stream().collect(Collectors.groupingBy(CrawlerMetaArticle::getChannelContentId,
+                Collectors.mapping(CrawlerMetaArticle::getCategory, Collectors.toList())));
+        for (Content content : contentList) {
+            content.setCategory(categoryMap.get(content.getCrawlerChannelContentId()));
+        }
+        log.info("setContentCategory cost:{}", System.currentTimeMillis() - start);
+    }
+
+    private List<CrawlerMetaArticle> getContentCategoryByChannelContentId(List<String> channelContentIds) {
+        if (CollectionUtils.isEmpty(channelContentIds)) {
+            return new ArrayList<>();
+        }
+        return crawlerMetaArticleRepository.getByChannelContentIdIn(channelContentIds);
+    }
+
+    public void setTitleAvgViewCount(List<Content> contentList) {
+        long start = System.currentTimeMillis();
+        List<String> titleList = contentList.stream().map(Content::getTitle).collect(Collectors.toList());
+        List<Article> hisArticleList = articleRepository.getByTitleIn(titleList);
+        Map<String, Map<Integer, List<Article>>> map = hisArticleList.stream()
+                .collect(Collectors.groupingBy(Article::getTitle, Collectors.groupingBy(Article::getItemIndex)));
+        for (Content content : contentList) {
+            List<Article> hisArticles = new ArrayList<>();
+            Map<Integer, Integer> indexAvgMap = new HashMap<>();
+            Map<Integer, List<Article>> indexArticleMap = map.get(content.getTitle());
+            if (Objects.nonNull(indexArticleMap) && !indexArticleMap.isEmpty()) {
+                for (Map.Entry<Integer, List<Article>> indexArticleEntry : indexArticleMap.entrySet()) {
+                    List<Article> indexArticleList = indexArticleEntry.getValue();
+                    int sumViewCount = indexArticleList.stream().mapToInt(Article::getShowViewCount).sum();
+                    indexAvgMap.put(indexArticleEntry.getKey(), sumViewCount/ indexArticleList.size());
+                    hisArticles.addAll(indexArticleList);
+                }
+            }
+            content.setHisPublishArticleList(new ArrayList<>());
+            for (Article hisArticle : hisArticles) {
+                ContentHisPublishArticle article = new ContentHisPublishArticle();
+                BeanUtils.copyProperties(hisArticle, article);
+                article.setAvgViewCount(indexAvgMap.get(article.getItemIndex()));
+                if (Objects.nonNull(article.getAvgViewCount()) && article.getAvgViewCount() > 0) {
+                    article.setViewCountRate((article.getShowViewCount() * 1.0) / article.getAvgViewCount());
+                }
+                content.getHisPublishArticleList().add(article);
+            }
+        }
+        log.info("setTitleAvgViewCount cost:{}", System.currentTimeMillis() - start);
+    }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recall/strategy/ColdStartBackupRecallStrategy.java

@@ -2,25 +2,22 @@ package com.tzld.longarticle.recommend.server.service.recall.strategy;
 
 import com.tzld.longarticle.recommend.server.model.Content;
 import com.tzld.longarticle.recommend.server.remote.AIGCRemoteService;
-import com.tzld.longarticle.recommend.server.repository.aigc.CrawlerMetaArticleRepository;
-import com.tzld.longarticle.recommend.server.repository.entity.aigc.CrawlerMetaArticle;
 import com.tzld.longarticle.recommend.server.service.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.filter.FilterService;
 import com.tzld.longarticle.recommend.server.service.recall.*;
-import org.apache.commons.collections4.CollectionUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
  * @author dyp
  */
 @Component
+@Slf4j
 public class ColdStartBackupRecallStrategy implements RecallStrategy {
 
     @Autowired
@@ -28,14 +25,16 @@ public class ColdStartBackupRecallStrategy implements RecallStrategy {
     @Autowired
     private AIGCRemoteService aigcRemoteService;
     @Autowired
-    CrawlerMetaArticleRepository crawlerMetaArticleRepository;
+    RecallService recallService;
 
     @Override
     public RecallResult.RecallData recall(RecallParam param) {
 
         List<Content> content = aigcRemoteService.getAllContent(param);
+        // 标题历史均值
+        recallService.setTitleAvgViewCount(content);
         // category 查询
-        setContentCategory(content);
+        recallService.setContentCategory(content);
         // 处理 content
         FilterParam filterParam = FilterParamFactory.create(param, content);
         filterParam.setBackup(true);
@@ -49,24 +48,4 @@ public class ColdStartBackupRecallStrategy implements RecallStrategy {
         return result;
     }
 
-    private void setContentCategory(List<Content> contentList) {
-        List<String> channelContentIds = contentList.stream().map(Content::getCrawlerChannelContentId).collect(Collectors.toList());
-        List<CrawlerMetaArticle> categoryList = getContentCategoryByChannelContentId(channelContentIds);
-        if (CollectionUtils.isEmpty(categoryList)) {
-            return;
-        }
-        Map<String, List<String>> categoryMap = categoryList.stream().collect(Collectors.groupingBy(CrawlerMetaArticle::getChannelContentId,
-                Collectors.mapping(CrawlerMetaArticle::getCategory, Collectors.toList())));
-        for (Content content : contentList) {
-            content.setCategory(categoryMap.get(content.getCrawlerChannelContentId()));
-        }
-    }
-
-    private List<CrawlerMetaArticle> getContentCategoryByChannelContentId(List<String> channelContentIds) {
-        if (CollectionUtils.isEmpty(channelContentIds)) {
-            return new ArrayList<>();
-        }
-        return crawlerMetaArticleRepository.getByChannelContentIdIn(channelContentIds);
-    }
-
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recall/strategy/DefaultRecallStrategy.java

@@ -2,8 +2,7 @@ package com.tzld.longarticle.recommend.server.service.recall.strategy;
 
 import com.tzld.longarticle.recommend.server.model.Content;
 import com.tzld.longarticle.recommend.server.remote.AIGCRemoteService;
-import com.tzld.longarticle.recommend.server.repository.aigc.CrawlerMetaArticleRepository;
-import com.tzld.longarticle.recommend.server.repository.entity.aigc.CrawlerMetaArticle;
+import com.tzld.longarticle.recommend.server.repository.mapper.crawler.CrawlerBaseMapper;
 import com.tzld.longarticle.recommend.server.service.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.filter.FilterService;
@@ -28,14 +27,16 @@ public class DefaultRecallStrategy implements RecallStrategy {
     @Autowired
     private AIGCRemoteService aigcRemoteService;
     @Autowired
-    CrawlerMetaArticleRepository crawlerMetaArticleRepository;
+    RecallService recallService;
 
     @Override
     public RecallResult.RecallData recall(RecallParam param) {
 
         List<Content> content = aigcRemoteService.getAllContent(param);
+        // 标题历史均值
+        recallService.setTitleAvgViewCount(content);
         // category 查询
-        setContentCategory(content);
+        recallService.setContentCategory(content);
         // 处理 content
         FilterParam filterParam = FilterParamFactory.create(param, content);
         FilterResult filterResult = filterService.filter(filterParam);
@@ -47,24 +48,4 @@ public class DefaultRecallStrategy implements RecallStrategy {
         return result;
     }
 
-    private void setContentCategory(List<Content> contentList) {
-        List<String> channelContentIds = contentList.stream().map(Content::getCrawlerChannelContentId).collect(Collectors.toList());
-        List<CrawlerMetaArticle> categoryList = getContentCategoryByChannelContentId(channelContentIds);
-        if (CollectionUtils.isEmpty(categoryList)) {
-            return;
-        }
-        Map<String, List<String>> categoryMap = categoryList.stream().collect(Collectors.groupingBy(CrawlerMetaArticle::getChannelContentId,
-                Collectors.mapping(CrawlerMetaArticle::getCategory, Collectors.toList())));
-        for (Content content : contentList) {
-            content.setCategory(categoryMap.get(content.getCrawlerChannelContentId()));
-        }
-    }
-
-    private List<CrawlerMetaArticle> getContentCategoryByChannelContentId(List<String> channelContentIds) {
-        if (CollectionUtils.isEmpty(channelContentIds)) {
-            return new ArrayList<>();
-        }
-        return crawlerMetaArticleRepository.getByChannelContentIdIn(channelContentIds);
-    }
-
 }