|
@@ -0,0 +1,212 @@
|
|
|
+package com.tzld.longarticle.recommend.server.service.rank.strategy;
|
|
|
+
|
|
|
+
|
|
|
+import com.tzld.longarticle.recommend.server.model.Content;
|
|
|
+import com.tzld.longarticle.recommend.server.model.ContentHisPublishArticle;
|
|
|
+import com.tzld.longarticle.recommend.server.service.AccountContentPoolConfigService;
|
|
|
+import com.tzld.longarticle.recommend.server.service.AccountIndexAvgViewCountService;
|
|
|
+import com.tzld.longarticle.recommend.server.service.rank.RankItem;
|
|
|
+import com.tzld.longarticle.recommend.server.service.rank.RankParam;
|
|
|
+import com.tzld.longarticle.recommend.server.service.rank.RankResult;
|
|
|
+import com.tzld.longarticle.recommend.server.service.rank.RankStrategy;
|
|
|
+import com.tzld.longarticle.recommend.server.service.score.ScoreParam;
|
|
|
+import com.tzld.longarticle.recommend.server.service.score.ScoreResult;
|
|
|
+import com.tzld.longarticle.recommend.server.service.score.ScoreService;
|
|
|
+import com.tzld.longarticle.recommend.server.service.score.strategy.*;
|
|
|
+import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
|
|
|
+import com.tzld.longarticle.recommend.server.util.JSONUtils;
|
|
|
+import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
|
|
|
+import com.tzld.longarticle.recommend.server.util.MathUtils;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.collections4.CollectionUtils;
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.stereotype.Service;
|
|
|
+
|
|
|
+import java.util.*;
|
|
|
+
|
|
|
+/**
|
|
|
+ * @author yangxiaohui
|
|
|
+ */
|
|
|
+@Service
|
|
|
+@Slf4j
|
|
|
+public class RankTestStrategy implements RankStrategy {
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private ScoreService scoreService;
|
|
|
+ @Autowired
|
|
|
+ AccountIndexAvgViewCountService accountIndexAvgViewCountService;
|
|
|
+ @Autowired
|
|
|
+ private AccountContentPoolConfigService accountContentPoolConfigService;
|
|
|
+
|
|
|
+ public RankResult rank(RankParam param) {
|
|
|
+
|
|
|
+ log.info("RankParam {}", JSONUtils.toJson(param));
|
|
|
+ ScoreResult scoreResult = scoreService.score(convertToScoreParam(param));
|
|
|
+ log.info("ScoreResult {}", JSONUtils.toJson(scoreResult));
|
|
|
+
|
|
|
+ Map<String, Map<String, Double>> scoreMap = scoreResult.getScoreMap();
|
|
|
+
|
|
|
+ double toutiaoAvgViewCount = accountIndexAvgViewCountService.getAvgReadCount(param.getGhId(), 1);
|
|
|
+ List<RankItem> items = CommonCollectionUtils.toList(param.getContents(), c -> {
|
|
|
+ RankItem item = new RankItem();
|
|
|
+ item.setContent(c);
|
|
|
+ item.setScoreMap(scoreMap.get(c.getId()));
|
|
|
+ double score = 2 * item.getScore(SimilarityStrategy.class.getSimpleName())
|
|
|
+ + item.getScore(ViewMultiplierStrategy.class.getSimpleName());
|
|
|
+ double showViewCountSum = 0D;
|
|
|
+ double avgViewCountSum = 0D;
|
|
|
+ for (ContentHisPublishArticle hisItem : item.getContent().getHisPublishArticleList()) {
|
|
|
+ if (hisItem.isInnerAccount() && hisItem.getShowViewCount() > 0 && hisItem.getAvgViewCount() > 0) {
|
|
|
+ showViewCountSum += hisItem.getShowViewCount();
|
|
|
+ avgViewCountSum += hisItem.getAvgViewCount();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ Map<String, Double> thisScoreMap = item.getScoreMap();
|
|
|
+ double viewCountRate = 0D; // 设置默认值
|
|
|
+ if (avgViewCountSum > 0) {
|
|
|
+ viewCountRate = showViewCountSum / avgViewCountSum;
|
|
|
+ }
|
|
|
+ thisScoreMap.put("showViewCountSum", showViewCountSum);
|
|
|
+ thisScoreMap.put("avgViewCountSum", avgViewCountSum);
|
|
|
+ thisScoreMap.put("viewCountRate", viewCountRate);
|
|
|
+ double viewCountRateW = MathUtils.sigmoid(avgViewCountSum, 0.0005, toutiaoAvgViewCount);
|
|
|
+ double viewCountRateScore = 0;
|
|
|
+ if (viewCountRate > 0) {
|
|
|
+ viewCountRateScore = (Math.min(viewCountRate, 5) - 1D) * viewCountRateW;
|
|
|
+ }
|
|
|
+ thisScoreMap.put("viewCountRateW", viewCountRateW);
|
|
|
+ thisScoreMap.put("viewCountRateScore", viewCountRateScore);
|
|
|
+ item.setScoreMap(thisScoreMap);
|
|
|
+ item.setScore(viewCountRateScore );
|
|
|
+
|
|
|
+ return item;
|
|
|
+ });
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ // 1 排序
|
|
|
+ Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
|
|
|
+
|
|
|
+ // 3 ITEM按照内容池分组
|
|
|
+ Map<String, List<RankItem>> itemMap = new HashMap<>();
|
|
|
+ for (RankItem item : items) {
|
|
|
+ List<RankItem> data = itemMap.computeIfAbsent(item.getContent().getContentPoolType(), k -> new ArrayList<>());
|
|
|
+ data.add(item);
|
|
|
+ }
|
|
|
+
|
|
|
+ log.info("SortResult {}", JSONUtils.toJson(items));
|
|
|
+ // 2 相似去重
|
|
|
+ List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
|
|
|
+ contents = deduplication(contents);
|
|
|
+ log.info("Deduplication {}", JSONUtils.toJson(contents));
|
|
|
+
|
|
|
+ // 3 文章按照内容池分组
|
|
|
+ Map<String, List<Content>> contentMap = new HashMap<>();
|
|
|
+ for (Content c : contents) {
|
|
|
+ List<Content> data = contentMap.computeIfAbsent(c.getContentPoolType(), k -> new ArrayList<>());
|
|
|
+ data.add(c);
|
|
|
+ }
|
|
|
+ log.info("ContentMap {}", JSONUtils.toJson(contentMap));
|
|
|
+ // 4 选文章
|
|
|
+ List<Content> result = new ArrayList<>();
|
|
|
+ String[] contentPools = accountContentPoolConfigService.getContentPools(param.getAccountName());
|
|
|
+
|
|
|
+
|
|
|
+ // 头、次
|
|
|
+ if (StringUtils.equals(contentPools[0], contentPools[1])) {
|
|
|
+ List<Content> pool = contentMap.get(contentPools[0]);
|
|
|
+ Integer level = accountContentPoolConfigService.getLevelByContentPool(contentPools[0]);
|
|
|
+ if (level == 1) {
|
|
|
+ if (CollectionUtils.isNotEmpty(pool)) {
|
|
|
+ result.add(pool.get(0));
|
|
|
+ if (pool.size() > 2) {
|
|
|
+ result.add(pool.get(2));
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // level2 兜底
|
|
|
+ pool = contentMap.get(accountContentPoolConfigService.getContentPoolByLevel(2));
|
|
|
+ if (CollectionUtils.isNotEmpty(pool)) {
|
|
|
+ result.add(pool.get(0));
|
|
|
+ if (pool.size() > 1) {
|
|
|
+ result.add(pool.get(1));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else if (level == 2) {
|
|
|
+ if (CollectionUtils.isNotEmpty(pool)) {
|
|
|
+ pool = pool.subList(0, Math.max(10, pool.size()));
|
|
|
+ Collections.shuffle(pool);
|
|
|
+ result.add(pool.get(0));
|
|
|
+ if (pool.size() > 1) {
|
|
|
+ result.add(pool.get(1));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // 配置错误 兜底
|
|
|
+ List<Content> pool1 = contentMap.get(contentPools[0]);
|
|
|
+ List<Content> pool2 = contentMap.get(contentPools[1]);
|
|
|
+ if (CollectionUtils.isNotEmpty(pool1)) {
|
|
|
+ result.add(pool1.get(0));
|
|
|
+ if (CollectionUtils.isNotEmpty(pool2)) {
|
|
|
+ result.add(pool2.get(0));
|
|
|
+ }
|
|
|
+ } else if (CollectionUtils.isNotEmpty(pool2)) {
|
|
|
+ result.add(pool2.get(0));
|
|
|
+ if (pool2.size() > 1) {
|
|
|
+ result.add(pool2.get(1));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 3-8
|
|
|
+ List<Content> pool = contentMap.get(contentPools[2]);
|
|
|
+ if (CollectionUtils.isNotEmpty(pool)) {
|
|
|
+ Collections.shuffle(pool);
|
|
|
+ result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
|
|
|
+ }
|
|
|
+
|
|
|
+ if (result.size() < param.getSize()) {
|
|
|
+ // 兜底
|
|
|
+ pool = param.getBackup();
|
|
|
+ pool = deduplication(pool);
|
|
|
+ log.info("Backup Deduplication {}", JSONUtils.toJson(pool));
|
|
|
+ Collections.shuffle(pool);
|
|
|
+ result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
|
|
|
+ }
|
|
|
+
|
|
|
+ return new RankResult(result);
|
|
|
+ }
|
|
|
+
|
|
|
+ private ScoreParam convertToScoreParam(RankParam param) {
|
|
|
+ ScoreParam scoreParam = new ScoreParam();
|
|
|
+ scoreParam.setGhId(param.getGhId());
|
|
|
+ scoreParam.setAccountName(param.getAccountName());
|
|
|
+ scoreParam.setContents(param.getContents());
|
|
|
+ scoreParam.setStrategy(param.getStrategy());
|
|
|
+ return scoreParam;
|
|
|
+ }
|
|
|
+
|
|
|
+ private List<Content> deduplication(List<Content> contents) {
|
|
|
+ List<String> titles = new ArrayList<>();
|
|
|
+ List<Content> result = new ArrayList<>();
|
|
|
+ // 遍历所有列表
|
|
|
+ for (Content c : contents) {
|
|
|
+ if (similarity(c.getTitle(), titles)) {
|
|
|
+ continue;
|
|
|
+ } else {
|
|
|
+ result.add(c);
|
|
|
+ titles.add(c.getTitle());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ private boolean similarity(String title, List<String> titles) {
|
|
|
+ return TitleSimilarCheckUtil.isDuplicateContent(title, titles);
|
|
|
+ }
|
|
|
+
|
|
|
+}
|