Browse Source

Merge branch 'feature/sunxiaoyi_recall' into feature/zhangbo_rank

# Conflicts:
#	recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Density.java
zhangbo 1 year ago
parent
commit
50ffc8c8b4

+ 3 - 1
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -28,6 +28,8 @@ public class RankRouter {
     private RankStrategy4RegionMergeModelV5 rankStrategy4RegionMergeModelV5;
     @Autowired
     private RankStrategy4RegionMergeModelV6 rankStrategy4RegionMergeModelV6;
+    @Autowired
+    private FestivalStrategy4RankModel festivalStrategy4RankModel;
 
     @Autowired
     private RankStrategyFlowThompsonModel rankStrategyFlowThompsonModel;
@@ -74,7 +76,7 @@ public class RankRouter {
                 return rankStrategy4RegionMergeModelV6.rank(param);
             case "60130":
                 // 先走默认排序,后续需要优化祝福类的视频排序
-                return rankService.rank(param);
+                return festivalStrategy4RankModel.rank(param);
             default:
                 break;
         }

+ 1 - 5
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -104,7 +104,7 @@ public class RankService {
         return mergeAndSort(param, rovRecallRank, flowPoolRank);
     }
 
-    public void rankFilter(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank){
+    public void rankFilter(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
         return;
     }
 
@@ -162,7 +162,6 @@ public class RankService {
                 || param.getAbCode().equals("60117")
                 || param.getAbCode().equals("60118")
                 || param.getAbCode().equals("60119")
-                || param.getAbCode().equals("60130")
         ) {
             // 地域召回要做截取,再做融合排序
             removeDuplicate(rovRecallRank);
@@ -180,9 +179,6 @@ public class RankService {
                 rovRecallRank.addAll(extractAndSort(param, FlowPoolLastDayTopRecallStrategy.PUSH_FORM));
                 rovRecallRank.addAll(extractAndSort(param, TopGoodPerformanceVideoRecallStrategy.PUSH_FORM));
             }
-            if (param.getAbCode().equals("60130")) {
-                rovRecallRank.addAll(extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM));
-            }
             removeDuplicate(rovRecallRank);
 
             // 融合排序

+ 44 - 0
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorBoost.java

@@ -0,0 +1,44 @@
+package com.tzld.piaoquan.recommend.server.service.rank.processor;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.math.BigDecimal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 提权处理器
+ *
+ * @author sunxy
+ */
+public class RankProcessorBoost {
+
+    public static void boostByTag(List<Video> rovList, Map<String, Map<String, String>> rulesMap) {
+        if (CollectionUtils.isEmpty(rovList) || rulesMap == null || rulesMap.isEmpty()) {
+            return;
+        }
+        Map<String, Double> densityRules = new HashMap<>();
+        for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+            String key = entry.getKey();
+            Map<String, String> value = entry.getValue();
+            if (value.containsKey("boost")) {
+                densityRules.put(key, Double.valueOf(value.get("boost")));
+            }
+        }
+
+
+        for (Map.Entry<String, Double> entry : densityRules.entrySet()) {
+            rovList.stream().filter(video -> video.getTags().contains(entry.getKey()))
+                    .forEach(video -> video.setSortScore(
+                            BigDecimal.valueOf(video.getSortScore())
+                                    .multiply(BigDecimal.valueOf(entry.getValue()))
+                                    .doubleValue()));
+        }
+        rovList.sort((o1, o2) -> Double.compare(o2.getSortScore(), o1.getSortScore()));
+
+    }
+
+
+}

+ 71 - 0
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorInsert.java

@@ -0,0 +1,71 @@
+package com.tzld.piaoquan.recommend.server.service.rank.processor;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.util.MathUtil;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author sunxy
+ */
+public class RankProcessorInsert {
+
+    public static void insertByTag(RankParam param, List<Video> rovList,
+                                   Map<String, Map<String, String>> rulesMap) {
+        if (CollectionUtils.isEmpty(rovList) || rulesMap == null || rulesMap.isEmpty()) {
+            return;
+        }
+        String insertRules = null, tagName = null;
+        for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+            Map<String, String> value = entry.getValue();
+            if (value.containsKey("insert")) {
+                insertRules = value.get("insert");
+                tagName = entry.getKey();
+                break;
+            }
+        }
+        if (StringUtils.isBlank(insertRules) || StringUtils.isBlank(tagName)) {
+            return;
+        }
+        final String finalTagName = tagName;
+
+        // 判断是否前几个已经有该tag的视频了
+        boolean isExistResultTagVideo = rovList.stream()
+                .limit(param.getSize())
+                .anyMatch(video -> video.getTags().contains(finalTagName));
+        if (isExistResultTagVideo) {
+            return;
+        }
+
+        // 获取需要插入的视频
+        Video insertTagVideo = rovList.stream()
+                .filter(video -> video.getTags().contains(finalTagName))
+                .findFirst().orElse(null);
+        if (insertTagVideo == null) {
+            return;
+        }
+
+        // 获取插入的权重
+        String[] insertWeight = insertRules.split(",");
+        List<Double> insertWeightList = Arrays.stream(insertWeight)
+                .map(Double::valueOf).collect(Collectors.toList());
+
+        // 遍历权重,随机插入
+        for (int i = 0; i < insertWeightList.size(); i++) {
+            double probability = insertWeightList.get(i);
+            double randomDouble = MathUtil.nextDouble(0, 1);
+            if (randomDouble <= probability) {
+                rovList.add(i, insertTagVideo);
+                break;
+            }
+        }
+
+    }
+
+}

+ 64 - 0
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/FestivalStrategy4RankModel.java

@@ -0,0 +1,64 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.FestivalRecallStrategyV1;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.ReturnVideoRecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.SimHotVideoRecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author sunxy
+ */
+@Service
+@Slf4j
+public class FestivalStrategy4RankModel extends RankService {
+
+    @Resource
+    private RankStrategy4Density rankStrategy4Density;
+
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        List<Video> rovRecallRank = new ArrayList<>();
+        rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM));
+        removeDuplicate(rovRecallRank);
+        // 融合排序
+        List<String> videoIdKeys = rovRecallRank.stream()
+                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+                .collect(Collectors.toList());
+        List<String> videoScores = redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("FestivalStrategy4RankModel mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+        if (CollectionUtils.isNotEmpty(videoScores)
+                && videoScores.size() == rovRecallRank.size()) {
+            for (int i = 0; i < videoScores.size(); i++) {
+                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+            }
+            // TODO 提权
+            Collections.sort(rovRecallRank,
+                    Comparator.comparingDouble(o -> -(o.getSortScore() * 0.4 * (o.getRovScore() / 100))));
+        }
+
+        return rovRecallRank;
+    }
+
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank,
+                                   List<Video> flowPoolRank) {
+        return rankStrategy4Density.mergeAndSort(param, rovRecallRank, flowPoolRank);
+    }
+}

+ 5 - 0
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Density.java

@@ -6,7 +6,10 @@ import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import lombok.extern.slf4j.Slf4j;
@@ -55,8 +58,10 @@ public class RankStrategy4Density extends RankService {
         }
 
         //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
 
         //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
 
         //7 流量池按比例强插
         List<Video> result = new ArrayList<>();

+ 43 - 0
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/DynamicGaussianFunction.java

@@ -0,0 +1,43 @@
+package com.tzld.piaoquan.recommend.server.service.score4recall.strategy;
+
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+
+/**
+ * 动态高斯函数
+ *
+ * @author sunxy
+ */
+public class DynamicGaussianFunction {
+    private static final double SIGMA = 24.0; // 标准差,可以根据需要调整这个值
+
+    public static double calculateValue(LocalDateTime testTime, LocalDateTime startTime, LocalDateTime endTime,
+                                        LocalDateTime peakTime) {
+        long testTimestamp = testTime.toEpochSecond(ZoneOffset.UTC) * 1000;
+        // 检查时间是否在指定范围内
+        if (testTimestamp < startTime.toEpochSecond(ZoneOffset.UTC) * 1000 ||
+                testTimestamp > endTime.toEpochSecond(ZoneOffset.UTC) * 1000) {
+            return 0;
+        }
+
+        // 计算 t_0,即从开始时间到峰值时间的小时数
+        double t_0 = (peakTime.toEpochSecond(ZoneOffset.UTC) - startTime.toEpochSecond(ZoneOffset.UTC)) / 3600.0;
+
+        // 计算时间差(以小时为单位)
+        double t = (testTimestamp / 1000.0 - startTime.toEpochSecond(ZoneOffset.UTC)) / 3600.0;
+
+        // 计算高斯函数值
+        return 100 * Math.exp(-Math.pow(t - t_0, 2) / (2 * Math.pow(SIGMA, 2)));
+    }
+
+    public static void main(String[] args) {
+        // 示例:计算2024-02-03 09:00的值
+        LocalDateTime startTime = LocalDateTime.of(2024, 1, 29, 0, 0);
+        LocalDateTime endTime = LocalDateTime.of(2024, 2, 3, 23, 59);
+        LocalDateTime peakTime = LocalDateTime.of(2024, 2, 3, 9, 0);
+        LocalDateTime testTime = LocalDateTime.of(2024, 1, 31, 22, 0);
+
+        double value = calculateValue(testTime, startTime, endTime, peakTime);
+        System.out.println("Function value at " + testTime + ": " + value);
+    }
+}

+ 75 - 15
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/FestivalRecallScore.java

@@ -3,23 +3,30 @@ package com.tzld.piaoquan.recommend.server.service.score4recall.strategy;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerConfigInfo;
 import com.tzld.piaoquan.recommend.server.service.score4recall.AbstractScorer4Recall;
 import com.tzld.piaoquan.recommend.server.service.score4recall.model4recall.Model4RecallKeyValue;
+import com.tzld.piaoquan.recommend.server.util.ListMerger;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
+import java.util.stream.Collectors;
 
 
 public class FestivalRecallScore extends AbstractScorer4Recall {
 
-    private static final Map<String, List<String>> FESTIVAL_TIME_MAP = new HashMap<String, List<String>>() {
+    private static final Map<String, String> DAILY_BLESSING_TIME_MAP = new HashMap<String, String>() {
+        {
+            put("晚安", "daily 21:00-24:00");
+            put("晚上好", "daily 18:00-20:00");
+            put("下午好", "daily 15:00-16:00");
+            put("中午好 ", "daily 11:00-13:00");
+            put("早上好", "daily 00:00-08:00");
+        }
+    };
+
+    private static final Map<String, List<String>> YEARLY_FESTIVAL_TIME_MAP = new HashMap<String, List<String>>() {
         {
-            put("晚安", Arrays.asList("daily 21:00-24:00"));
-            put("晚上好", Arrays.asList("daily 18:00-20:00"));
-            put("下午好", Arrays.asList("daily 15:00-16:00"));
-            put("中午好 ", Arrays.asList("daily 11:00-13:00"));
-            put("早上好", Arrays.asList("daily 00:00-08:00"));
             put("圣诞节", Arrays.asList("2024-12-20 00:00~2024-12-25 08:00", "2025-12-20 00:00~2025-12-25 08:00", "2026-12-20 " +
                     "00:00~2026-12-25 08:00"));
             put("平安夜", Arrays.asList("2024-12-19 00:00~2024-12-24 08:00", "2025-12-19 00:00~2025-12-24 08:00", "2026-12-19 " +
@@ -134,33 +141,86 @@ public class FestivalRecallScore extends AbstractScorer4Recall {
     }
 
     @Override
-    public List<Pair<Long, Double>> recall(Map<String, String> params){
+    public List<Pair<Long, Double>> recall(Map<String, String> params) {
         // 节假日、时效性,判断
         Model4RecallKeyValue model = (Model4RecallKeyValue) this.getModel();
         if (model == null || model.kv == null) {
             return new ArrayList<>();
         }
-        List<Pair<Long, Double>> result = new ArrayList<>();
         LocalDateTime now = LocalDateTime.now();
-        for (Map.Entry<String, List<String>> entry : FESTIVAL_TIME_MAP.entrySet()) {
+        // 节日祝福-每年
+        List<Pair<Long, Double>> yearResult = new ArrayList<>();
+        for (Map.Entry<String, List<String>> entry : YEARLY_FESTIVAL_TIME_MAP.entrySet()) {
             String festival = entry.getKey();
             List<String> timeRangeList = entry.getValue();
             if (isFestivalTime(now, timeRangeList)) {
+                Pair<LocalDateTime, LocalDateTime> startTimeAndEndTime = getStartTimeAndEndTime(timeRangeList.get(0));
+                if (startTimeAndEndTime == null) {
+                    continue;
+                }
+                // 节日峰值设置为结束时间的当天的9点
+                double weight = DynamicGaussianFunction.calculateValue(LocalDateTime.now(), startTimeAndEndTime.getLeft(),
+                        startTimeAndEndTime.getRight(), startTimeAndEndTime.getRight().withHour(9));
+
+                List<Pair<Long, Double>> festivalLists = model.kv.getOrDefault(festival, new ArrayList<>());
+                if (festivalLists.isEmpty()) {
+                    continue;
+                }
+                festivalLists = festivalLists.stream().map(pair -> Pair.of(pair.getLeft(), weight))
+                        .limit(Math.min(50, festivalLists.size()))
+                        .collect(Collectors.toList());
+                yearResult.addAll(festivalLists);
+            }
+        }
+        List<Pair<Long, Double>> dayResult = new ArrayList<>();
+        // 每日祝福-每天固定时间段
+        for (Map.Entry<String, String> entry : DAILY_BLESSING_TIME_MAP.entrySet()) {
+            String festival = entry.getKey();
+            String timeRange = entry.getValue();
+            if (isFestivalTime(now, Collections.singletonList(timeRange))) {
                 List<Pair<Long, Double>> festivalLists = model.kv.getOrDefault(festival, new ArrayList<>());
                 if (festivalLists.isEmpty()) {
                     continue;
                 }
-                festivalLists = festivalLists.subList(0, Math.min(100, festivalLists.size()));
-                result.addAll(festivalLists);
+                festivalLists = festivalLists.stream().map(pair -> Pair.of(pair.getLeft(), 0.0))
+                        .limit(Math.min(50, festivalLists.size()))
+                        .collect(Collectors.toList());
+                dayResult.addAll(festivalLists);
             }
         }
-        // 固定获取常规祝福类的小程序
+        // 常规祝福类的小程序-任意时间
+        List<Pair<Long, Double>> anyResult = new ArrayList<>();
         List<Pair<Long, Double>> festivalLists = model.kv.getOrDefault("祝福", new ArrayList<>());
         if (!festivalLists.isEmpty()) {
-            festivalLists = festivalLists.subList(0, Math.min(100, festivalLists.size()));
-            result.addAll(festivalLists);
+            festivalLists = festivalLists.stream().map(pair -> Pair.of(pair.getLeft(), 0.0))
+                    .limit(Math.min(50, festivalLists.size()))
+                    .collect(Collectors.toList());
+            anyResult.addAll(festivalLists);
+        }
+        return ListMerger.mergeLists(yearResult, dayResult, anyResult);
+    }
+
+    public Pair<LocalDateTime, LocalDateTime> getStartTimeAndEndTime(String timeRangeList) {
+        if (timeRangeList == null || timeRangeList.isEmpty()) {
+            return null;
+        }
+        // 时间格式 2024-12-20 00:00~2024-12-25 08:00
+        if (StringUtils.startsWith(timeRangeList, "daily")) {
+            // 判断是否是 daily 开头
+            return null;
+        } else {
+            String[] split = StringUtils.split(timeRangeList, "~");
+            if (split.length != 2) {
+                return null;
+            }
+            String startTime = split[0];
+            String endTime = split[1];
+            // 解析 startTime endTime
+            DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
+            LocalDateTime startLocalDateTime = LocalDateTime.parse(startTime, dateTimeFormatter);
+            LocalDateTime endLocalDateTime = LocalDateTime.parse(endTime, dateTimeFormatter);
+            return Pair.of(startLocalDateTime, endLocalDateTime);
         }
-        return result;
     }
 
     public boolean isFestivalTime(LocalDateTime now, List<String> timeRangeList) {

+ 43 - 0
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/ListMerger.java

@@ -0,0 +1,43 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * 合并多个List
+ *
+ * @author sunxy
+ */
+public class ListMerger {
+
+    public static <T> List<T> mergeLists(List<T> list1, List<T> list2, List<T> list3) {
+        List<T> result = new ArrayList<>();
+        int maxSize = Math.max(list1.size(), Math.max(list2.size(), list3.size()));
+
+        for (int i = 0; i < maxSize; i++) {
+            if (i < list1.size()) {
+                result.add(list1.get(i));
+            }
+            if (i < list2.size()) {
+                result.add(list2.get(i));
+            }
+            if (i < list3.size()) {
+                result.add(list3.get(i));
+            }
+        }
+
+        return result;
+    }
+
+    public static void main(String[] args) {
+        // 示例
+        List<Integer> list1 = Lists.newArrayList(1, 4, 7);
+        List<Integer> list2 = Lists.newArrayList(2, 5, 8, 10, 11);
+        List<Integer> list3 = Lists.newArrayList(3, 6, 9);
+
+        List<Integer> mergedList = mergeLists(list1, list2, list3);
+        System.out.println(mergedList);
+    }
+}

+ 17 - 0
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/MathUtil.java

@@ -0,0 +1,17 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import java.util.Random;
+
+/**
+ * @author sunxy
+ */
+public class MathUtil {
+
+    /**
+     * 生成max到min范围的浮点数
+     * */
+    public static double nextDouble(final double min, final double max) {
+        return min + ((max - min) * new Random().nextDouble());
+    }
+
+}