Merge branch 'master' into addfeature/20240520/sunxy/addPushFromRank

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4ShareDeepAndWidth.java

@@ -5,20 +5,11 @@ import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.springframework.stereotype.Service;
 
 import java.text.SimpleDateFormat;
@@ -31,25 +22,11 @@ import java.util.stream.Collectors;
  */
 @Service
 @Slf4j
-public class RankStrategy4ShareDeepAndWidth extends RankService {
+public class RankStrategy4ShareDeepAndWidth extends RankStrategy4RegionMergeModelBasic {
     @ApolloJsonValue("${rank.score.merge.weightv645:}")
     private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private final Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
 
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
         Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
@@ -58,6 +35,9 @@ public class RankStrategy4ShareDeepAndWidth extends RankService {
         //-------------------逻-------------------
         //-------------------辑-------------------
 
+        List<Video> rovRecallRank = new ArrayList<>();
+        Set<Long> setVideo = new HashSet<>();
+        //-------------------老地域召回-------------------
         List<Video> oldRovs = new ArrayList<>();
         oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
@@ -69,33 +49,44 @@ public class RankStrategy4ShareDeepAndWidth extends RankService {
         List<Video> v0 = oldRovs.size() <= sizeReturn
                 ? oldRovs
                 : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v0);
-
-
-        //-------------------相关性召回 融合+去重-------------------
+        rovRecallRank.addAll(v0);
+        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------sim相似召回------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        v5 = v5.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v5 = v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size()));
+        rovRecallRank.addAll(v5);
+        setVideo.addAll(v5.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------return相似召回------------------
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------分享深度、广度 融合+去重-------------------
-        List<Video> v9 = extractAndSort(param, ShareWidthRecallStrategy.PUSH_FORM);
-        List<Video> v10 = extractAndSort(param, ShareDeepRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v9);
-        //-------------------地域相关召回 融合+去重-------------------
+        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(v6);
+        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回------------------
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        //-------------------节日扶持召回 融合+去重-------------------
+        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        rovRecallRank.addAll(v1);
+        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------节日特殊召回-------------------
         List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(v0);
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 5.0).intValue(), v9.size())));
-        rovRecallRank.addAll(v10.subList(0, Math.min(mergeWeight.getOrDefault("v10", 5.0).intValue(), v10.size())));
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size())));
+        v7 = v7.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v7 = v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size()));
+        rovRecallRank.addAll(v7);
+        setVideo.addAll(v7.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------基于分享宽度 召回-------------------
+        List<Video> v8 = extractAndSort(param, ShareWidthRecallStrategy.PUSH_FORM);
+        v8 = v8.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v8 = v8.subList(0, Math.min(mergeWeight.getOrDefault("v8", 5.0).intValue(), v8.size()));
+        rovRecallRank.addAll(v8);
+        setVideo.addAll(v8.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------基于分享深度 召回-------------------
+        List<Video> v9 = extractAndSort(param, ShareDeepRecallStrategy.PUSH_FORM);
+        v9 = v9.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v9 = v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 5.0).intValue(), v9.size()));
+        rovRecallRank.addAll(v9);
+        setVideo.addAll(v9.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -141,7 +132,9 @@ public class RankStrategy4ShareDeepAndWidth extends RankService {
             Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
 
-            Double sumShareCount = shares.stream().mapToDouble(Double::doubleValue).sum();
+
+            Map<String, Double> shareUvList1h = itemRealMap.getOrDefault("share_uv_list_1h", new HashMap<>());
+            Double sumShareCount = shareUvList1h.values().stream().mapToDouble(Double::doubleValue).sum();
             item.scoresMap.put("sumShareCount", sumShareCount);
 
 
@@ -183,41 +176,6 @@ public class RankStrategy4ShareDeepAndWidth extends RankService {
         return result;
     }
 
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
-    }
-
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
 
     public List<RankItem> model(List<Video> videos, RankParam param,
                                 List<String> rtFeaPart) {
@@ -333,130 +291,6 @@ public class RankStrategy4ShareDeepAndWidth extends RankService {
         return rankItems;
     }
 
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
     public static void main(String[] args) {
-//        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
-        String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
-        String down1 = "2024031012:2019,2024031013:1676,2024031014:1626,2024031015:1458,2024031016:1508,2024031017:1510,2024031018:1713,2024031019:1972,2024031020:2500,2024031021:2348,2024031022:2061,2024031023:1253,2024031100:659,2024031101:243,2024031102:191,2024031103:282,2024031104:246,2024031105:439,2024031106:1079,2024031107:1911,2024031108:2023,2024031109:1432,2024031110:1632,2024031111:1183,2024031112:1024,2024031113:938,2024031114:701,2024031115:541";
-
-//        String up2 = "2024031012:215,2024031013:242,2024031014:166,2024031015:194,2024031016:209,2024031017:245,2024031018:320,2024031019:332,2024031020:400,2024031021:375,2024031022:636,2024031023:316,2024031100:167,2024031101:45,2024031102:22,2024031103:26,2024031104:12,2024031105:22,2024031106:24,2024031107:143,2024031108:181,2024031109:199,2024031110:194,2024031111:330,2024031112:423,2024031113:421,2024031114:497,2024031115:424";
-        String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
-        String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
-
-        RankStrategy4RegionMergeModelV567 job = new RankStrategy4RegionMergeModelV567();
-        List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeightNoTimeDecay(l1);
-
-        System.out.println(d1);
-
-        List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeightNoTimeDecay(l2);
-
-        System.out.println(d2);
-
-    }
-
-    List<Double> help(String s, String date, Integer h) {
-        Map<String, Double> maps = Arrays.stream(s.split(",")).map(pair -> pair.split(":"))
-                .collect(Collectors.toMap(
-                        arr -> arr[0],
-                        arr -> Double.valueOf(arr[1])
-                ));
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        List<Double> result = new ArrayList<>();
-        for (int i = 0; i < h; ++i) {
-            Double d = (result.isEmpty() ? 0.0 : result.get(result.size() - 1));
-            result.add(d + maps.getOrDefault(date, 0D));
-            datehours.add(date);
-            date = ExtractorUtils.subtractHours(date, 1);
-        }
-        return result;
     }
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -159,18 +159,6 @@ public class RecallService implements ApplicationContextAware {
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
                 break;
-            case "60150": // 645
-                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
-                strategies.add(strategyMap.get(ShareWidthRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(ShareDeepRecallStrategy.class.getSimpleName()));
-                break;
-            case "60151": // 650
-                strategies.addAll(getRegionRecallStrategy(param));
-                strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(ContentBaseRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));
-                break;
             case "60117": // 567
             case "60118": // 568
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
@@ -298,14 +286,6 @@ public class RecallService implements ApplicationContextAware {
             case "60115": // 565
             case "60117": // 567
             case "60118": // 568
-            case "60150": // 645
-                if (!hitUserBlacklist || !isInBlacklist) {
-                    strategies.add(strategyMap.get(FlowPoolLastDayTopRecallStrategy.class.getSimpleName()));
-                }
-                strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));
-                break;
             case "60104": // 去掉sim的对比实验
                 strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                 break;
@@ -324,6 +304,21 @@ public class RecallService implements ApplicationContextAware {
                 strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                 strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));
                 break;
+            case "60150": // 645
+                strategies.add(strategyMap.get(ShareWidthRecallStrategy.class.getSimpleName()));
+                strategies.add(strategyMap.get(ShareDeepRecallStrategy.class.getSimpleName()));
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
+                strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
+                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));
+                break;
+            case "60151": // 650
+                strategies.add(strategyMap.get(ContentBaseRecallStrategy.class.getSimpleName()));
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
+                strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
+                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));
+                break;
             case "60654":
                 strategies.add(strategyMap.get(CFRovnRecallStrategyV1.class.getSimpleName()));
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));