Merge branch 'feature/zhangbo_rank' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/VideoRecommendService.java

@@ -14,7 +14,7 @@ import com.tzld.piaoquan.recommend.server.model.RecommendParam;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.strategy.RankStrategy4RegionMergeModelV2;
+import com.tzld.piaoquan.recommend.server.service.rank.strategy.RankStrategy4RegionMergeModelV547;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -45,7 +45,7 @@ public class VideoRecommendService {
     @Resource
     private FlowPoolRecommendPipeline flowPoolRecommendPipeline;
     @Resource
-    private RankStrategy4RegionMergeModelV2 rankStrategy4RegionMergeModelV2;
+    private RankStrategy4RegionMergeModelV547 rankStrategy4RegionMergeModelV2;
     @Autowired
     @Qualifier("redisTemplate")
     private RedisTemplate<String, String> redisTemplate;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -21,12 +21,12 @@ public class RankRouter {
     @Autowired
     private RankStrategy4RegionMergeModelV536 rankStrategy4RegionMergeModelV536;
     @Autowired
-    private RankStrategy4RegionMergeModelV2 rankStrategy4RegionMergeModelV2;
-    @Autowired
     private RankStrategy4RegionMergeModelV3 rankStrategy4RegionMergeModelV3;
     @Autowired
     private RankStrategy4RegionMergeModelV4 rankStrategy4RegionMergeModelV4;
     @Autowired
+    private RankStrategy4RegionMergeModelV546 rankStrategy4RegionMergeModelV546;
+    @Autowired
     private RankStrategy4RegionMergeModelV547 rankStrategy4RegionMergeModelV547;
     @Autowired
     private RankStrategy4RegionMergeModelV548 rankStrategy4RegionMergeModelV548;
@@ -70,11 +70,11 @@ public class RankRouter {
             case "60121": // 536
                 return rankStrategy4RegionMergeModelV536.rank(param);
             case "60122": // 537
-                return rankStrategy4RegionMergeModelV2.rank(param);
+                return rankStrategy4RegionMergeModelV546.rank(param);
             case "60123": // 541
                 return rankStrategy4RegionMergeModelV3.rank(param);
             case "60124": // 546
-                return rankStrategy4RegionMergeModelV2.rank(param);
+                return rankStrategy4RegionMergeModelV546.rank(param);
             case "60125": // 547
                 return rankStrategy4RegionMergeModelV547.rank(param);
             case "60126": // 548

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV2.java → recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV546.java

@@ -1,17 +1,10 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
-import java.util.stream.Collectors;
-import java.util.Arrays;
-import java.util.Map;
-
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
@@ -23,9 +16,7 @@ import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBo
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
@@ -38,7 +29,6 @@ import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 
 import java.text.SimpleDateFormat;
 import java.util.*;
@@ -50,63 +40,13 @@ import java.util.stream.Collectors;
  */
 @Service
 @Slf4j
-public class RankStrategy4RegionMergeModelV2 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv2:}")
+public class RankStrategy4RegionMergeModelV546 extends RankService {
+    @ApolloJsonValue("${rank.score.merge.weightv546:}")
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
     private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
 
-    @Override
-    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
-        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
-        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
-            return quickFlowPoolVideos;
-        } else {
-            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
-        }
-    }
-
-    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
-
-        //初始化 userid
-        UserFeature userFeature = new UserFeature();
-        userFeature.setMid(param.getMid());
-
-        // 初始化RankItem
-        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
-                .filter(d -> d.getPushFrom().equals(pushFrom))
-                .findFirst();
-        if (!data.isPresent()){
-            return Collections.emptyList();
-        }
-        List<Video> videoList = data.get().getVideos();
-        if (videoList == null) {
-            return Collections.emptyList();
-        }
-        List<RankItem> rankItems = new ArrayList<>();
-        for (int i = 0; i < videoList.size(); i++) {
-            RankItem rankItem = new RankItem(videoList.get(i));
-            rankItems.add(rankItem);
-        }
-
-        // 初始化上下文参数
-        ScoreParam scoreParam = convert(param);
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
-                .scoring(scoreParam, userFeature, rankItems);
-
-        if (rovRecallScore == null) {
-            return Collections.emptyList();
-        }
-
-        return CommonCollectionUtils.toList(rovRecallScore, i -> {
-            // hard code 将排序分数 赋值给video的sortScore
-            Video v = i.getVideo();
-            v.setSortScore(i.getScore());
-            return v;
-        });
-    }
-
     public void duplicate(Set<Long> setVideo, List<Video> videos) {
         Iterator<Video> iterator = videos.iterator();
         while (iterator.hasNext()) {
@@ -169,7 +109,6 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
         rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
-
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
@@ -202,36 +141,36 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
             List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
             List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
             List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
             List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
 
-            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeight(share2return);
-            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
+            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
             List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeight(view2play);
+            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
             List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeight(play2share);
+            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
             item.scoresMap.put("share2returnScore", share2returnScore);
             item.scoresMap.put("view2returnScore", view2returnScore);
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
             // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(returns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeight(share2allreturn);
-            List<Double> view2allreturn = getRateData(returns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeight(view2allreturn);
+            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
+            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
+            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
+            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
             item.scoresMap.put("share2allreturnScore", share2allreturnScore);
             item.scoresMap.put("view2allreturnScore", view2allreturnScore);
 
             // 全部回流
-            Double allreturnsScore = calScoreWeight(allreturns);
+            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
 
             // 平台回流
-            Double preturnsScore = calScoreWeight(returns);
+            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
             item.scoresMap.put("preturnsScore", preturnsScore);
 
             // rov的趋势
@@ -247,12 +186,12 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         List<Video> result = new ArrayList<>();
         double a = mergeWeight.getOrDefault("a", 0.1);
         double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double c = mergeWeight.getOrDefault("c", 0.000001);
         double d = mergeWeight.getOrDefault("d", 1.0);
         double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
-        double h = mergeWeight.getOrDefault("h", 20.0);
+        double f = mergeWeight.getOrDefault("f", 0.8);
+        double g = mergeWeight.getOrDefault("g", 2.0);
+        double h = mergeWeight.getOrDefault("h", 240.0);
         double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
         for (RankItem item : items) {
             double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
@@ -315,12 +254,12 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         return sum;
     }
 
-    public Double calScoreWeight(List<Double> data) {
+    public Double calScoreWeightNoTimeDecay(List<Double> data) {
         Double up = 0.0;
         Double down = 0.0;
         for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 / (i + 1) * data.get(i);
-            down += 1.0 / (i + 1);
+            up += 1.0 * data.get(i);
+            down += 1.0;
         }
         return down > 1E-8 ? up / down : 0.0;
     }
@@ -552,7 +491,6 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         }
 
 
-
         List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
         return rovRecallScore;
@@ -683,14 +621,14 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
         String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
 
-        RankStrategy4RegionMergeModelV2 job = new RankStrategy4RegionMergeModelV2();
+        RankStrategy4RegionMergeModelV547 job = new RankStrategy4RegionMergeModelV547();
         List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeight(l1);
+        Double d1 = job.calScoreWeightNoTimeDecay(l1);
 
         System.out.println(d1);
 
         List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeight(l2);
+        Double d2 = job.calScoreWeightNoTimeDecay(l2);
 
         System.out.println(d2);
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -141,7 +141,7 @@ public class RecallService implements ApplicationContextAware {
             }else{
                 if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL)) {
                     strategies.add(strategyMap.get(QuickFlowPoolWithLevelRecallStrategy.class.getSimpleName()));
-                    if ("60126".equals(abCode) || "60125".equals(abCode)){
+                    if ("60126".equals(abCode) || "60125".equals(abCode) || "60124".equals(abCode)){
                         strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategyTomson.class.getSimpleName()));
                     }else {
                         strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategy.class.getSimpleName()));
@@ -154,7 +154,7 @@ public class RecallService implements ApplicationContextAware {
                     strategies.add(strategyMap.get(FlowPoolWithScoreRecallStrategy.class.getSimpleName()));
                 }
             }
-//            if ("60126".equals(abCode) || "60125".equals(abCode)){
+//            if ("60126".equals(abCode) || "60125".equals(abCode) || "60124".equals(abCode)){
 //                strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategyTomsonFilterDigit.class.getSimpleName()));
 //            }else{
 //                int lastDigit = param.getLastDigit();