Merge branch 'feature/vlog_merge_refactor_smz_zhangbo' of algorithm/recommend-server into vlog_merge_refactor_smz

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -77,7 +77,7 @@ public class RankRouter {
             case "60123": // 541
                 return rankStrategy4RegionMergeModelV3.rank(param);
             case "60124": // 546
-                return rankStrategy4RegionMergeModelV4.rank(param);
+                return rankStrategy4RegionMergeModelV2.rank(param);
             case "60125": // 547
                 return rankStrategy4RegionMergeModelV2.rank(param);
             case "60126": // 548

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -88,17 +88,12 @@ public class RankService {
 
         // 2 正常走分发 排序+冷启动
         List<Video> rovRecallRank = mergeAndRankRovRecall(param);
-        // log.info("mergeAndRankRovRecall rovRecallRank={}", JSONUtils.toJson(rovRecallRank));
         List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
-        // log.info("mergeAndRankFlowPoolRecall flowPoolRank={}", JSONUtils.toJson(flowPoolRank));
 
         rankFilter(param, rovRecallRank, flowPoolRank);
 
         removeDuplicate(param, rovRecallRank, flowPoolRank);
 
-//        log.info("removeDuplicate rovRecallRank={}, flowPoolRank={}",
-//                JSONUtils.toJson(rovRecallRank),
-//                JSONUtils.toJson(flowPoolRank));
 
         // 融合排序
         return mergeAndSort(param, rovRecallRank, flowPoolRank);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorDensity.java

@@ -71,7 +71,7 @@ public class RankProcessorDensity {
             String push = pushes.get(j);
             List<Video> candidate;
             if (!push.isEmpty()) {
-                // 5 如果是flow的video 取不到 不做替换
+                // 5 如果是flow的video0 取不到 不做替换
                 candidate = flow;
             } else {
                 // 5 如果是rov的video  取不到 不做替换

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV6.java

@@ -4,20 +4,24 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeatureV2;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeatureV2;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
@@ -34,9 +38,10 @@ import org.springframework.stereotype.Service;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
+
 /**
  * @author zhangbo
- * @desc 地域召回融合
+ * @desc 地域召回融合 流量池汤姆森
  */
 @Service
 @Slf4j
@@ -44,8 +49,9 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
     @ApolloJsonValue("${rank.score.merge.weightv6:}")
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private final Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
+    private final Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
+
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
         while(iterator.hasNext()){
@@ -59,53 +65,64 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
     }
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        List<Video> oldRovs = new ArrayList<>();
+        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        int sizeReturn = param.getSize();
+        removeDuplicate(oldRovs);
+        oldRovs = oldRovs.size() <= sizeReturn
+                ? oldRovs
+                : oldRovs.subList(0, sizeReturn);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, oldRovs);
+
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1_default.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3_default.PUSH_FORM);
         List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v1);
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
+        //-------------------节日扶持召回 融合+去重-------------------
         List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
         this.duplicate(setVideo, v7);
 
+        rovRecallRank.addAll(oldRovs);
         rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
         rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
         rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 0.0).intValue(), v4.size())));
         rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
         rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
         rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
+
+
+
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
-//        List<String> videoIdKeys = rovRecallRank.stream()
-//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-//                .collect(Collectors.toList());
-//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-//                JSONUtils.toJson(videoScores));
-//        if (CollectionUtils.isNotEmpty(videoScores)
-//                && videoScores.size() == rovRecallRank.size()) {
-//            for (int i = 0; i < videoScores.size(); i++) {
-//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-//            }
-//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        }
+
         // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<RankItem> items = model(rovRecallRank, param);
         List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
         List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
         Calendar calendar = Calendar.getInstance();
@@ -119,7 +136,7 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         }
         // 2 统计分
         String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
         for (int i=0; i<24; ++i){
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
@@ -146,9 +163,22 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流的rov和ros
+            List<Double> share2allreturn = getRateData(returns, shares, 1.0, 10.0);
+            Double share2allreturnScore = calScoreWeight(share2allreturn);
+            List<Double> view2allreturn = getRateData(returns, views, 0.0, 0.0);
+            Double view2allreturnScore = calScoreWeight(view2allreturn);
+            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
+            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
+
+            // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
 
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
             // rov的趋势
             double trendScore = calTrendScore(view2return);
             item.scoresMap.put("trendScore", trendScore);
@@ -160,22 +190,45 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
-        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
-        double beta = this.mergeWeight.getOrDefault("beta", 1.0);
+        double a = mergeWeight.getOrDefault("a", 0.1);
+        double b = mergeWeight.getOrDefault("b", 0.0);
+        double bb = mergeWeight.getOrDefault("bb", 0.005);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double f = mergeWeight.getOrDefault("f", 0.1);
+        double g = mergeWeight.getOrDefault("g", 1.0);
+        double h = mergeWeight.getOrDefault("h", 20.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
         for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 0.0 ?
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
-                    + alpha * trendScore
-                    + beta * newVideoScore
-                    ;
+            double strScore = item.getScoreStr();
+            double rosScoreModel = item.getScoreRos();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
+            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScore + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+
+            }
+            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
+            if (allreturnsScore > h){
+                score += (bb * rosScoreModel + f * share2allreturnScore + g * view2allreturnScore);
+            }
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
             video.setScoreStr(item.getScoreStr());
+            video.setScoreRos(item.getScoreRos());
             video.setScoresMap(item.getScoresMap());
             result.add(video);
         }
@@ -184,10 +237,10 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
     }
     public double calNewVideoScore(Map<String, String> itemBasicMap){
         double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 8){
+        if (existenceDays > 5){
             return 0.0;
         }
-        double score = 1.0 / (existenceDays + 5.0);
+        double score = 1.0 / (existenceDays + 10.0);
         return score;
     }
     public double calTrendScore(List<Double> data){
@@ -207,7 +260,6 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         }
         return sum;
     }
-
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -220,9 +272,13 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
     public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
         List<Double> data = new LinkedList<>();
         for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
+            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)){
+                data.add(0.0);
+            }else{
+                data.add(
+                        (ups.get(i) + up) / (downs.get(i) + down)
+                );
+            }
         }
         return data;
     }
@@ -237,9 +293,7 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         }
         return views;
     }
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart){
+    public List<RankItem> model(List<Video> videos, RankParam param){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -275,14 +329,13 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                 JSONObject obj = new JSONObject();
                 obj.put("name", "user_key_in_model_is_null");
                 obj.put("class", this.CLASS_NAME);
-                log.info(obj.toString());
-//                return videos;
             }
         }
         final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
                 "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
                 "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
         ));
         Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
         while (iterator.hasNext()) {
@@ -291,27 +344,29 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                 iterator.remove();
             }
         }
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+        Map<String, String> f1 = RankExtractorUserFeatureV2.getOriginFeature(userFeatureMap,
                 new HashSet<String>(Arrays.asList(
                         "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
                 ))
         );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+        Map<String, Double> f2__ = RankExtractorUserFeatureV2.getUserRateFeature(userFeatureMap);
+        Map<String, String> f2 = RankExtractorUserFeatureV2.rateFeatureChange(f2__);
+        Map<String, String> f3 = RankExtractorUserFeatureV2.cntFeatureChange(userFeatureMap,
                 new HashSet<String>(Arrays.asList(
                         "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                        "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
                 ))
         );
         f1.putAll(f2);
         f1.putAll(f3);
-//        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
                 "total_time", "play_count_total",
                 "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"
         ));
 
         List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
@@ -337,12 +392,14 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                             iteratorIn.remove();
                         }
                     }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                    Map<String, Double> f4__ = RankExtractorItemFeatureV2.getItemRateFeature(vfMap);
+                    Map<String, String> f4 = RankExtractorItemFeatureV2.rateFeatureChange(f4__);
+                    Map<String, String> f5 = RankExtractorItemFeatureV2.cntFeatureChange(vfMap,
                             new HashSet<String>(Arrays.asList(
                                     "total_time", "play_count_total",
                                     "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                                    "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"))
                     );
                     f4.putAll(f5);
                     rankItems.get(i).setFeatureMap(f4);
@@ -375,7 +432,6 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         videoRtKeys1.addAll(videoRtKeys2);
         List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
 
-
         if (videoRtFeatures != null){
             int j = 0;
             for (RankItem item: rankItems){
@@ -404,7 +460,8 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                 }catch (Exception e){
                     log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
                 item.getFeatureMap().putAll(f8);
             }
             for (RankItem item: rankItems){
@@ -435,26 +492,19 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                 }catch (Exception e){
                     log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
                 item.getFeatureMap().putAll(f8);
             }
         }
 
-
-//        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
-
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline("feeds_score_config_20240228.conf")
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-//        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
         JSONObject obj = new JSONObject();
         obj.put("name", "user_key_in_model_is_not_null");
         obj.put("class", this.CLASS_NAME);
-        log.info(obj.toString());
         return rovRecallScore;
     }
-
     private Map<String, String> getSceneFeature(RankParam param) {
         Map<String, String> sceneFeatureMap = new HashMap<>();
         String provinceCn = param.getProvince();
@@ -483,7 +533,6 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
 
         return sceneFeatureMap;
     }
-
     @Override
     public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -96,6 +96,7 @@ public class RecallService implements ApplicationContextAware {
         } else {
             switch (abCode) {
                 case "60121": // 536
+                case "60126": // 548
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1_default.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3_default.class.getSimpleName()));
@@ -103,6 +104,7 @@ public class RecallService implements ApplicationContextAware {
                     strategies.addAll(getRegionRecallStrategy(param));
                     break;
                 case "60122": // 537
+                case "60124": // 546
                 case "60125": // 547
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
@@ -112,8 +114,6 @@ public class RecallService implements ApplicationContextAware {
                     break;
                 case "60120": // 576
                 case "60123": // 541
-                case "60124": // 546
-                case "60126": // 548
                 case "60112": // 562
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
@@ -126,10 +126,18 @@ public class RecallService implements ApplicationContextAware {
             //2:通过“流量池标记”控制“流量池召回子策略” 其中有9组会走EXPERIMENTAL_FLOW_SET_LEVEL 有1组会走EXPERIMENTAL_FLOW_SET_LEVEL_SCORE
             if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL)) {
                 strategies.add(strategyMap.get(QuickFlowPoolWithLevelRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategy.class.getSimpleName()));
+                if ("60126".equals(abCode)){
+                    strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategyTomson.class.getSimpleName()));
+                }else {
+                    strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategy.class.getSimpleName()));
+                }
             } else if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL_SCORE)) {
                 strategies.add(strategyMap.get(QuickFlowPoolWithLevelScoreRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(FlowPoolWithLevelScoreRecallStrategy.class.getSimpleName()));
+                if ("60126".equals(abCode)){
+                    strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategyTomson.class.getSimpleName()));
+                }else {
+                    strategies.add(strategyMap.get(FlowPoolWithLevelScoreRecallStrategy.class.getSimpleName()));
+                }
             } else {
                 strategies.add(strategyMap.get(QuickFlowPoolWithScoreRecallStrategy.class.getSimpleName()));
                 strategies.add(strategyMap.get(FlowPoolWithScoreRecallStrategy.class.getSimpleName()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithLevelRecallStrategyTomson.java

@@ -0,0 +1,163 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConfigService;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants.KEY_WITH_LEVEL_FORMAT;
+
+/**
+ * @author zhangbo
+ */
+@Service
+@Slf4j
+public class FlowPoolWithLevelRecallStrategyTomson extends AbstractFlowPoolWithLevelRecallStrategy {
+
+    @Autowired
+    private FlowPoolConfigService flowPoolConfigService;
+
+    @Override
+    Pair<String, String> flowPoolKeyAndLevel(RecallParam param) {
+        //# 1. 获取流量池各层级分发概率权重
+        Map<String, Double> levelWeightMap = flowPoolConfigService.getLevelWeight();
+
+        // 2. 判断各层级是否有视频需分发
+        List<LevelWeight> availableLevels = new ArrayList<>();
+        for (Map.Entry<String, Double> entry : levelWeightMap.entrySet()) {
+            String levelKey = String.format(KEY_WITH_LEVEL_FORMAT, param.getAppType(), entry.getKey());
+            if (redisTemplate.hasKey(levelKey)) {
+                LevelWeight lw = new LevelWeight();
+                lw.setLevel(entry.getKey());
+                lw.setLevelKey(levelKey);
+                lw.setWeight(entry.getValue());
+                availableLevels.add(lw);
+            }
+        }
+        if (CollectionUtils.isEmpty(availableLevels)) {
+            return Pair.of("", "");
+        }
+
+        // 3. 根据可分发层级权重设置分发概率
+        Collections.sort(availableLevels, Comparator.comparingDouble(LevelWeight::getWeight));
+
+        double weightSum = availableLevels.stream().mapToDouble(o -> o.getWeight()).sum();
+        BigDecimal weightSumBD = new BigDecimal(weightSum);
+        double level_p_low = 0;
+        double weight_temp = 0;
+        double level_p_up = 0;
+        Map<String, LevelP> level_p_mapping = new HashMap<>();
+        for (LevelWeight lw : availableLevels) {
+            BigDecimal bd = new BigDecimal(weight_temp + lw.getWeight());
+            level_p_up = bd.divide(weightSumBD, 2, RoundingMode.HALF_UP).doubleValue();
+            LevelP levelP = new LevelP();
+            levelP.setMin(level_p_low);
+            levelP.setMax(level_p_up);
+            levelP.setLevelKey(lw.getLevelKey());
+            level_p_mapping.put(lw.level, levelP);
+            level_p_low = level_p_up;
+
+            weight_temp += lw.getWeight();
+        }
+
+        // 4. 随机生成[0,1)之间数，返回相应概率区间的key
+        double random_p = RandomUtils.nextDouble(0, 1);
+        for (Map.Entry<String, LevelP> entry : level_p_mapping.entrySet()) {
+            if (random_p >= entry.getValue().getMin()
+                    && random_p <= entry.getValue().getMax()) {
+                return Pair.of(entry.getValue().getLevelKey(), entry.getKey());
+            }
+        }
+        return Pair.of("", "");
+    }
+
+    @Data
+    static class LevelWeight {
+        private String level;
+        private String levelKey;
+        private Double weight;
+    }
+
+    @Data
+    static class LevelP {
+        private String levelKey;
+        private double min;
+        private double max;
+    }
+
+    @Override
+    public String pushFrom() {
+        return FlowPoolConstants.PUSH_FORM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        Pair<String, String> flowPoolKeyAndLevel = flowPoolKeyAndLevel(param);
+        String flowPoolKey = flowPoolKeyAndLevel.getLeft();
+        String level = flowPoolKeyAndLevel.getRight();
+        Set<String> data = redisTemplate.opsForSet().members(flowPoolKey);
+        if (CollectionUtils.isEmpty(data)) {
+            return null;
+        }
+        Map<String, String> videoFlowPoolMap = new LinkedHashMap<>();
+        Map<Long, String> videoFlowPoolMap_ = new LinkedHashMap<>();
+        for (String value : data) {
+            String[] values = value.split("-");
+            videoFlowPoolMap.put(values[0], values[1]);
+            videoFlowPoolMap_.put(NumberUtils.toLong(values[0], 0), values[1]);
+        }
+        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_tomson.conf");
+        List<List<Pair<Long, Double>>> results = pipeline.recall(videoFlowPoolMap);
+        List<Pair<Long, Double>> result = results.get(0);
+        Map<Long, Double> resultmap = result.stream()
+                .collect(Collectors.toMap(
+                        Pair::getLeft, // 键是Pair的left值
+                        Pair::getRight, // 值是Pair的right值
+                        (existingValue, newValue) -> existingValue, // 如果键冲突，选择保留现有的值（或者你可以根据需要定义其他合并策略）
+                        LinkedHashMap::new // 使用LinkedHashMap来保持插入顺序（如果需要的话）
+                ));
+        // 3 召回内部过滤
+        FilterParam filterParam = FilterParamFactory.create(param, result.stream()
+                .map(Pair::getLeft)
+                .collect(Collectors.toList()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        List<Video> videosResult = new ArrayList<>();
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            filterResult.getVideoIds().forEach(vid -> {
+                Video recallData = new Video();
+                recallData.setVideoId(vid);
+                recallData.setAbCode(param.getAbCode());
+                recallData.setRovScore(resultmap.getOrDefault(vid, 0.0));
+                recallData.setPushFrom(pushFrom());
+                recallData.setFlowPool(videoFlowPoolMap_.get(vid));
+                recallData.setFlowPoolAbtestGroup(param.getFlowPoolAbtestGroup());
+                recallData.setLevel(level);
+                videosResult.add(recallData);
+            });
+        }
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
+        return videosResult;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -40,6 +40,7 @@ public final class ScorerUtils {
         ScorerUtils.init4Recall("feeds_recall_config_region_v4.conf");
         ScorerUtils.init4Recall("feeds_score_config_festival.conf");
         ScorerUtils.init4Recall("feeds_score_config_bless.conf");
+        ScorerUtils.init4Recall("feeds_recall_config_tomson.conf");
     }
 
     private ScorerUtils() {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/model4recall/Model4RecallTomson.java

@@ -0,0 +1,48 @@
+package com.tzld.piaoquan.recommend.server.service.score4recall.model4recall;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class Model4RecallTomson extends AbstractModel {
+    private static final Logger LOGGER = LoggerFactory.getLogger(Model4RecallTomson.class);
+    public Map<Long, Pair<Double, Double>> kv;
+    public Model4RecallTomson() {
+        //配置不同环境的hdfs conf
+        this.kv = new HashMap<>();
+    }
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 12) {
+                continue;
+            }
+            try{
+                Long key = Long.valueOf(items[0].trim());
+                Double v1 = Double.valueOf(items[1].trim());
+                Double v2 = Double.valueOf(items[10].trim());
+                Pair<Double, Double> pair = Pair.of(v2, v1); // left是good， right是wrong。
+                kv.put(key, pair);
+            }catch (Exception e){
+                LOGGER.error(String.format("something is wrong with line: %s", line), e);
+            }
+        }
+        input.close();
+        in.close();
+        return true;
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/FlowPoolScorer.java

@@ -0,0 +1,58 @@
+package com.tzld.piaoquan.recommend.server.service.score4recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.service.score.ScorerConfigInfo;
+import com.tzld.piaoquan.recommend.server.service.score4recall.AbstractScorer4Recall;
+import com.tzld.piaoquan.recommend.server.service.score4recall.model4recall.Model4RecallKeyValue;
+import com.tzld.piaoquan.recommend.server.service.score4recall.model4recall.Model4RecallTomson;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.math3.distribution.BetaDistribution;
+
+import java.util.*;
+
+
+public class FlowPoolScorer extends AbstractScorer4Recall {
+
+    public FlowPoolScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+    @Override
+    public void loadModel() {
+        doLoadModel(Model4RecallTomson.class);
+    }
+
+    @Override
+    public List<Pair<Long, Double>> recall(Map<String, String> params){
+        Map<String, String> randomSubset = new HashMap<>();
+        if (params.size() > 3000){
+            // 将键转换为列表
+            List<String> keys = new ArrayList<>(params.keySet());
+            // 打乱键列表
+            Collections.shuffle(keys);
+            // 限制要取出的键的数量为4000，或者如果键的数量少于4000，则取所有键
+            int maxKeysToTake = Math.min(keys.size(), 3000);
+            // 从打乱后的键列表中取出前10000个键，并获取对应的值
+            for (int i = 0; i < maxKeysToTake; i++) {
+                String key = keys.get(i);
+                randomSubset.put(key, params.get(key));
+            }
+        }else{
+            randomSubset = params;
+        }
+        // todo zhangbo 这里要写实现汤姆森采样的具体功能
+        Model4RecallTomson model = (Model4RecallTomson) this.getModel();
+        Map<Long, Pair<Double, Double>> maps = model.kv;
+        List<Pair<Long, Double>> id2BetaScore = new ArrayList<>();
+        BetaDistribution betaSample;
+        for (Map.Entry<String, String> entry: randomSubset.entrySet()){
+            String id = entry.getKey();
+            Long idLong = Long.valueOf(id);
+            Pair<Double, Double> betas = maps.getOrDefault(idLong, Pair.of(0.0, 0.0));
+            betaSample = new BetaDistribution(betas.getLeft(), betas.getRight());
+            id2BetaScore.add(Pair.of(idLong, betaSample.sample()));
+        }
+        id2BetaScore.sort(Comparator.comparingDouble(o -> -o.getRight()));
+        return id2BetaScore.subList(0, Math.min(60, id2BetaScore.size()));
+    }
+
+
+}

recommend-server-service/src/main/resources/feeds_recall_config_tomson.conf

@@ -0,0 +1,8 @@
+scorer-config = {
+    score1-config = {
+        scorer-name = "com.tzld.piaoquan.recommend.server.service.score4recall.strategy.FlowPoolScorer"
+        scorer-priority = 99
+        model-path = "alg_recall_file/10_coldstart_tomson.txt"
+    }
+
+}