Merge branch 'feature/zhangbo_rank' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV1.java

@@ -6,16 +6,15 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeatureV2;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeatureV2;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
@@ -35,7 +34,7 @@ import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
-
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -116,13 +115,26 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         //-------------------逻-------------------
         //-------------------辑-------------------
 
+        List<Video> oldRovs = new ArrayList<>();
+        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        int sizeReturn = param.getSize();
+        removeDuplicate(oldRovs);
+        oldRovs = oldRovs.size() <= sizeReturn
+                ? oldRovs
+                : oldRovs.subList(0, sizeReturn);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, oldRovs);
+
         //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
         List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
         List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v1);
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
@@ -136,22 +148,25 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
         this.duplicate(setVideo, v7);
 
+        rovRecallRank.addAll(oldRovs);
         rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
         rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
         rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 0.0).intValue(), v4.size())));
         rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
         rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
         rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
+
+
+
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
         // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<RankItem> items = model(rovRecallRank, param);
         List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
         List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
         Calendar calendar = Calendar.getInstance();
@@ -192,6 +207,14 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流的rov和ros
+            List<Double> share2allreturn = getRateData(returns, shares, 1.0, 10.0);
+            Double share2allreturnScore = calScoreWeight(share2allreturn);
+            List<Double> view2allreturn = getRateData(returns, views, 0.0, 0.0);
+            Double view2allreturnScore = calScoreWeight(view2allreturn);
+            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
+            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
+
             // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
@@ -211,19 +234,26 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 1.0);
-        double b = mergeWeight.getOrDefault("b", 1.0);
+        double a = mergeWeight.getOrDefault("a", 0.1);
+        double b = mergeWeight.getOrDefault("b", 0.0);
+        double bb = mergeWeight.getOrDefault("bb", 0.005);
         double c = mergeWeight.getOrDefault("c", 0.0002);
         double d = mergeWeight.getOrDefault("d", 1.0);
         double e = mergeWeight.getOrDefault("e", 1.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
+        double f = mergeWeight.getOrDefault("f", 0.1);
+        double g = mergeWeight.getOrDefault("g", 1.0);
+        double h = mergeWeight.getOrDefault("h", 20.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
         for (RankItem item : items){
             double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
             double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
             double strScore = item.getScoreStr();
+            double rosScoreModel = item.getScoreRos();
             double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
+            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
             double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
             double score = 0.0;
             if (ifAdd < 0.5){
@@ -232,18 +262,23 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
             }else {
                 score = a * strScore + b * rosScore + c * preturnsScore +
                         (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+
+            }
+            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
+            if (allreturnsScore > h){
+                score += (bb * rosScoreModel + f * share2allreturnScore + g * view2allreturnScore);
             }
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
             video.setScoreStr(item.getScoreStr());
+            video.setScoreRos(item.getScoreRos());
             video.setScoresMap(item.getScoresMap());
             result.add(video);
         }
         Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
         return result;
     }
-
     public double calNewVideoScore(Map<String, String> itemBasicMap){
         double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
         if (existenceDays > 5){
@@ -281,9 +316,13 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
     public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
         List<Double> data = new LinkedList<>();
         for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
+            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)){
+                data.add(0.0);
+            }else{
+                data.add(
+                        (ups.get(i) + up) / (downs.get(i) + down)
+                );
+            }
         }
         return data;
     }
@@ -298,8 +337,7 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         }
         return views;
     }
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart){
+    public List<RankItem> model(List<Video> videos, RankParam param){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -335,14 +373,13 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 JSONObject obj = new JSONObject();
                 obj.put("name", "user_key_in_model_is_null");
                 obj.put("class", this.CLASS_NAME);
-//                log.info(obj.toString());
-//                return videos;
             }
         }
         final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
                 "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
                 "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
         ));
         Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
         while (iterator.hasNext()) {
@@ -351,27 +388,29 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 iterator.remove();
             }
         }
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+        Map<String, String> f1 = RankExtractorUserFeatureV2.getOriginFeature(userFeatureMap,
                 new HashSet<String>(Arrays.asList(
                         "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
                 ))
         );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+        Map<String, Double> f2__ = RankExtractorUserFeatureV2.getUserRateFeature(userFeatureMap);
+        Map<String, String> f2 = RankExtractorUserFeatureV2.rateFeatureChange(f2__);
+        Map<String, String> f3 = RankExtractorUserFeatureV2.cntFeatureChange(userFeatureMap,
                 new HashSet<String>(Arrays.asList(
                         "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                        "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
                 ))
         );
         f1.putAll(f2);
         f1.putAll(f3);
-//        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
                 "total_time", "play_count_total",
                 "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"
         ));
 
         List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
@@ -397,12 +436,14 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                             iteratorIn.remove();
                         }
                     }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                    Map<String, Double> f4__ = RankExtractorItemFeatureV2.getItemRateFeature(vfMap);
+                    Map<String, String> f4 = RankExtractorItemFeatureV2.rateFeatureChange(f4__);
+                    Map<String, String> f5 = RankExtractorItemFeatureV2.cntFeatureChange(vfMap,
                             new HashSet<String>(Arrays.asList(
                                     "total_time", "play_count_total",
                                     "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                                    "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"))
                     );
                     f4.putAll(f5);
                     rankItems.get(i).setFeatureMap(f4);
@@ -435,7 +476,6 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         videoRtKeys1.addAll(videoRtKeys2);
         List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
 
-
         if (videoRtFeatures != null){
             int j = 0;
             for (RankItem item: rankItems){
@@ -464,7 +504,8 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 }catch (Exception e){
                     log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
                 item.getFeatureMap().putAll(f8);
             }
             for (RankItem item: rankItems){
@@ -495,23 +536,17 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 }catch (Exception e){
                     log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
                 item.getFeatureMap().putAll(f8);
             }
         }
 
-
-//        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
-
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline("feeds_score_config_20240228.conf")
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-//        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
         JSONObject obj = new JSONObject();
         obj.put("name", "user_key_in_model_is_not_null");
         obj.put("class", this.CLASS_NAME);
-//        log.info(obj.toString());
         return rovRecallScore;
     }
     private Map<String, String> getSceneFeature(RankParam param) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -95,6 +95,7 @@ public class RecallService implements ApplicationContextAware {
             return strategies;
         } else {
             switch (abCode) {
+                case "60121": // 536
                 case "60122": // 537
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
@@ -103,7 +104,6 @@ public class RecallService implements ApplicationContextAware {
                     strategies.addAll(getRegionRecallStrategy(param));
                     break;
                 case "60120": // 576
-                case "60121": // 536
                 case "60123": // 541
                 case "60124": // 546
                 case "60125": // 547