Merge branch 'feature/zhangbo_model' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV999.java

@@ -19,6 +19,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.*;
+import java.util.stream.Collectors;
 
 @Service
 @Slf4j
@@ -53,29 +54,26 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
                 : oldRovs.subList(0, sizeReturn);
         Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v0);
-
-
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        List<Video> rovRecallRank = new ArrayList<>(v0);
+        //-------------------return相似召回------------------
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------流量池直接送 融合+去重-------------------
-        List<Video> v9 = extractAndSort(param, FlowPoolLastDayTopRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v9);
-        //-------------------地域相关召回 融合+去重-------------------
+        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(v6);
+        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回------------------
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(v0);
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 5.0).intValue(), v9.size())));
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size())));
+        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        rovRecallRank.addAll(v1);
+        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------7天ROVn召回------------------
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        v2 = v2.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v2 = v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 16.0).intValue(), v2.size()));
+        rovRecallRank.addAll(v2);
+        setVideo.addAll(v2.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -84,6 +82,7 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
 
         // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
         List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+
         // k1:视频、k2:表、k3:特征、v:特征值
         String provinceCn = param.getProvince().replaceAll("省$", "");
         String headVid = String.valueOf(param.getHeadVid());
@@ -304,9 +303,27 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
 
         List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240609.conf")
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
         List<Video> result = new ArrayList<>();
+        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
+        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
+
         for (RankItem item : items) {
-            double score = item.getScoreRov();
+            double score = 0.0;
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault(hasReturnRovKey, "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double fmRov = item.getScoreRov();
+            item.getScoresMap().put("fmRov", fmRov);
+            if (chooseFunction == 0){
+                score = fmRov * (1 + hasReturnRovScore);
+            }else if (chooseFunction == 1){
+                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
+            }else {
+                score = fmRov * ExtractorUtils.sigmoid(hasReturnRovScore);
+            }
+
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -165,6 +165,11 @@ public class RecallService implements ApplicationContextAware {
                 strategies.addAll(getRegionRecallStrategy(param));
                 break;
             case "60118": // 568
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
+                strategies.addAll(getRegionRecallStrategy(param));
+                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                break;
             case "60119": // 569
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                 strategies.addAll(getRegionRecallStrategy(param));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV4.java

@@ -57,10 +57,10 @@ public class RegionRealtimeRecallStrategyV4 implements RecallStrategy {
                 videosResult.add(video);
             });
         }
-        Collections.sort(videosResult, Comparator.comparingDouble(o -> -o.getRovScore()));
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
         return videosResult;
     }
-    public static final String PUSH_FORM = "recall_strategy_noregion_1h";
+    public static final String PUSH_FORM = "recall_strategy_rovn_7d";
     @Override
     public String pushFrom(){
         return PUSH_FORM;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/model4recall/Model4RecallList.java

@@ -8,6 +8,7 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 
@@ -39,6 +40,7 @@ public class Model4RecallList extends AbstractModel {
                 LOGGER.error(String.format("something is wrong with parse pair %s %s: ", id, score), e);
             }
         }
+        this.recallList.sort(Comparator.comparingDouble(o -> -o.getRight()));
         input.close();
         in.close();
         return true;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/RegionRecallScorerV4.java

@@ -26,7 +26,7 @@ public class RegionRecallScorerV4 extends AbstractScorer4Recall {
         Model4RecallList model = (Model4RecallList) this.getModel();
         List<Pair<Long, Double>> lists = model.recallList;
         lists.sort(Comparator.comparingDouble(o -> -o.getRight()));
-        return lists.subList(0, Math.min(100, lists.size()));
+        return lists.subList(0, Math.min(400, lists.size()));
     }