feat:修改565实验

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV565.java

@@ -1,7 +1,6 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.FeatureService;
@@ -12,16 +11,12 @@ import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.ExtractFeature20250218;
 import com.tzld.piaoquan.recommend.server.util.FeatureBucketUtils;
-import com.tzld.piaoquan.recommend.server.util.SimilarityUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.math3.util.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.util.*;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Service
@@ -33,14 +28,6 @@ public class RankStrategy4RegionMergeModelV565 extends RankStrategy4RegionMergeM
     @Autowired
     private FeatureService featureService;
 
-    private static final List<String> shortPeriod = Arrays.asList("1h", "2h", "4h", "6h", "12h", "24h", "7d");
-    private static final List<String> middlePeriod = Arrays.asList("14d", "30d");
-    private static final List<String> longPeriod = Arrays.asList("7d", "35d", "90d", "365d");
-    private static final List<String> cfRosList = Collections.singletonList("rosn");
-    private static final List<String> cfRovList = Collections.singletonList("rovn");
-    private static final List<String> videoSimAttrs = Arrays.asList("cate1_list", "cate2", "cate2_list",
-            "keywords", "style", "theme", "title", "topic", "user_value");
-
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
         Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
@@ -76,6 +63,31 @@ public class RankStrategy4RegionMergeModelV565 extends RankStrategy4RegionMergeM
         v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
         rovRecallRank.addAll(v1);
         setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------scene cf rovn------------------
+        List<Video> sceneCFRovn = extractAndSort(param, SceneCFRovnRecallStrategy.PUSH_FORM);
+        sceneCFRovn = sceneCFRovn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        sceneCFRovn = sceneCFRovn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), sceneCFRovn.size()));
+        rovRecallRank.addAll(sceneCFRovn);
+        setVideo.addAll(sceneCFRovn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------scene cf rosn------------------
+        List<Video> sceneCFRosn = extractAndSort(param, SceneCFRosnRecallStrategy.PUSH_FORM);
+        sceneCFRosn = sceneCFRosn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
+        rovRecallRank.addAll(sceneCFRosn);
+        setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------省份ros指标实时召回------------------
+        List<Video> hourROSRecall = extractAndSort(param, RegionRealtimeRecallStrategyROS.PUSH_FORM);
+        hourROSRecall = hourROSRecall.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        hourROSRecall = hourROSRecall.subList(0, Math.min(mergeWeight.getOrDefault("hourROSRecall", 5.0).intValue(), hourROSRecall.size()));
+        rovRecallRank.addAll(hourROSRecall);
+        setVideo.addAll(hourROSRecall.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------长周期ros------------------
+        List<Video> LongTermROSRecall = extractAndSort(param, RegionRealtimeRecallStrategyV7LongTermV1.PUSH_FORM);
+        LongTermROSRecall = LongTermROSRecall.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        LongTermROSRecall = LongTermROSRecall.subList(0, Math.min(mergeWeight.getOrDefault("LongTermROSRecall", 1.0).intValue(), LongTermROSRecall.size()));
+        rovRecallRank.addAll(LongTermROSRecall);
+        setVideo.addAll(LongTermROSRecall.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -98,7 +110,7 @@ public class RankStrategy4RegionMergeModelV565 extends RankStrategy4RegionMergeM
 
         long ts = System.currentTimeMillis() / 1000;
 
-        FeatureService.Feature feature = featureService.getFeatureByNewLabel(appType, sceneType, provinceCn, brand, param.getMid(), param.getHeadVid().toString(), vids, videoBaseInfoMap);
+        FeatureService.Feature feature = featureService.getFeatureByNewLabel(appType, sceneType, provinceCn, brand, param.getMid(), headVid, vids, videoBaseInfoMap);
         Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
         Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
         Map<String, String> headVideoInfo = videoBaseInfoMap.getOrDefault(headVid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
@@ -182,10 +194,12 @@ public class RankStrategy4RegionMergeModelV565 extends RankStrategy4RegionMergeM
         }
         // 4 排序模型计算
         double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.05);
+        double calcVorMode = mergeWeight.getOrDefault("calc_vor_mode", 1d);
+
         Map<String, String> sceneFeatureMap = new HashMap<>(0);
         List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_xgb_rov_20250228.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
         // 5 排序公式特征
-        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor_4share:");
         List<Video> result = new ArrayList<>();
         for (RankItem item : items) {
             double score;
@@ -194,11 +208,20 @@ public class RankStrategy4RegionMergeModelV565 extends RankStrategy4RegionMergeM
             double fmRov = restoreScore(fmRovOrigin, xgbRovNegRate);
             item.getScoresMap().put("fmRov", fmRov);
             item.getScoresMap().put("xgbRovNegRate", xgbRovNegRate);
-            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
-            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
-            double vor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
-            item.getScoresMap().put("vor", vor);
-            score = fmRov * (0.1 + hasReturnRovScore) * (0.1 + vor);
+
+
+            Map<String, String> vidFeatureMap = vid2MapFeature.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>());
+            double ros24h = Double.parseDouble(vidFeatureMap.getOrDefault("ros_24h", "0"));
+            double vor24h = Double.parseDouble(vidFeatureMap.getOrDefault("vor_24h", "0"));
+            if (calcVorMode == 1d) {
+                vor24h = ExtractorUtils.calLog(vor24h);
+            } else if (vor24h == 2d) {
+                double vorCoefficient = mergeWeight.getOrDefault("vor_coefficient", 1d);
+                vor24h = vorCoefficient * vor24h;
+            }
+            score = fmRov * (0.1 + ros24h) * (0.1 + ExtractorUtils.calLog(vor24h));
+
+
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -222,118 +245,4 @@ public class RankStrategy4RegionMergeModelV565 extends RankStrategy4RegionMergeM
         return result;
     }
 
-    private Map<String, String> getVideoOneTypeInfo(String vid, String name,
-                                                    Map<String, Map<String, Map<String, String>>> videoAllInfoMap) {
-        if (null == videoAllInfoMap) {
-            return new HashMap<>();
-        }
-        return videoAllInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault(name, new HashMap<>());
-    }
-
-    private double getVideoOneInfo(String name, Map<String, String> infoMap) {
-        if (null == infoMap) {
-            return 0.0;
-        }
-        return infoMap.isEmpty() ? 0 : Double.parseDouble(infoMap.getOrDefault(name, "0.0"));
-    }
-
-    private void addVideoStatFeature(String vid, Map<String, Map<String, Map<String, String>>> videoAllInfoMap,
-                                     Map<String, Double> featureMap) {
-        List<Tuple3> vidStatInfo = Arrays.asList(
-                new Tuple3("b20", shortPeriod, getVideoOneTypeInfo(vid, "alg_cate2_feature", videoAllInfoMap)),
-                new Tuple3("b21", shortPeriod, getVideoOneTypeInfo(vid, "alg_cate1_feature", videoAllInfoMap)),
-                new Tuple3("b22", shortPeriod, getVideoOneTypeInfo(vid, "alg_vid_source_feature", videoAllInfoMap)),
-                new Tuple3("b28", shortPeriod, getVideoOneTypeInfo(vid, "alg_sence_type_feature", videoAllInfoMap)),
-                new Tuple3("b29", shortPeriod, getVideoOneTypeInfo(vid, "alg_videoid_feature", videoAllInfoMap)),
-                new Tuple3("b23", middlePeriod, getVideoOneTypeInfo(vid, "alg_cate2_feature_day", videoAllInfoMap)),
-                new Tuple3("b24", middlePeriod, getVideoOneTypeInfo(vid, "alg_cate1_feature_day", videoAllInfoMap)),
-                new Tuple3("b25", middlePeriod, getVideoOneTypeInfo(vid, "alg_video_source_feature_day", videoAllInfoMap)),
-                new Tuple3("b26", longPeriod, getVideoOneTypeInfo(vid, "alg_video_unionid_feature_day", videoAllInfoMap)),
-                new Tuple3("b27", longPeriod, getVideoOneTypeInfo(vid, "alg_vid_feature_day", videoAllInfoMap))
-        );
-        for (Tuple3 tuple3 : vidStatInfo) {
-            String infoType = tuple3.first;
-            List<String> infoPeriod = tuple3.second;
-            Map<String, String> infoMap = tuple3.third;
-            for (String period : infoPeriod) {
-                double share = getVideoOneInfo("share_" + period, infoMap);
-                double return_ = getVideoOneInfo("return_" + period, infoMap);
-                double view_hasreturn = getVideoOneInfo("view_hasreturn_" + period, infoMap);
-                double share_hasreturn = getVideoOneInfo("share_hasreturn_" + period, infoMap);
-                double ros = getVideoOneInfo("ros_" + period, infoMap);
-                double rov = getVideoOneInfo("rov_" + period, infoMap);
-                double r_cnt = getVideoOneInfo("r_cnt_" + period, infoMap);
-                double r_rate = getVideoOneInfo("r_rate_" + period, infoMap);
-                double r_cnt4s = getVideoOneInfo("r_cnt4s_" + period, infoMap);
-                double str = getVideoOneInfo("str_" + period, infoMap);
-
-                featureMap.put(infoType + "_" + period + "_" + "share", ExtractorUtils.calLog(share));
-                featureMap.put(infoType + "_" + period + "_" + "return", ExtractorUtils.calLog(return_));
-                featureMap.put(infoType + "_" + period + "_" + "view_hasreturn", ExtractorUtils.calLog(view_hasreturn));
-                featureMap.put(infoType + "_" + period + "_" + "share_hasreturn", ExtractorUtils.calLog(share_hasreturn));
-                featureMap.put(infoType + "_" + period + "_" + "ros", ros);
-                featureMap.put(infoType + "_" + period + "_" + "rov", rov);
-                featureMap.put(infoType + "_" + period + "_" + "r_cnt", r_cnt);
-                featureMap.put(infoType + "_" + period + "_" + "r_rate", r_rate);
-                featureMap.put(infoType + "_" + period + "_" + "r_cnt4s", r_cnt4s);
-                featureMap.put(infoType + "_" + period + "_" + "str", str);
-            }
-        }
-    }
-
-    private void addVideoCFFeature(String vid, Map<String, Map<String, Map<String, String>>> videoAllInfoMap,
-                                   Map<String, Double> featureMap) {
-        List<Tuple3> vidCFInfo = Arrays.asList(
-                new Tuple3("d2", cfRosList, getVideoOneTypeInfo(vid, "alg_recsys_feature_weak_cf_i2i_scene_ros", videoAllInfoMap)),
-                new Tuple3("d3", cfRosList, getVideoOneTypeInfo(vid, "alg_recsys_feature_cf_i2i_scene_ros", videoAllInfoMap)),
-                new Tuple3("d4", cfRovList, getVideoOneTypeInfo(vid, "alg_recsys_feature_weak_cf_i2i_scene_rov", videoAllInfoMap)),
-                new Tuple3("d5", cfRovList, getVideoOneTypeInfo(vid, "alg_recsys_feature_cf_i2i_scene_rov", videoAllInfoMap))
-        );
-        for (Tuple3 tuple3 : vidCFInfo) {
-            String infoType = tuple3.first;
-            List<String> valTypeList = tuple3.second;
-            Map<String, String> infoMap = tuple3.third;
-            if (!infoMap.isEmpty()) {
-                for (String valType : valTypeList) {
-                    String expKey = "exp";
-                    if (valType.equals("rosn")) {
-                        expKey = "share";
-                    }
-                    double exp = getVideoOneInfo(expKey, infoMap);
-                    double return_n = getVideoOneInfo("return_n", infoMap);
-                    double value = getVideoOneInfo(valType, infoMap);
-
-                    featureMap.put(infoType + "_exp", ExtractorUtils.calLog(exp));
-                    featureMap.put(infoType + "_return_n", ExtractorUtils.calLog(return_n));
-                    featureMap.put(infoType + "_" + valType, value);
-                }
-            }
-        }
-    }
-
-    private void addVideoSimFeature(Map<String, String> headInfo, Map<String, String> rankInfo, Map<String, Double> featureMap) {
-        if (!headInfo.isEmpty() && !rankInfo.isEmpty()) {
-            List<Future<Pair<String, Double>>> futures = new ArrayList<>();
-            for (String attr : videoSimAttrs) {
-                String headAttr = headInfo.getOrDefault(attr, "");
-                String rankAttr = rankInfo.getOrDefault(attr, "");
-                if (!"".equals(headAttr) && !"unknown".equals(headAttr) && !"".equals(rankAttr) && !"unknown".equals(rankAttr)) {
-                    String key = "video_sim_" + attr;
-                    Future<Pair<String, Double>> future = ThreadPoolFactory.defaultPool().submit(() -> {
-                        double simScore = SimilarityUtils.word2VecSimilarity(headAttr, rankAttr);
-                        return Pair.create(key, simScore);
-                    });
-                    futures.add(future);
-                }
-            }
-            try {
-                for (Future<Pair<String, Double>> future : futures) {
-                    Pair<String, Double> pair = future.get(1000, TimeUnit.MILLISECONDS);
-                    featureMap.put(pair.getFirst(), pair.getSecond());
-                }
-            } catch (Exception e) {
-                log.error("video attr similarity error", e);
-            }
-        }
-    }
 }