Merge branch 'feature/zhangbo_rank' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelBasic.java

@@ -56,6 +56,32 @@ public class RankStrategy4RegionMergeModelBasic extends RankService {
         return result;
     }
 
+    public double calNewVideoScore(Map<String, String> itemBasicMap) {
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5) {
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
+
+    public double calTrendScore(List<Double> data) {
+        double sum = 0.0;
+        int size = data.size();
+        for (int i = 0; i < size - 4; ++i) {
+            sum += data.get(i) - data.get(i + 4);
+        }
+        if (sum * 10 > 0.6) {
+            sum = 0.6;
+        } else {
+            sum = sum * 10;
+        }
+        if (sum > 0) {
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
     public Double calScoreWeightNoTimeDecay(List<Double> data) {
         Double up = 0.0;
         Double down = 0.0;
@@ -233,6 +259,45 @@ public class RankStrategy4RegionMergeModelBasic extends RankService {
         return new RankResult(resultWithDensity);
     }
 
+    public void getVideoFeatureFromRedis(List<RankItem> items){
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(items, RankItem::getVideoId);
+        List<String> videoKeys = videoIds.stream().map(r -> "item_rt_fea_1hrootall_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoKeys);
+        int j = 0;
+        if (videoRtFeatures != null) {
+            for (RankItem item : items) {
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null) {
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try {
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
+                    }, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
+                        String value = entry.getValue();
+                        if (value == null) {
+                            continue;
+                        }
+                        String[] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1) {
+                            String[] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeRootFeature(vfMapNew);
+                } catch (Exception e) {
+                    log.error(String.format("parse video item_rt_fea_1hrootall_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+    }
+
     public static void main(String[] args) {
 
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV655.java

@@ -46,15 +46,44 @@ public class RankStrategy4RegionMergeModelV655 extends RankStrategy4RegionMergeM
         List<Video> v0 = oldRovs.size() <= sizeReturn
                 ? oldRovs
                 : oldRovs.subList(0, sizeReturn);
-        this.duplicate(setVideo, v0);
         rovRecallRank.addAll(v0);
         setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回------------------
+        //-------------------sim相似召回------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        v5 = v5.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v5 = v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size()));
+        rovRecallRank.addAll(v5);
+        setVideo.addAll(v5.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------return相似召回------------------
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(v6);
+        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回v1------------------
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 25.0).intValue(), v1.size()));
         rovRecallRank.addAll(v1);
         setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回v2------------------
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2_sort.PUSH_FORM);
+        v2 = v2.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v2 = v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size()));
+        rovRecallRank.addAll(v2);
+        setVideo.addAll(v2.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回v3------------------
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        v3 = v3.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v3 = v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size()));
+        rovRecallRank.addAll(v3);
+        setVideo.addAll(v3.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回v4------------------
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        v4 = v4.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v4 = v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size()));
+        rovRecallRank.addAll(v4);
+        setVideo.addAll(v4.stream().map(Video::getVideoId).collect(Collectors.toSet()));
         //-------------------节日特殊召回-------------------
         List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
         v7 = v7.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
@@ -64,7 +93,7 @@ public class RankStrategy4RegionMergeModelV655 extends RankStrategy4RegionMergeM
         //-------------------基于cf rov n 召回-------------------
         List<Video> v8 = extractAndSort(param, CFRovnRecallStrategyV1.PUSH_FORM);
         v8 = v8.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v8 = v8.subList(0, Math.min(mergeWeight.getOrDefault("v8", 16.0).intValue(), v8.size()));
+        v8 = v8.subList(0, Math.min(mergeWeight.getOrDefault("v8", 6.0).intValue(), v8.size()));
         rovRecallRank.addAll(v8);
         setVideo.addAll(v8.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
@@ -95,39 +124,101 @@ public class RankStrategy4RegionMergeModelV655 extends RankStrategy4RegionMergeM
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
         }
+        List<String> datehoursRoot = new LinkedList<>();
+        for (int i = 0; i < 24; ++i) {
+            datehoursRoot.add(String.valueOf(i+1));
+        }
+        // 2.1 item特征提取
+        this.getVideoFeatureFromRedis(items);
+
+
         for (RankItem item : items) {
+            Map<String, Map<String, Double>> itemRealRootMap = item.getItemRealTimeRootFeature();
+            List<Double> views_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "exp");
+            List<Double> share_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "share");
+            List<Double> return_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "return");
+            List<Double> rov_20240410 = getRateData(return_20240410, views_20240410, 0.0, 0.0);
+            Double rovScore_20240410 = calScoreWeightNoTimeDecay(rov_20240410);
+            List<Double> ros_20240410 = getRateData(return_20240410, share_20240410, 1.0, 10.0);
+            Double rosScore_20240410 = calScoreWeightNoTimeDecay(ros_20240410);
+            item.scoresMap.put("rovScore_20240410", rovScore_20240410);
+            item.scoresMap.put("rosScore_20240410", rosScore_20240410);
+
             Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_uv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_uv_list_1h");
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
             List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
 
+            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
+            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
             // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 0.0, 0.0);
+            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
             Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
             List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
             Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
+            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
             item.scoresMap.put("view2allreturnScore", view2allreturnScore);
 
             // 全部回流
             Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
 
+            // 平台回流
+            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
 
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
+        double a = mergeWeight.getOrDefault("a", 0.1);
+        double b = mergeWeight.getOrDefault("b", 0.0);
+        double c = mergeWeight.getOrDefault("c", 0.000001);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
         double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
+        double g = mergeWeight.getOrDefault("g", 2.0);
+        double h = mergeWeight.getOrDefault("h", 50.0);
         for (RankItem item : items) {
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
+            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double strScore = item.getScoreStr();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double rovScore_20240410 = item.scoresMap.getOrDefault("rovScore_20240410", 0.0);
+            double rosScore_20240410 = item.scoresMap.getOrDefault("rosScore_20240410", 0.0);
+
             double score = 0.0;
+            score = a * strScore + b * rosScore + c * preturnsScore +
+                    (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
             double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > 50) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
+            if (allreturnsScore > h) {
+                score += (f * rosScore_20240410 + g * rovScore_20240410);
             }else{
-                score += (f * share2allreturnScore + g * view2allreturnScore) * 0.01;
+                score += (f * rosScore_20240410 + g * rovScore_20240410) * 0.01;
             }
             Video video = item.getVideo();
             video.setScore(score);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -333,6 +333,11 @@ public class RecallService implements ApplicationContextAware {
             case "60655":
                 strategies.add(strategyMap.get(CFRovnRecallStrategyV1.class.getSimpleName()));
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2_sort.class.getSimpleName()));
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
+                strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
+                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                 strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));
             default:
                 strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));