Merge branch 'feature/rank_enhance_hot' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV566.java

@@ -19,11 +19,9 @@ import com.tzld.piaoquan.recommend.server.util.FeatureBucketUtils;
 import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.util.*;
@@ -37,25 +35,11 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
     @ApolloJsonValue("${rank.score.merge.weightv566:}")
     private Map<String, Double> mergeWeight;
 
-    @Value("#{'${rootsessionid.tails.566:}'.split(',')}")
-    private Set<String> rootSessionIdTails;
-
     @Autowired
     private FeatureService featureService;
 
-    @Autowired
-    private RankStrategy4RegionMergeModelV567 modelV567;
-
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        String rootSessionId = param.getRootSessionId();
-        if (StringUtils.isBlank(rootSessionId) || CollectionUtils.isEmpty(rootSessionIdTails)) {
-            return modelV567.mergeAndRankRovRecall(param);
-        }
-        String tail = rootSessionId.substring(rootSessionId.length() - 1);
-        if (!rootSessionIdTails.contains(tail)) {
-            return modelV567.mergeAndRankRovRecall(param);
-        }
         Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
 
         //-------------------融-------------------
@@ -123,14 +107,6 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
         headCate2Rov = headCate2Rov.subList(0, Math.min(mergeWeight.getOrDefault("headCate2Rov", 5.0).intValue(), headCate2Rov.size()));
         rovRecallRank.addAll(headCate2Rov);
         setVideo.addAll(headCate2Rov.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //------------------- head cate2 and channel of rovn------------------
-        List<Video> headCate2AndChannelRov = extractAndSort(param, HeadCate2AndChannelRovRecallStrategy.PUSH_FROM);
-        // 视频去重
-        removeDuplicate(headCate2AndChannelRov);
-        headCate2AndChannelRov = headCate2AndChannelRov.stream().filter(o -> !setVideo.contains(o.getVideoId())).collect(Collectors.toList());
-        headCate2AndChannelRov = headCate2AndChannelRov.subList(0, Math.min(mergeWeight.getOrDefault("headCate2AndChannelRov", 5.0).intValue(), headCate2AndChannelRov.size()));
-        rovRecallRank.addAll(headCate2AndChannelRov);
-        setVideo.addAll(headCate2AndChannelRov.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -176,7 +152,7 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
             String headVidStr = String.valueOf(param.getHeadVid());
             String mergeCate2 = this.findVideoMergeCate2(videoBaseInfoMap, headVidStr);
             Double length = mergeWeight.getOrDefault("cate2CoefficientLength", 10000d);
-            Map<String, Double> simCateScore = this.findSimCateScore(mergeCate2, param.getChannelName(), length.intValue());
+            Map<String, Double> simCateScore = this.findSimCateScore(mergeCate2, length.intValue());
             cate2Coefficient.putAll(simCateScore);
         }
         Double cate2CoefficientDenominator = mergeWeight.getOrDefault("cate2CoefficientDenominator", 1d);
@@ -425,12 +401,12 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
         }
     }
 
-    private Map<String, Double> findSimCateScore(String headCate2, String channelName, int length) {
+    private Map<String, Double> findSimCateScore(String headCate2, int length) {
         if (StringUtils.isBlank(headCate2)) {
             return new HashMap<>();
         }
 
-        String redisKey = String.format("alg_recsys_channel_good_cate_pair_list:%s:%s", channelName, headCate2);
+        String redisKey = String.format("alg_recsys_good_cate_pair_list:%s", headCate2);
         String cate2Value = redisTemplate.opsForValue().get(redisKey);
         if (StringUtils.isEmpty(cate2Value)) {
             return new HashMap<>();
@@ -460,6 +436,7 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
         for (int i = 0; i < minLength; i++) {
             resultMap.put(valueList[i].trim(), Double.parseDouble(scoreList[i].trim()));
         }
+
         return resultMap;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/tansform/FeatureV6.java

@@ -13,6 +13,7 @@ public class FeatureV6 {
     private static final int seqMaxN = 2;
     private static final int seqLastN = 2;
     private static final double smoothPlus = 5.0;
+    private static final double largerSmoothPlus = 30.0;
     private static final double log1Scale = 10.0;
     private static final List<String> c1Periods = Arrays.asList("72h", "168h");
     private static final List<String> b0Periods = Arrays.asList("1h", "3h", "6h", "12h");
@@ -28,11 +29,12 @@ public class FeatureV6 {
     private static final List<String> b10Periods = Arrays.asList("1h", "12h");
     private static final List<String> b11Periods = Arrays.asList("12h", "168h");
     private static final List<String> b13Periods = Arrays.asList("24h", "168h");
-    private static final List<String> videoCateAttrs = Arrays.asList(FeatureUtils.cate1Attr, FeatureUtils.cate2Attr, FeatureUtils.festive1Attr, FeatureUtils.channelAttr);
+    private static final List<String> videoCateAttrs = Arrays.asList(FeatureUtils.cate1Attr, FeatureUtils.cate2Attr, FeatureUtils.festive1Attr, FeatureUtils.channelAttr, FeatureUtils.sourceAttr);
     private static final List<String> videoSimAttrs = Arrays.asList("title", "cate2", "cate2_list", "keywords");
     private static final List<String> hVideoSimAttrs = Arrays.asList("title");
     private static final List<String> cfList = Arrays.asList("share", "return");
     private static final List<String> userAttrList = Arrays.asList("province", "city", "model", "brand", "system", "user_channel", "user_level");
+    private static final Set<String> appSet = new HashSet<>(Arrays.asList("0", "2", "4"));
     private static final Set<String> hotSceneSet = new HashSet<>(Arrays.asList("1008", "1007", "1058", "1074", "1010"));
 
     public static void getContextFeature(long currentMs, String appType, String hotSceneType, Map<String, Double> featureMap) {
@@ -44,7 +46,13 @@ public class FeatureV6 {
         featureMap.put(String.format("%s@%d", "week", week), 1.0);
         featureMap.put(String.format("%s@%d", "hour", hour), 1.0);
         featureMap.put("hour", hour * 1.0);
-        featureMap.put(String.format("%s@%s", "app", appType), 1.0);
+        String app;
+        if (appSet.contains(appType)) {
+            app = appType;
+        } else {
+            app = "other";
+        }
+        featureMap.put(String.format("%s@%s", "app", app), 1.0);
         String hot;
         if (hotSceneSet.contains(hotSceneType)) {
             hot = hotSceneType;
@@ -302,7 +310,7 @@ public class FeatureV6 {
         if (videoInfo.containsKey("keywords")) {
             String keywords = videoInfo.get("keywords");
             if (null != keywords && !keywords.isEmpty()) {
-                for (String kw : keywords.split(",")) {
+                for (String kw : keywords.split("[,，、]")) {
                     kw = kw.replaceAll("(\\s+|\\t|:)", "");
                     if (!kw.isEmpty()) {
                         String featKey = String.format("%s@kw@%s", prefix, kw);
@@ -409,6 +417,12 @@ public class FeatureV6 {
             double ros_n = FeatureUtils.plusSmooth(return_n_uv, share_cnt, smoothPlus);
             double ros_minus = FeatureUtils.plusSmooth(return_n_uv, is_return_1, smoothPlus);
 
+            // larger smooth
+            double l_rovn = FeatureUtils.plusSmooth(return_n_uv, exp, largerSmoothPlus, 1);
+            double l_ros = FeatureUtils.plusSmooth(return_n_uv / 5, is_share, largerSmoothPlus, 2);
+            double l_ros_n = FeatureUtils.plusSmooth(return_n_uv / 5, share_cnt, largerSmoothPlus, 2);
+            double l_ros_minus = FeatureUtils.plusSmooth(return_n_uv / 5, is_return_1, largerSmoothPlus, 2);
+
             featMap.put(prefix + "_" + period + "@" + "exp", exp_s);
             featMap.put(prefix + "_" + period + "@" + "is_share", is_share_s);
             featMap.put(prefix + "_" + period + "@" + "share_cnt", share_cnt_s);
@@ -421,6 +435,12 @@ public class FeatureV6 {
             featMap.put(prefix + "_" + period + "@" + "ros", ros);
             featMap.put(prefix + "_" + period + "@" + "ros_n", ros_n);
             featMap.put(prefix + "_" + period + "@" + "ros_minus", ros_minus);
+
+            // larger smooth
+            featMap.put(prefix + "_" + period + "@" + "rovn_#", l_rovn);
+            featMap.put(prefix + "_" + period + "@" + "ros_#", l_ros);
+            featMap.put(prefix + "_" + period + "@" + "ros_n_#", l_ros_n);
+            featMap.put(prefix + "_" + period + "@" + "ros_minus_#", l_ros_minus);
         }
     }
 
@@ -460,6 +480,17 @@ public class FeatureV6 {
             double ros_n = FeatureUtils.plusSmooth(return_n_uv, share_cnt, smoothPlus);
             double ros_minus = FeatureUtils.plusSmooth(return_n_uv, is_return_1, smoothPlus);
 
+            // larger smooth
+            double l_rovn1 = FeatureUtils.plusSmooth(return_1_uv, exp, largerSmoothPlus, 1);
+            double l_ros1 = FeatureUtils.plusSmooth(return_1_uv / 5, is_share, largerSmoothPlus, 2);
+            double l_ros_n1 = FeatureUtils.plusSmooth(return_1_uv / 5, share_cnt, largerSmoothPlus, 2);
+            double l_ros_minus1 = FeatureUtils.plusSmooth(return_1_uv / 5, is_return_1, largerSmoothPlus, 2);
+
+            double l_rovn = FeatureUtils.plusSmooth(return_n_uv, exp, largerSmoothPlus, 1);
+            double l_ros = FeatureUtils.plusSmooth(return_n_uv / 5, is_share, largerSmoothPlus, 2);
+            double l_ros_n = FeatureUtils.plusSmooth(return_n_uv / 5, share_cnt, largerSmoothPlus, 2);
+            double l_ros_minus = FeatureUtils.plusSmooth(return_n_uv / 5, is_return_1, largerSmoothPlus, 2);
+
             featMap.put(prefix + "_" + period + "@" + "exp", exp_s);
             featMap.put(prefix + "_" + period + "@" + "is_share", is_share_s);
             featMap.put(prefix + "_" + period + "@" + "share_cnt", share_cnt_s);
@@ -477,6 +508,16 @@ public class FeatureV6 {
             featMap.put(prefix + "_" + period + "@" + "ros", ros);
             featMap.put(prefix + "_" + period + "@" + "ros_n", ros_n);
             featMap.put(prefix + "_" + period + "@" + "ros_minus", ros_minus);
+
+            // larger smooth
+            featMap.put(prefix + "_" + period + "@" + "rovn1_#", l_rovn1);
+            featMap.put(prefix + "_" + period + "@" + "ros1_#", l_ros1);
+            featMap.put(prefix + "_" + period + "@" + "ros_n1_#", l_ros_n1);
+            featMap.put(prefix + "_" + period + "@" + "ros_minus1_#", l_ros_minus1);
+            featMap.put(prefix + "_" + period + "@" + "rovn_#", l_rovn);
+            featMap.put(prefix + "_" + period + "@" + "ros_#", l_ros);
+            featMap.put(prefix + "_" + period + "@" + "ros_n_#", l_ros_n);
+            featMap.put(prefix + "_" + period + "@" + "ros_minus_#", l_ros_minus);
         }
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -114,9 +114,6 @@ public class RecallService implements ApplicationContextAware {
         strategies.add(strategyMap.get(HeadProvinceCate2RecallStrategy.class.getSimpleName()));
         strategies.add(strategyMap.get(HeadCate2RovRecallStrategy.class.getSimpleName()));
         Set<String> abExpCodes = param.getAbExpCodes();
-        if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("566")) {
-            strategies.add(strategyMap.get(HeadCate2AndChannelRovRecallStrategy.class.getSimpleName()));
-        }
         if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("568")) {
             strategies.add(strategyMap.get(PremiumROVRecallStrategy.class.getSimpleName()));
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/FeatureUtils.java

@@ -12,6 +12,7 @@ public class FeatureUtils {
     public static final String cate2Attr = "cate2";
     public static final String festive1Attr = "festive_label1";
     public static final String channelAttr = "channel";
+    public static final String sourceAttr = "vid_source";
     public static final double twoMonthMs = 2 * 30 * 24 * 3600 * 1000.0;
 
     private static final String goodMorningRegex = "(早安|早上好|早晨好|上午好)";
@@ -81,6 +82,17 @@ public class FeatureUtils {
         return a / (b + plus);
     }
 
+    public static double plusSmooth(double a, double b, double plus, double upper) {
+        if (a == 0 || b == 0) {
+            return 0D;
+        }
+        double value = a / (b + plus);
+        if (value > upper) {
+            value = upper + Math.log(value - upper + 1) / 10;
+        }
+        return value;
+    }
+
     public static double wilsonScore(double click, double exposure) {
         if (exposure <= 0) {
             return 0.0;