str 实时特征模型

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/ExtractorUtils.java

@@ -0,0 +1,136 @@
+package com.tzld.piaoquan.recommend.server.service.rank.extractor;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ExtractorUtils {
+
+    public static Double division(String s1, String s2, Map<String, String> maps){
+        double rate = 0.0;
+        if (maps.containsKey(s1) && maps.containsKey(s2)){
+            Double d1 = Double.valueOf(maps.get(s1));
+            if (isDoubleEqualToZero(d1)){
+                return rate;
+            }
+            Double d2 = Double.valueOf(maps.get(s2));
+            rate = d2 / d1;
+        }
+        return rate;
+    }
+    public static Double divisionDouble(Double d1, Double d2){
+        double rate = 0.0;
+        if (isDoubleEqualToZero(d1)){
+            return rate;
+        }
+        rate = d2 / d1;
+        return rate;
+    }
+
+    public static boolean isDoubleEqualToZero(double value) {
+        final double epsilon = 1e-10; // 定义一个很小的误差范围
+        // 判断value是否在误差范围内
+        return Math.abs(value) < epsilon;
+    }
+
+
+
+    public static double calculateVariance(List<Double> numbers) {
+        double average = numbers.stream()
+                .mapToDouble(Double::doubleValue)
+                .average()
+                .orElse(0.0);
+
+        double squaredDiffSum = numbers.stream()
+                .mapToDouble(Double::doubleValue)
+                .map(x -> Math.pow(x - average, 2))
+                .average()
+                .orElse(0.0);
+
+        return squaredDiffSum;
+    }
+
+    public static double calculateAverage(List<Double> numbers) {
+        if (numbers == null || numbers.isEmpty()) {
+            return 0.0;
+        }
+        return numbers.stream()
+                .mapToDouble(Number::doubleValue)
+                .average()
+                .orElse(0.0);
+    }
+
+    public static List<Double> calculateDifferences(List<Double> numbers) {
+        List<Double> differences = new ArrayList<>();
+
+        for (int i = 0; i < numbers.size() - 1; i++) {
+            Double diff = 0.0;
+            if (!isDoubleEqualToZero(numbers.get(i))){
+                diff = (numbers.get(i + 1) - numbers.get(i)) / numbers.get(i);
+            }
+            differences.add(diff);
+        }
+
+        return differences;
+    }
+
+    public static List<String> generateHourStrings(String timeString, int N) {
+        LocalDateTime dateTime = LocalDateTime.parse(timeString, DateTimeFormatter.ofPattern("yyyyMMddHH"));
+        List<String> hourStrings = new ArrayList<>();
+        for (int i = 0; i < N; i++) {
+            hourStrings.add(dateTime.minusHours(i).format(DateTimeFormatter.ofPattern("yyyyMMddHH")));
+        }
+
+        return hourStrings;
+    }
+
+    public static String subtractHours(String inputDateTime, int hoursToSubtract) {
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHH");
+        LocalDateTime dateTime = LocalDateTime.parse(inputDateTime, formatter);
+        LocalDateTime subtractedDateTime = dateTime.minusHours(hoursToSubtract);
+        return subtractedDateTime.format(formatter);
+    }
+
+    // 针对0-1的数字，进行分桶。
+    public static Integer ceilLogRate(Double key) {
+        double bucket = Math.ceil(
+                Math.pow(key, 0.2) * 100
+        );
+        if (bucket > 300) {
+            bucket = 300;
+        }
+        if (bucket < 0) {
+            bucket = 0;
+        }
+        return (int)bucket;
+    }
+
+    // 针对大于1的数字，进行分桶。
+    public static int bucketCnt(Double key) {
+        long bucket = Math.round(Math.log((key * 10 + 1.0)) * 10);
+        if (bucket > 300) {
+            bucket = 300;
+        }
+        if (bucket < 0) {
+            bucket = 0;
+        }
+        return (int)bucket;
+    }
+
+    public static void main(String[] args) {
+//        System.out.println(ceilLogRate(0.0002));
+//        System.out.println(ceilLogRate(0.01));
+//        System.out.println(ceilLogRate(0.2));
+//        System.out.println(ceilLogRate(4.));
+//        System.out.println(bucketCnt(1.));
+//        System.out.println(bucketCnt(20.));
+//        System.out.println(bucketCnt(500.));
+//        System.out.println(bucketCnt(50000.));
+
+        System.out.println(generateHourStrings("2024011603", 5));
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/RankExtractorItemFeature.java

@@ -0,0 +1,324 @@
+package com.tzld.piaoquan.recommend.server.service.rank.extractor;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class RankExtractorItemFeature {
+    public static Map<String, String> getItemRateFeature(Map<String, String> maps) {
+
+        double d;
+        Map<String, Double> result = new HashMap<>();
+        d = ExtractorUtils.division("i_1day_exp_cnt", "i_1day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_1day_ctr",d);
+        }
+        d = ExtractorUtils.division("i_1day_exp_cnt", "i_1day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_1day_str",d);
+        }
+        d = ExtractorUtils.division("i_1day_exp_cnt", "i_1day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_1day_rov",d);
+        }
+        d = ExtractorUtils.division("i_1day_share_cnt", "i_1day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_1day_ros",d);
+        }
+
+        d = ExtractorUtils.division("i_3day_exp_cnt", "i_3day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3day_ctr",d);
+        }
+        d = ExtractorUtils.division("i_3day_exp_cnt", "i_3day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3day_str",d);
+        }
+        d = ExtractorUtils.division("i_3day_exp_cnt", "i_3day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3day_rov",d);
+        }
+        d = ExtractorUtils.division("i_3day_share_cnt", "i_3day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3day_ros",d);
+        }
+
+        d = ExtractorUtils.division("i_7day_exp_cnt", "i_7day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_7day_ctr",d);
+        }
+        d = ExtractorUtils.division("i_7day_exp_cnt", "i_7day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_7day_str",d);
+        }
+        d = ExtractorUtils.division("i_7day_exp_cnt", "i_7day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_7day_rov",d);
+        }
+        d = ExtractorUtils.division("i_7day_share_cnt", "i_7day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_7day_ros",d);
+        }
+
+        d = ExtractorUtils.division("i_3month_exp_cnt", "i_3month_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3month_ctr",d);
+        }
+        d = ExtractorUtils.division("i_3month_exp_cnt", "i_3month_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3month_str",d);
+        }
+        d = ExtractorUtils.division("i_3month_exp_cnt", "i_3month_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3month_rov",d);
+        }
+        d = ExtractorUtils.division("i_3month_share_cnt", "i_3month_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3month_ros",d);
+        }
+
+
+        return rateFeatureChange(result);
+    }
+
+    public static Map<String, String> getItemRealtimeTrend(Map<String, Map<String, Double>> maps, String date, String hour){
+        Map<String, Double> result1 = new HashMap<>();
+        Map<String, Double> result2 = new HashMap<>();
+        if (date.isEmpty() || hour.isEmpty()){
+            return rateFeatureChange(result1);
+        }
+        int N = 6;
+
+        List<String> hourStrs = ExtractorUtils.generateHourStrings(date + hour, N);
+
+        String key;
+
+        key = "share_uv_list_1day";
+        if (maps.containsKey(key)){
+            Map<String, Double> fList = maps.get(key);
+            List<Double> arrs = hourStrs.stream().map(r -> fList.getOrDefault(r, 0.0D)).collect(Collectors.toList());
+            Collections.reverse(arrs);
+            result1.put(key+"_"+N+"_avg", ExtractorUtils.calculateAverage(arrs));
+            result1.put(key+"_"+N+"_var", ExtractorUtils.calculateVariance(arrs));
+
+            List<Double> arrsDiff = ExtractorUtils.calculateDifferences(arrs);
+            result2.put(key+"_diff_"+N+"_avg", ExtractorUtils.calculateAverage(arrsDiff));
+            result2.put(key+"_diff_"+N+"_var", ExtractorUtils.calculateVariance(arrsDiff));
+        }
+
+        key = "return_uv_list_1day";
+        if (maps.containsKey(key)){
+            Map<String, Double> fList = maps.get(key);
+            List<Double> arrs = hourStrs.stream().map(r -> fList.getOrDefault(r, 0.0D)).collect(Collectors.toList());
+            Collections.reverse(arrs);
+            result1.put(key+"_"+N+"_avg", ExtractorUtils.calculateAverage(arrs));
+            result1.put(key+"_"+N+"_var", ExtractorUtils.calculateVariance(arrs));
+
+            List<Double> arrsDiff = ExtractorUtils.calculateDifferences(arrs);
+            result2.put(key+"_diff_"+N+"_avg", ExtractorUtils.calculateAverage(arrsDiff));
+            result2.put(key+"_diff_"+N+"_var", ExtractorUtils.calculateVariance(arrsDiff));
+        }
+
+        key = "share_uv_list_1h";
+        if (maps.containsKey(key)){
+            Map<String, Double> fList = maps.get(key);
+            List<Double> arrs = hourStrs.stream().map(r -> fList.getOrDefault(r, 0.0D)).collect(Collectors.toList());
+            Collections.reverse(arrs);
+            result1.put(key+"_"+N+"_avg", ExtractorUtils.calculateAverage(arrs));
+            result1.put(key+"_"+N+"_var", ExtractorUtils.calculateVariance(arrs));
+
+            List<Double> arrsDiff = ExtractorUtils.calculateDifferences(arrs);
+            result2.put(key+"_diff_"+N+"_avg", ExtractorUtils.calculateAverage(arrsDiff));
+            result2.put(key+"_diff_"+N+"_var", ExtractorUtils.calculateVariance(arrsDiff));
+        }
+
+        key = "return_uv_list_1h";
+        if (maps.containsKey(key)){
+            Map<String, Double> fList = maps.get(key);
+            List<Double> arrs = hourStrs.stream().map(r -> fList.getOrDefault(r, 0.0D)).collect(Collectors.toList());
+            Collections.reverse(arrs);
+            result1.put(key+"_"+N+"_avg", ExtractorUtils.calculateAverage(arrs));
+            result1.put(key+"_"+N+"_var", ExtractorUtils.calculateVariance(arrs));
+
+            List<Double> arrsDiff = ExtractorUtils.calculateDifferences(arrs);
+            result2.put(key+"_diff_"+N+"_avg", ExtractorUtils.calculateAverage(arrsDiff));
+            result2.put(key+"_diff_"+N+"_var", ExtractorUtils.calculateVariance(arrsDiff));
+        }
+        Map<String, String> r1 = cntFeatureChange4Double(result1);
+        Map<String, String> r2 = rateFeatureChange(result2);
+        r1.putAll(r2);
+
+        return r1;
+    }
+
+
+    public static Map<String, String> rateFeatureChange(Map<String, Double> maps){
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, Double> entry : maps.entrySet()){
+            int value = ExtractorUtils.ceilLogRate(entry.getValue());
+            result.put(entry.getKey(), String.valueOf(value));
+        }
+        return result;
+    }
+    public static Map<String, String> cntFeatureChange4Double(Map<String, Double> maps){
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, Double> entry : maps.entrySet()){
+            int value = ExtractorUtils.bucketCnt(Double.valueOf(entry.getValue()));
+            result.put(entry.getKey(), String.valueOf(value));
+        }
+        return result;
+    }
+
+    public static Map<String, String> cntFeatureChange(Map<String, String> maps,
+                                                       Set<String> names){
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, String> entry : maps.entrySet()){
+            if (!names.contains(entry.getKey())){
+                continue;
+            }
+            int value = ExtractorUtils.bucketCnt(Double.valueOf(entry.getValue()));
+            result.put(entry.getKey(), String.valueOf(value));
+        }
+        return result;
+    }
+
+    public static Map<String, String> getItemRealtimeCnt(Map<String, Map<String, Double>> maps,
+                                                         Set<String> names,
+                                                         String date, String hour){
+        Map<String, String> result = new HashMap<>();
+        if (date.isEmpty() || hour.isEmpty()){
+            return result;
+        }
+        String dateHour = ExtractorUtils.subtractHours(date + hour, 0);
+        for (Map.Entry<String, Map<String, Double>> entry : maps.entrySet()){
+            if (!names.contains(entry.getKey())){
+                continue;
+            }
+            Double num = entry.getValue().getOrDefault(dateHour, 0.0);
+            if (!ExtractorUtils.isDoubleEqualToZero(num)){
+                result.put(entry.getKey(), String.valueOf(ExtractorUtils.bucketCnt(num)));
+            }
+        }
+        return result;
+    }
+
+    public static Map<String, String> getItemRealtimeRate(Map<String, Map<String, Double>> maps,
+                                                         String datehour){
+        Map<String, Double> result = new HashMap<>();
+        if (datehour.isEmpty()){
+            return rateFeatureChange(result);
+        }
+        String dateHour = ExtractorUtils.subtractHours(datehour, 0);
+
+        double d, d1, d2;
+        String k1, k2;
+
+        k1 = "view_pv_list_1day";
+        k2 = "play_pv_list_1day";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1day_ctr_rt", d);
+            }
+        }
+
+        k1 = "view_pv_list_1day";
+        k2 = "share_pv_list_1day";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1day_str_rt", d);
+            }
+        }
+
+        k1 = "share_pv_list_1day";
+        k2 = "return_uv_list_1day";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1day_ros_rt", d);
+            }
+        }
+
+        k1 = "view_pv_list_1day";
+        k2 = "return_uv_list_1day";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1day_rov_rt", d);
+            }
+        }
+
+        //---
+        k1 = "view_pv_list_1h";
+        k2 = "play_pv_list_1h";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1h_ctr_rt", d);
+            }
+        }
+
+        k1 = "view_pv_list_1h";
+        k2 = "share_pv_list_1h";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1h_str_rt", d);
+            }
+        }
+
+        k1 = "share_pv_list_1day";
+        k2 = "return_uv_list_1h";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1h_ros_rt", d);
+            }
+        }
+
+        k1 = "view_pv_list_1h";
+        k2 = "return_uv_list_1h";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1h_rov_rt", d);
+            }
+        }
+
+
+        return rateFeatureChange(result);
+    }
+
+    public static void main(String[] args) {
+        String s1 = "share_uv_list_1day";
+        String s2 = "2024011300:2,2024011301:2,2024011304:2,2024011309:3,2024011311:3,2024011314:4,2024011315:4,2024011321:1,2024011323:1,2024011400:1,2024011401:1,2024011404:1,2024011406:1,2024011407:1,2024011408:1,2024011410:1,2024011423:1,2024011302:2,2024011305:2,2024011312:4,2024011313:4,2024011317:4,2024011318:4,2024011319:3,2024011320:1,2024011403:1,2024011409:1,2024011411:1,2024011419:1,2024011420:1,2024011422:1,2024011303:2,2024011306:2,2024011307:2,2024011308:2,2024011310:3,2024011316:4,2024011322:1,2024011402:1,2024011405:1,2024011421:1";
+        Map<String, Double> m1 = new HashMap<>();
+        Map<String, Map<String, Double>> maps = new HashMap<>();
+        for (String s : s2.split(",")){
+            String s3 = s.split(":")[0];
+            String s4 = s.split(":")[1];
+            m1.put(s3, Double.valueOf(s4));
+        }
+        maps.put(s1, m1);
+
+        String date = "20240114";
+        String hour = "20";
+        System.out.println(getItemRealtimeTrend(maps, date, hour));
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/RankExtractorUserFeature.java

@@ -0,0 +1,116 @@
+package com.tzld.piaoquan.recommend.server.service.rank.extractor;
+
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class RankExtractorUserFeature {
+    public static Map<String, String> getUserRateFeature(Map<String, String> maps) {
+
+        double d;
+        Map<String, Double> result = new HashMap<>();
+        d = ExtractorUtils.division("u_1day_exp_cnt", "u_1day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_1day_ctr",d);
+        }
+        d = ExtractorUtils.division("u_1day_exp_cnt", "u_1day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_1day_str",d);
+        }
+        d = ExtractorUtils.division("u_1day_exp_cnt", "u_1day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_1day_rov",d);
+        }
+        d = ExtractorUtils.division("u_1day_share_cnt", "u_1day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_1day_ros",d);
+        }
+
+        d = ExtractorUtils.division("u_3day_exp_cnt", "u_3day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3day_ctr",d);
+        }
+        d = ExtractorUtils.division("u_3day_exp_cnt", "u_3day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3day_str",d);
+        }
+        d = ExtractorUtils.division("u_3day_exp_cnt", "u_3day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3day_rov",d);
+        }
+        d = ExtractorUtils.division("u_3day_share_cnt", "u_3day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3day_ros",d);
+        }
+
+        d = ExtractorUtils.division("u_7day_exp_cnt", "u_7day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_7day_ctr",d);
+        }
+        d = ExtractorUtils.division("u_7day_exp_cnt", "u_7day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_7day_str",d);
+        }
+        d = ExtractorUtils.division("u_7day_exp_cnt", "u_7day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_7day_rov",d);
+        }
+        d = ExtractorUtils.division("u_7day_share_cnt", "u_7day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_7day_ros",d);
+        }
+
+        d = ExtractorUtils.division("u_3month_exp_cnt", "u_3month_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3month_ctr",d);
+        }
+        d = ExtractorUtils.division("u_3month_exp_cnt", "u_3month_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3month_str",d);
+        }
+        d = ExtractorUtils.division("u_3month_exp_cnt", "u_3month_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3month_rov",d);
+        }
+        d = ExtractorUtils.division("u_3month_share_cnt", "u_3month_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3month_ros",d);
+        }
+
+        return rateFeatureChange(result);
+    }
+
+
+    public static Map<String, String> rateFeatureChange(Map<String, Double> maps){
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, Double> entry : maps.entrySet()){
+            int value = ExtractorUtils.ceilLogRate(entry.getValue());
+            result.put(entry.getKey(), String.valueOf(value));
+        }
+        return result;
+    }
+
+    public static Map<String, String> cntFeatureChange(Map<String, String> maps, Set<String> names){
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, String> entry : maps.entrySet()){
+            if (!names.contains(entry.getKey())){
+                continue;
+            }
+            int value = ExtractorUtils.bucketCnt(Double.valueOf(entry.getValue()));
+            result.put(entry.getKey(), String.valueOf(value));
+        }
+        return result;
+    }
+
+    public static Map<String, String> getOriginFeature(Map<String, String> maps, Set<String> names){
+        Map<String, String> result = new HashMap<>();
+        for (String name: names){
+            if (maps.containsKey(name)){
+                result.put(name, maps.get(name));
+            }
+        }
+        return result;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/OfflineVlogFeatureGroup.java

@@ -10,19 +10,19 @@ public enum OfflineVlogFeatureGroup {
     u_1day_click_cnt,
     u_1day_share_cnt,
     u_1day_return_cnt,
-    u_ctr_1day,
-    u_str_1day,
-    u_rov_1day,
-    u_ros_1day,
+    u_1day_ctr,
+    u_1day_str,
+    u_1day_rov,
+    u_1day_ros,
 
     u_3day_exp_cnt,
     u_3day_click_cnt,
     u_3day_share_cnt,
     u_3day_return_cnt,
-    u_ctr_3day,
-    u_str_3day,
-    u_rov_3day,
-    u_ros_3day,
+    u_3day_ctr,
+    u_3day_str,
+    u_3day_rov,
+    u_3day_ros,
 
 
     total_time,
@@ -32,24 +32,71 @@ public enum OfflineVlogFeatureGroup {
     i_1day_click_cnt,
     i_1day_share_cnt,
     i_1day_return_cnt,
-    i_ctr_1day,
-    i_str_1day,
-    i_rov_1day,
-    i_ros_1day,
+    i_1day_ctr,
+    i_1day_str,
+    i_1day_rov,
+    i_1day_ros,
 
     i_3day_exp_cnt,
     i_3day_click_cnt,
     i_3day_share_cnt,
     i_3day_return_cnt,
-    i_ctr_3day,
-    i_str_3day,
-    i_rov_3day,
-    i_ros_3day,
+    i_3day_ctr,
+    i_3day_str,
+    i_3day_rov,
+    i_3day_ros,
 
     ctx_week,
     ctx_hour,
     ctx_region,
     ctx_city,
+
+    //    share_uv_list_1day_6_avg,
+//    share_uv_list_1day_6_var,
+//    share_uv_list_1day_diff_6_avg,
+//    share_uv_list_1day_diff_6_var,
+//    return_uv_list_1day_6_avg,
+//    return_uv_list_1day_6_var,
+//    return_uv_list_1day_diff_6_avg,
+//    return_uv_list_1day_diff_6_var,
+//    share_uv_list_1h_6_avg,
+//    share_uv_list_1h_6_var,
+//    share_uv_list_1h_diff_6_avg,
+//    share_uv_list_1h_diff_6_var,
+//    return_uv_list_1h_6_avg,
+//    return_uv_list_1h_6_var,
+//    return_uv_list_1h_diff_6_avg,
+//    return_uv_list_1h_diff_6_var,
+    view_pv_list_1day,
+    view_uv_list_1day,
+    play_pv_list_1day,
+    play_uv_list_1day,
+    share_pv_list_1day,
+    share_uv_list_1day,
+    return_uv_list_1day,
+    p_view_uv_list_1day,
+    p_view_pv_list_1day,
+    p_return_uv_list_1day,
+    share_uv_list_2day,
+    share_pv_list_2day,
+    share_uv_list_3day,
+    share_pv_list_3day,
+    view_uv_list_1h,
+    view_pv_list_1h,
+    play_uv_list_1h,
+    play_pv_list_1h,
+    share_uv_list_1h,
+    share_pv_list_1h,
+    return_uv_list_1h,
+    p_return_uv_list_1h,
+    i_1day_ctr_rt,
+    i_1day_str_rt,
+    i_1day_ros_rt,
+    i_1day_rov_rt,
+    i_1h_ctr_rt,
+    i_1h_str_rt,
+    i_1h_ros_rt,
+    i_1h_rov_rt
     ;
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RankModel.java

@@ -15,14 +15,14 @@ import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.math.NumberUtils;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
 import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Service;
-
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -35,8 +35,6 @@ import java.util.stream.Collectors;
 @Slf4j
 public class RankStrategy4RankModel extends RankService {
 
-    @Value("${video.model.weight:}")
-    private Double mergeWeight;
     @ApolloJsonValue("${video.model.weightv2:}")
     private Map<String, Double> mergeWeightNew;
     final private String CLASS_NAME = this.getClass().getSimpleName();
@@ -61,6 +59,7 @@ public class RankStrategy4RankModel extends RankService {
         rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
         rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
 
+        //-------------------地域内部去重+截断-------------------
         removeDuplicate(rovRecallRank);
         rovRecallRank = rovRecallRank.size() <= param.getSize()
                 ? rovRecallRank
@@ -69,6 +68,7 @@ public class RankStrategy4RankModel extends RankService {
         //-------------------地域 sim returnv2 融合-------------------
         rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
         rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
+        //-------------------地域 sim returnv2 去重-------------------
         removeDuplicate(rovRecallRank);
 
         //-------------------排-------------------
@@ -151,6 +151,10 @@ public class RankStrategy4RankModel extends RankService {
         redisTemplate.setDefaultSerializer(new StringRedisSerializer());
         redisTemplate.afterPropertiesSet();
 
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
         Map<String, String> userFeatureMap = new HashMap<>();
         if (param.getMid() != null && !param.getMid().isEmpty()){
             String midKey = "user_info_4video_" + param.getMid();
@@ -161,8 +165,7 @@ public class RankStrategy4RankModel extends RankService {
                             new TypeToken<Map<String, String>>() {},
                             userFeatureMap);
                 }catch (Exception e){
-                    log.error(String.format("parse user json is wrong in {} with {}",
-                            this.CLASS_NAME, e));
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
             }else{
                 JSONObject obj = new JSONObject();
@@ -172,30 +175,39 @@ public class RankStrategy4RankModel extends RankService {
             }
         }
         final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_ctr_1day","u_str_1day","u_rov_1day","u_ros_1day",
-                "u_3day_exp_cnt","u_3day_click_cnt","u_3day_share_cnt","u_3day_return_cnt",
-                "u_ctr_3day","u_str_3day","u_rov_3day","u_ros_3day"
+            "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+            "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+            "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
         ));
         Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
         while (iterator.hasNext()) {
             Map.Entry<String, String> entry = iterator.next();
             if (!userFeatureSet.contains(entry.getKey())) {
-                // 删除键值对
                 iterator.remove();
             }
         }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                    "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                    "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                    "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
 
-        log.info("userFeature in model = {}", JSONUtils.toJson(userFeatureMap));
-
+        // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
                 "total_time", "play_count_total",
                 "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_ctr_1day", "i_str_1day", "i_rov_1day", "i_ros_1day",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
-                "i_ctr_3day", "i_str_3day", "i_rov_3day", "i_ros_3day"
-        ));
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+                ));
 
         List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
         List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
@@ -215,20 +227,114 @@ public class RankStrategy4RankModel extends RankService {
                     while (iteratorIn.hasNext()) {
                         Map.Entry<String, String> entry = iteratorIn.next();
                         if (!itemFeatureSet.contains(entry.getKey())) {
-                            // 删除键值对
                             iteratorIn.remove();
                         }
                     }
-                    rankItems.get(i).setFeatureMap(vfMap);
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                            "total_time", "play_count_total",
+                            "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                            "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
                 }catch (Exception e){
-                    log.error(String.format("parse video json is wrong in {} with {}",
-                            this.CLASS_NAME, e));
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
             }
         }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPart = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPart != null){
+            if (rtFeaPart.get(0) != null){
+                rtFeaPart1day = rtFeaPart.get(0);
+            }
+            if (rtFeaPart.get(1) != null){
+                rtFeaPart1h = rtFeaPart.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
         log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
 
-        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
 
         List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);

recommend-server-service/src/main/resources/feeds_score_config_baseline.conf

@@ -2,7 +2,7 @@ scorer-config = {
   str-score-config = {
     scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogShareLRScorer"
     scorer-priority = 99
-    model-path = "video_str_model/model_sharev2_20231220_change.txt"
+    model-path = "video_str_model/model_str_mid.txt"
   }
   ros-score-config = {
     scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogShareLRScorer4Ros"