并行获取特征

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/FeatureService.java

@@ -22,7 +22,7 @@ public class FeatureService {
     private FeatureV2RemoteService remoteService;
 
     /**
-     * @return k1:视频、k2：表、k3：特征、v：值
+     * @return k1:视频、k2:表、k3:特征、v:特征值
      */
     public Map<String, Map<String, Map<String, String>>> getFeature(List<String> vidList, String apptype,
                                                                     String province) {
@@ -52,6 +52,7 @@ public class FeatureService {
             Map<String, Map<String, String>> tableFeatureMap = data.getOrDefault(vid, new HashMap<>());
             tableFeatureMap.put(table, JSONUtils.fromJson(feature, new TypeToken<Map<String, String>>() {
             }, Collections.emptyMap()));
+            data.put(vid, tableFeatureMap);
         });
 
         return data;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV999.java

@@ -1,9 +1,9 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
@@ -14,29 +14,25 @@ import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDe
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
 
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV999 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv567:}")
-    private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
     private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
+
+    @Autowired
+    private FeatureService featureService;
 
     public void duplicate(Set<Long> setVideo, List<Video> videos) {
         Iterator<Video> iterator = videos.iterator();
@@ -100,72 +96,42 @@ public class RankStrategy4RegionMergeModelV999 extends RankService {
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_uv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+        // TODO 1 批量获取特征  省份参数要对齐
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+        // k1:视频、k2:表、k3:特征、v:特征值
+        Map<String, Map<String, Map<String, String>>> featureMap = featureService.getFeature(vids,
+                String.valueOf(param.getAppType()), param.getProvince());
 
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 0.0, 0.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
+        // TODO 2 特征处理
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        Map<String, String> videoFeatureMap = extrctVideoFeature(featureMap);
+        rankItems.set();
+        Map<String, String> sceneFeatureMap = extractSceneFeature(featureMap);
+        Map<String, String> userFeatureMap = extractUserFeature(featureMap);
 
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
 
+        // TODO 3 排序
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+
+        // TODO
 
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
-        for (RankItem item : items) {
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double score = 0.0;
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > 50) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            }else{
-                score += (f * share2allreturnScore + g * view2allreturnScore) * 0.01;
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
         return result;
     }
 
+    private Map<String, String> extrctVideoFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+    }
+
+    private Map<String, String> extractSceneFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+    }
+
+    private Map<String, String> extractUserFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+    }
+
+
     public Double calScoreWeightNoTimeDecay(List<Double> data) {
         Double up = 0.0;
         Double down = 0.0;
@@ -202,60 +168,6 @@ public class RankStrategy4RegionMergeModelV999 extends RankService {
         return views;
     }
 
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-
-        // 2-2: item 实时特征处理
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys2);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-
-        return rankItems;
-    }
-
     @Override
     public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
 
@@ -343,43 +255,4 @@ public class RankStrategy4RegionMergeModelV999 extends RankService {
         return new RankResult(resultWithDensity);
     }
 
-    public static void main(String[] args) {
-//        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
-        String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
-        String down1 = "2024031012:2019,2024031013:1676,2024031014:1626,2024031015:1458,2024031016:1508,2024031017:1510,2024031018:1713,2024031019:1972,2024031020:2500,2024031021:2348,2024031022:2061,2024031023:1253,2024031100:659,2024031101:243,2024031102:191,2024031103:282,2024031104:246,2024031105:439,2024031106:1079,2024031107:1911,2024031108:2023,2024031109:1432,2024031110:1632,2024031111:1183,2024031112:1024,2024031113:938,2024031114:701,2024031115:541";
-
-//        String up2 = "2024031012:215,2024031013:242,2024031014:166,2024031015:194,2024031016:209,2024031017:245,2024031018:320,2024031019:332,2024031020:400,2024031021:375,2024031022:636,2024031023:316,2024031100:167,2024031101:45,2024031102:22,2024031103:26,2024031104:12,2024031105:22,2024031106:24,2024031107:143,2024031108:181,2024031109:199,2024031110:194,2024031111:330,2024031112:423,2024031113:421,2024031114:497,2024031115:424";
-        String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
-        String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
-
-        RankStrategy4RegionMergeModelV999 job = new RankStrategy4RegionMergeModelV999();
-        List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeightNoTimeDecay(l1);
-
-        System.out.println(d1);
-
-        List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeightNoTimeDecay(l2);
-
-        System.out.println(d2);
-
-    }
-
-    List<Double> help(String s, String date, Integer h) {
-        Map<String, Double> maps = Arrays.stream(s.split(",")).map(pair -> pair.split(":"))
-                .collect(Collectors.toMap(
-                        arr -> arr[0],
-                        arr -> Double.valueOf(arr[1])
-                ));
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        List<Double> result = new ArrayList<>();
-        for (int i = 0; i < h; ++i) {
-            Double d = (result.isEmpty() ? 0.0 : result.get(result.size() - 1));
-            result.add(d + maps.getOrDefault(date, 0D));
-            datehours.add(date);
-            date = ExtractorUtils.subtractHours(date, 1);
-        }
-        return result;
-    }
-
 }