后处理规则

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/RankExtractorItemTags.java

@@ -0,0 +1,51 @@
+package com.tzld.piaoquan.recommend.server.service.rank.extractor;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import org.springframework.data.redis.core.RedisTemplate;
+
+import java.util.*;
+
+public class RankExtractorItemTags {
+    public RedisTemplate<String, String> redisTemplate;
+    public RankExtractorItemTags(RedisTemplate<String, String> redisTemplate){
+        this.redisTemplate = redisTemplate;
+    }
+
+    public void processor(List<Video> rovVideos, List<Video> flowVideos){
+        List<Long> videoIds = new ArrayList<>();
+        for (Video v : rovVideos) {
+            videoIds.add(v.getVideoId());
+        }
+        for (Video v : flowVideos) {
+            videoIds.add(v.getVideoId());
+        }
+        Map<Long, List<String>> videoTagDict = getVideoTags(redisTemplate, videoIds);
+        for (Video v : rovVideos) {
+            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        }
+        for (Video v : flowVideos) {
+            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        }
+    }
+
+    public static Map<Long, List<String>> getVideoTags(RedisTemplate<String, String> redisHelper, List<Long> videoIds) {
+        String REDIS_PREFIX = "alg_recsys_video_tags_";
+        List<String> redisKeys = new ArrayList<>();
+        for (Long videoId : videoIds) {
+            redisKeys.add(REDIS_PREFIX + String.valueOf(videoId));
+        }
+        List<String> videoTags = redisHelper.opsForValue().multiGet(redisKeys);
+        Map<Long, List<String>> videoTagDict = new HashMap<>();
+        if (videoTags != null) {
+            for (int i = 0; i < videoTags.size(); i++) {
+                String tagsStr = videoTags.get(i);
+                List<String> tags = new ArrayList<>();
+                if (tagsStr != null && !tagsStr.isEmpty()) {
+                    String[] tagsArray = tagsStr.split(",");
+                    tags = new ArrayList<>(Arrays.asList(tagsArray));
+                }
+                videoTagDict.put(videoIds.get(i), tags);
+            }
+        }
+        return videoTagDict;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Density.java

@@ -11,7 +11,7 @@ import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorFeature;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
@@ -21,132 +21,105 @@ import java.util.*;
 import java.util.stream.Collectors;
 
 /**
- * @author zhangbo
+ * @author zhangbo sunxiaoyi
- * @desc 带密度控制的后处理 排序实验
+ * @desc 后处理规则 + roc池flow池的合并
+ * 后处理参考文档：https://w42nne6hzg.feishu.cn/wiki/MYaGwCnF1iTFXUkSddAcA6CanFe
+ * roc池flow池的合并 文档： 暂无
  */
 @Service
 @Slf4j
 public class RankStrategy4Density extends RankService {
     @ApolloJsonValue("${RankStrategy4DensityFilter:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules;
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
 
-        if (CollectionUtils.isEmpty(rovRecallRank)) {
+         //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-            if (param.getSize() < flowPoolRank.size()) {
+        if (CollectionUtils.isEmpty(rovVideos)) {
-                return new RankResult(flowPoolRank.subList(0, param.getSize()));
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
             } else {
-                return new RankResult(flowPoolRank);
+                return new RankResult(flowVideos);
-            }
-        }
-        // 1 读取多样性密度控制规则------------------
-//        String appType = String.valueOf(param.getAppType());
-//        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
-//        Map<String, Integer> densityRules = new HashMap<>();
-//        if (ruleStr != null){
-//            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
-//                    new TypeToken<Map<String, Map<String, Object>>>() {},
-//                    Collections.emptyMap());
-//            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
-//                String k = entry.getKey();
-//                if (!entry.getValue().containsKey(appType)){
-//                    continue;
-//                }
-//                JSONObject jb = (JSONObject) entry.getValue().get(appType);
-//                try{
-//                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
-//                        densityRules.put(k, jb.getInteger("density"));
-//                    }
-//                }catch (Exception e){
-//                    log.error("parse densityRules is wrong:", e);
-//                }
-//            }
-//        }
-        Map<String, Integer> densityRules = new HashMap<>();
-        String abCode = param.getAbCode();
-        if (this.filterRules != null && this.filterRules.containsKey(abCode)){
-            Map<String, Map<String, String>> rule = this.filterRules.get(abCode);
-            for (Map.Entry<String, Map<String, String>> entry : rule.entrySet()){
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")){
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
             }
         }
 
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
 
-        // 2 读取video的tags------------------
+        //3 标签读取
-        List<Long> videoIds = new ArrayList<>();
+        if (rulesMap != null && !rulesMap.isEmpty()){
-        for (Video v : rovRecallRank) {
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            videoIds.add(v.getVideoId());
+            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        for (Video v : flowPoolRank) {
-            videoIds.add(v.getVideoId());
-        }
-        Map<Long, List<String>> videoTagDict = RankExtractorFeature.getVideoTags(this.redisTemplate, videoIds);
-        for (Video v : rovRecallRank) {
-            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
         }
-        for (Video v : flowPoolRank) {
+        //6 合并结果时间卡控
-            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.tagFitlter(rovVideos, flowVideos, rulesMap);
         }
 
-        // 3 读取过滤规则，根据tag和过滤规则进行过滤------------------
+        //4 rov池提权功能
-        if (this.filterRules != null && this.filterRules.containsKey(abCode)){
+
-            Map<String, Map<String, String>> rule = this.filterRules.get(abCode);
+        //5 rov池强插功能
-            RankProcessorTagFilter.tagFitlter(rovRecallRank, flowPoolRank, rule);
-        }
 
-        // 4 流量池按比例强插---------------------
+        //7 流量池按比例强插
         List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovRecallRank.size(); i++) {
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovRecallRank.get(i));
+            result.add(rovVideos.get(i));
         }
-
         double flowPoolP = getFlowPoolP(param);
         int flowPoolIndex = 0;
         int rovPoolIndex = param.getTopK();
-
         for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
             double rand = RandomUtils.nextDouble(0, 1);
             log.info("rand={}, flowPoolP={}", rand, flowPoolP);
             if (rand < flowPoolP) {
-                if (flowPoolIndex < flowPoolRank.size()) {
+                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowPoolRank.get(flowPoolIndex++));
+                    result.add(flowVideos.get(flowPoolIndex++));
                 } else {
                     break;
                 }
             } else {
-                if (rovPoolIndex < rovRecallRank.size()) {
+                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovRecallRank.get(rovPoolIndex++));
+                    result.add(rovVideos.get(rovPoolIndex++));
                 } else {
                     break;
                 }
             }
         }
-        if (rovPoolIndex >= rovRecallRank.size()) {
+        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowPoolRank.size() && result.size() < param.getSize(); i++) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowPoolRank.get(i));
+                result.add(flowVideos.get(i));
             }
         }
-        if (flowPoolIndex >= flowPoolRank.size()) {
+        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovRecallRank.size() && result.size() < param.getSize(); i++) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovRecallRank.get(i));
+                result.add(rovVideos.get(i));
             }
         }
 
-        // 3 进行密度控制------------------
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
         Set<Long> videosSet = result.stream().map(r-> r.getVideoId()).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovRecallRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowPoolRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
         List<Video> resultWithDnsity = RankProcessorDensity.mergeDensityControl(result,
                 rovRecallRankNew, flowPoolRankNew, densityRules);
-//        log.info("zhangbo22 old={}, new={}", JSONUtils.toJson(result),
+
-//                    JSONUtils.toJson(resultWithDnsity));
         return new RankResult(resultWithDnsity);
     }
 
+}
+
+
 
 
 //    public Video getTestVideo(Long id, String s){
@@ -156,6 +129,26 @@ public class RankStrategy4Density extends RankService {
 //        return a1;
 //    }
 
-
+// 1 读取多样性密度控制规则------------------
-
+//        String appType = String.valueOf(param.getAppType());
-}
+//        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
+//        Map<String, Integer> densityRules = new HashMap<>();
+//        if (ruleStr != null){
+//            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
+//                    new TypeToken<Map<String, Map<String, Object>>>() {},
+//                    Collections.emptyMap());
+//            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
+//                String k = entry.getKey();
+//                if (!entry.getValue().containsKey(appType)){
+//                    continue;
+//                }
+//                JSONObject jb = (JSONObject) entry.getValue().get(appType);
+//                try{
+//                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
+//                        densityRules.put(k, jb.getInteger("density"));
+//                    }
+//                }catch (Exception e){
+//                    log.error("parse densityRules is wrong:", e);
+//                }
+//            }
+//        }