Merge branch 'feature/zhangbo_rank' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -32,6 +32,15 @@ public class RankRouter {
             case "60101":
                 return rankStrategy4RankModel.rank(param);
             case "60098":
+            case "60111":
+            case "60112":
+            case "60113":
+            case "60114":
+            case "60115":
+            case "60116":
+            case "60117":
+            case "60118":
+            case "60119":
                 return rankStrategy4Density.rank(param);
             case "60107":
                 return rankStrategyFlowThompsonModel.rank(param);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -71,6 +71,8 @@ public class RankService {
         List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
         // log.info("mergeAndRankFlowPoolRecall flowPoolRank={}", JSONUtils.toJson(flowPoolRank));
 
+        rankFilter(param, rovRecallRank, flowPoolRank);
+
         removeDuplicate(param, rovRecallRank, flowPoolRank);
 
 //        log.info("removeDuplicate rovRecallRank={}, flowPoolRank={}",
@@ -81,6 +83,10 @@ public class RankService {
         return mergeAndSort(param, rovRecallRank, flowPoolRank);
     }
 
+    public void rankFilter(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank){
+        return;
+    }
+
     public List<Video> mergeAndRankRovRecall(RankParam param) {
         // TODO ab test
         // TODO 抽象成Strategy
@@ -133,6 +139,15 @@ public class RankService {
                 || param.getAbCode().equals("60105")
                 || param.getAbCode().equals("60107")
                 || param.getAbCode().equals("60110")
+                || param.getAbCode().equals("60111")
+                || param.getAbCode().equals("60112")
+                || param.getAbCode().equals("60113")
+                || param.getAbCode().equals("60114")
+                || param.getAbCode().equals("60115")
+                || param.getAbCode().equals("60116")
+                || param.getAbCode().equals("60117")
+                || param.getAbCode().equals("60118")
+                || param.getAbCode().equals("60119")
         ) {
             // 地域召回要做截取，再做融合排序
             removeDuplicate(rovRecallRank);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorTagFilter.java

@@ -0,0 +1,72 @@
+package com.tzld.piaoquan.recommend.server.service.rank.processor;
+import com.tzld.piaoquan.recommend.server.model.Video;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+public class RankProcessorTagFilter {
+
+    public static void tagFitlter(List<Video> rov, List<Video> flow, Map<String, Map<String, String>> rule) {
+
+        Map<String, List<String>> tag2Dates = new HashMap<>();
+//        Map<String, List<String>> tag2Apps = new HashMap<>();
+        Map<String, Integer> tag2Hour = new HashMap<>();
+        for (Map.Entry<String, Map<String, String>> entry : rule.entrySet()){
+            String key = entry.getKey();
+            Map<String, String> value = entry.getValue();
+            if (value.containsKey("date")){
+                tag2Dates.put(key, Arrays.asList(value.get("date").split(",")));
+            }
+//            if (value.containsKey("app")){
+//                tag2Apps.put(key, Arrays.asList(value.get("app").split(",")));
+//            }
+            if (value.containsKey("hour")){
+                tag2Hour.put(key, Integer.valueOf(value.get("hour")));
+            }
+        }
+
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        Integer hour = Integer.valueOf(new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        Iterator<Video> iterator = rov.iterator();
+        while (iterator.hasNext()) {
+            Video video = iterator.next();
+            List<String> tags = video.getTags();
+            boolean filter = false;
+            for (String tag : tags){
+                if (tag2Dates.containsKey(tag) && !tag2Dates.get(tag).isEmpty() && !tag2Dates.get(tag).contains(date)){
+                    // 如果有日期存在，但日期没有命中，则不过滤，跳过即可。
+                    continue;
+                }
+                if (tag2Hour.containsKey(tag) && hour >= tag2Hour.get(tag)){
+                    filter = true;
+                }
+            }
+            if (filter){
+                iterator.remove();
+            }
+        }
+
+        iterator = flow.iterator();
+        while (iterator.hasNext()) {
+            Video video = iterator.next();
+            List<String> tags = video.getTags();
+            boolean filter = false;
+            for (String tag : tags){
+                if (tag2Dates.containsKey(tag) && !tag2Dates.get(tag).isEmpty() && !tag2Dates.get(tag).contains(date)){
+                    // 如果有日期存在，但日期没有命中，则不过滤，跳过即可。
+                    continue;
+                }
+                if (tag2Hour.containsKey(tag) && hour >= tag2Hour.get(tag)){
+                    filter = true;
+                }
+            }
+            if (filter){
+                iterator.remove();
+            }
+        }
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Density.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 
 import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
@@ -9,8 +10,7 @@ import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorFeature;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.SpecialRecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
@@ -27,20 +27,10 @@ import java.util.stream.Collectors;
 @Service
 @Slf4j
 public class RankStrategy4Density extends RankService {
-
+    @ApolloJsonValue("${RankStrategy4DensityFilter:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules;
+    @Override
     public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
-        // 测试，添加数据
-//        rovRecallRank.add(0, getTestVideo(1070462L, ""));
-//        rovRecallRank.add(0, getTestVideo(1085062L, ""));
-//        rovRecallRank.add(0, getTestVideo(1102552L, ""));
-//        rovRecallRank.add(0, getTestVideo(1115929L, ""));
-//
-//        flowPoolRank = new ArrayList<>();
-//        flowPoolRank.add(getTestVideo(1263931L, "flow"));
-//        flowPoolRank.add(getTestVideo(1284797L, "flow"));
-//        flowPoolRank.add(getTestVideo(1289001L, "flow"));
-//        flowPoolRank.add(getTestVideo(1331503L, "flow"));
-
 
         if (CollectionUtils.isEmpty(rovRecallRank)) {
             if (param.getSize() < flowPoolRank.size()) {
@@ -49,31 +39,44 @@ public class RankStrategy4Density extends RankService {
                 return new RankResult(flowPoolRank);
             }
         }
-        // 1 读取多样性密度控制规则
-        String appType = String.valueOf(param.getAppType());
-        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
+        // 1 读取多样性密度控制规则------------------
+//        String appType = String.valueOf(param.getAppType());
+//        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
+//        Map<String, Integer> densityRules = new HashMap<>();
+//        if (ruleStr != null){
+//            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
+//                    new TypeToken<Map<String, Map<String, Object>>>() {},
+//                    Collections.emptyMap());
+//            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
+//                String k = entry.getKey();
+//                if (!entry.getValue().containsKey(appType)){
+//                    continue;
+//                }
+//                JSONObject jb = (JSONObject) entry.getValue().get(appType);
+//                try{
+//                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
+//                        densityRules.put(k, jb.getInteger("density"));
+//                    }
+//                }catch (Exception e){
+//                    log.error("parse densityRules is wrong:", e);
+//                }
+//            }
+//        }
         Map<String, Integer> densityRules = new HashMap<>();
-        if (ruleStr != null){
-            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
-                    new TypeToken<Map<String, Map<String, Object>>>() {},
-                    Collections.emptyMap());
-            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
-                String k = entry.getKey();
-                if (!entry.getValue().containsKey(appType)){
-                    continue;
-                }
-                JSONObject jb = (JSONObject) entry.getValue().get(appType);
-                try{
-                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
-                        densityRules.put(k, jb.getInteger("density"));
-                    }
-                }catch (Exception e){
-                    log.error("parse densityRules is wrong:", e);
+        String abCode = param.getAbCode();
+        if (this.filterRules != null && this.filterRules.containsKey(abCode)){
+            Map<String, Map<String, String>> rule = this.filterRules.get(abCode);
+            for (Map.Entry<String, Map<String, String>> entry : rule.entrySet()){
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")){
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
                 }
-
             }
         }
-        // 2 读取video的tags
+
+
+        // 2 读取video的tags------------------
         List<Long> videoIds = new ArrayList<>();
         for (Video v : rovRecallRank) {
             videoIds.add(v.getVideoId());
@@ -88,9 +91,14 @@ public class RankStrategy4Density extends RankService {
         for (Video v : flowPoolRank) {
             v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
         }
-        // ------------------读取video的tags完成---------------------
 
+        // 3 读取过滤规则，根据tag和过滤规则进行过滤------------------
+        if (this.filterRules != null && this.filterRules.containsKey(abCode)){
+            Map<String, Map<String, String>> rule = this.filterRules.get(abCode);
+            RankProcessorTagFilter.tagFitlter(rovRecallRank, flowPoolRank, rule);
+        }
 
+        // 4 流量池按比例强插---------------------
         List<Video> result = new ArrayList<>();
         for (int i = 0; i < param.getTopK() && i < rovRecallRank.size(); i++) {
             result.add(rovRecallRank.get(i));
@@ -128,15 +136,19 @@ public class RankStrategy4Density extends RankService {
             }
         }
 
-        // 3 进行密度控制
+        // 3 进行密度控制------------------
         Set<Long> videosSet = result.stream().map(r-> r.getVideoId()).collect(Collectors.toSet());
         List<Video> rovRecallRankNew = rovRecallRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
         List<Video> flowPoolRankNew = flowPoolRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
         List<Video> resultWithDnsity = RankProcessorDensity.mergeDensityControl(result,
                 rovRecallRankNew, flowPoolRankNew, densityRules);
+//        log.info("zhangbo22 old={}, new={}", JSONUtils.toJson(result),
+//                    JSONUtils.toJson(resultWithDnsity));
         return new RankResult(resultWithDnsity);
     }
 
+
+
 //    public Video getTestVideo(Long id, String s){
 //        Video a1 = new Video();
 //        a1.setVideoId(id);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -110,6 +110,15 @@ public class RecallService implements ApplicationContextAware {
             ;
         } else {
             switch (abCode) {
+                case "60111":
+                case "60112":
+                case "60113":
+                case "60114":
+                case "60115":
+                case "60116":
+                case "60117":
+                case "60118":
+                case "60119":
                 case "60096":
                     strategies.add(strategyMap.get(RegionHWithoutDupRecallStrategy.class.getSimpleName()));
                     strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/DahanRecallStrategy.java

@@ -0,0 +1,72 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.google.common.collect.Lists;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.util.DateUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.springframework.data.redis.core.ZSetOperations;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * 不区分地域小时（不做离线去重）
+ *
+ * @author dyp
+ */
+@Service
+public class DahanRecallStrategy extends AbstractRegionRecallStrategy {
+    public static final String PUSH_FORM = "dahan_recall";
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> results = new ArrayList<>();
+
+        List<Long> ss = new ArrayList<>(Arrays.asList(10409992L, 10436156L, 18160019L, 18247521L, 18254840L));
+        ss.stream().forEach(vid -> {
+            Video video = new Video();
+            video.setVideoId(vid);
+            video.setAbCode(param.getAbCode());
+            video.setRovScore(Math.random());
+            video.setPushFrom(pushFrom());
+            video.setLastVideoKey("zhangbo");
+            results.add(video);
+        });
+
+
+
+        Collections.sort(results, Comparator.comparingDouble(o -> -o.getRovScore()));
+
+        return results;
+    }
+    @Override
+    protected String recordKey(RecallParam param) {
+        return String.format("recall:last:record:h:%s:%s", param.getAppType(), param.getMid());
+    }
+
+    @Override
+    protected String poolKey(RecallParam param, String now_dt, int h) {
+        // hard code  因为生产任务，会给region填充默认值-1
+        return String.format("recall:item:score:h:%s:%s:%s:%s",
+                param.getDataKey(), param.getRuleKey(), now_dt, h);
+    }
+
+    @Override
+    protected String lastVideoKey(RecallParam param) {
+        return String.format("LAST_VIDEO_KEY_FORMAT", param.getAppType(), param.getMid());
+    }
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+}