屏蔽过滤

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/RecommendParam.java

@@ -4,6 +4,10 @@ import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 /**
  * @author dyp
  */
@@ -27,4 +31,11 @@ public class RecommendParam {
     private String mid;
     private int size;
     private String uid;
+
+    // 风险过滤
+    private int expansionFactor;
+    private Boolean riskFilterFlag;
+    private Map<Integer, List<String>> appRegionFiltered;
+    private List<Long> videosWithRisk;
+    private String regionCode;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -35,6 +35,7 @@ import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
+import com.tzld.piaoquan.recommend.server.util.ParserUtils;
 
 /**
  * @author dyp
@@ -270,6 +271,41 @@ public class RecommendService {
             }
         }
 
+        // 风险过滤
+        String RISK_SHIELD_FILTER_RULE_V1_JSON = redisTemplate.opsForValue().get("RISK_SHIELD_FILTER_RULE_V1_JSON");
+        Map<Integer, List<String>> appRegionFiltered = ParserUtils.parseJsonForRiskRule(RISK_SHIELD_FILTER_RULE_V1_JSON);
+        String RISK_SHIELD_FILTER_VIDEO_V1_STR = redisTemplate.opsForValue().get("RISK_SHIELD_FILTER_VIDEO_V1_STR");
+        List<Long> videosWithRisk = ParserUtils.parseJsonForRiskVideos(RISK_SHIELD_FILTER_VIDEO_V1_STR);
+        String RISK_SHIELD_FILTER_EXPANSION_FACTOR_INT = redisTemplate.opsForValue().get("RISK_SHIELD_FILTER_EXPANSION_FACTOR_INT");
+        int expansionFactor = 5;
+        if (RISK_SHIELD_FILTER_EXPANSION_FACTOR_INT != null) {
+            try {
+                expansionFactor = Integer.parseInt(RISK_SHIELD_FILTER_EXPANSION_FACTOR_INT);
+            } catch (NumberFormatException e) {
+                expansionFactor = 5;
+            }
+        }
+        // 容灾
+        if (expansionFactor < 5) {
+            expansionFactor = 5;
+        } else if (expansionFactor > 25) {
+            expansionFactor = 25;
+        }
+        String RISK_SHIELD_FILTER_FLAG_BOOL = redisTemplate.opsForValue().get("RISK_SHIELD_FILTER_FLAG_BOOL");
+        Boolean riskFilterFlag = false;
+        if (RISK_SHIELD_FILTER_FLAG_BOOL != null) {
+            try {
+                riskFilterFlag = Boolean.parseBoolean(RISK_SHIELD_FILTER_FLAG_BOOL.toLowerCase());
+            } catch (Exception e) {
+                riskFilterFlag = false;
+            }
+        }
+        param.setAppRegionFiltered(appRegionFiltered);
+        param.setVideosWithRisk(videosWithRisk);
+        param.setExpansionFactor(expansionFactor);
+        param.setRiskFilterFlag(riskFilterFlag);
+
+
         return param;
     }
 
@@ -328,6 +364,12 @@ public class RecommendService {
         recallParam.setMid(param.getMid());
         recallParam.setSize(param.getSize());
         recallParam.setUid(param.getUid());
+        //风险过滤
+        recallParam.setExpansionFactor(param.getExpansionFactor());
+        recallParam.setRiskFilterFlag(param.getRiskFilterFlag());
+        recallParam.setVideosWithRisk(param.getVideosWithRisk());
+        recallParam.setAppRegionFiltered(param.getAppRegionFiltered());
+
         return recallParam;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -1,6 +1,7 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
 import com.google.common.base.Stopwatch;
+import com.thoughtworks.xstream.mapper.Mapper;
 import com.tzld.piaoquan.recommend.server.service.PreViewedService;
 import com.tzld.piaoquan.recommend.server.service.ViewedService;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
@@ -9,7 +10,9 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -18,15 +21,23 @@ import java.util.stream.Collectors;
  */
 @Slf4j
 public abstract class AbstractFilterService implements FilterService {
+    public Integer forceTruncation;
     @Autowired
     private PreViewedService preViewedService;
 
     @Autowired
     private ViewedService viewedService;
 
+
+
     protected List<Long> viewFilter(FilterParam param) {
         log.info("filterParam={}", JSONUtils.toJson(param));
-        List<Long> videoIds = filterByPreViewed(param.getAppType(), param.getMid(), param.getVideoIds());
+        // 风险过滤
+        List<Long> videoIds = filterWithRiskVideo(param.getRiskFilterFlag(),
+                param.getAppType(), param.getRegionCode(), param.getAppRegionFiltered(), param.getVideosWithRisk(),
+                param.getVideoIds());
+        log.info("filterByRiskVideos videoIds={}", JSONUtils.toJson(videoIds));
+        videoIds = filterByPreViewed(param.getAppType(), param.getMid(), param.getVideoIds());
         log.info("filterByPreViewed videoIds={}", JSONUtils.toJson(videoIds));
         videoIds = filterByViewed(param.getAppType(), param.getMid(), param.getUid(), videoIds);
         log.info("filterByViewed videoIds={}", JSONUtils.toJson(videoIds));
@@ -56,4 +67,43 @@ public abstract class AbstractFilterService implements FilterService {
                 .collect(Collectors.toList());
 
     }
+
+    private List<Long> filterWithRiskVideo(boolean riskFlag,
+                                           int appType,
+                                           String regionCode,
+                                           Map<Integer, List<String>> rules,
+                                           List<Long> videosWithRisk,
+                                           List<Long> videoIds){
+        if (!riskFlag){
+            return this.truncation(videoIds);
+        }
+        // 1 判断是否过滤,不展示的app+区域列表。
+        boolean filterFlag;
+        if (rules.containsKey(appType)){
+            filterFlag = false;
+            if (rules.get(appType).contains(regionCode)){
+                filterFlag = true;
+            }
+        } else {
+            filterFlag = true;
+        }
+        if (!filterFlag){
+            return this.truncation(videoIds);
+        }
+        // 2 开始过滤。
+        List<Long> videoIdNew = new ArrayList<>();
+        for (Long videoId : videoIds) {
+            if (!videosWithRisk.contains(videoId)) {
+                videoIdNew.add(videoId);
+            }
+        }
+        return this.truncation(videoIdNew);
+    }
+    private List<Long> truncation(List<Long> videoIds){
+        if (this.forceTruncation == null){
+            return videoIds;
+        }else{
+            return videoIds.subList(0, Math.min(this.forceTruncation, videoIds.size()));
+        }
+    }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterParam.java

@@ -15,4 +15,10 @@ public class FilterParam {
     private int appType;
     private String mid;
     private String uid;
+
+    // 风险过滤
+    private Boolean riskFilterFlag;
+    private Map<Integer, List<String>> appRegionFiltered;
+    private List<Long> videosWithRisk;
+    private String regionCode;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/RegionFilterService.java

@@ -1,5 +1,7 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
+import lombok.Getter;
+import lombok.Setter;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
@@ -9,6 +11,11 @@ import java.util.List;
  */
 @Service
 public class RegionFilterService extends AbstractFilterService {
+
+    public RegionFilterService(Integer forceTruncation){
+        this.forceTruncation = forceTruncation;
+    }
+    public RegionFilterService(){}
     public FilterResult filter(FilterParam param) {
         List<Long> videoIds = viewFilter(param);
         return new FilterResult(videoIds);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/FilterParamFactory.java

@@ -17,6 +17,12 @@ public class FilterParamFactory {
         filterParam.setMid(param.getMid());
         filterParam.setUid(param.getUid());
         filterParam.setFlowPoolMap(videoFlowPoolMap);
+
+        // 风险过滤
+        filterParam.setRiskFilterFlag(param.getRiskFilterFlag());
+        filterParam.setVideosWithRisk(param.getVideosWithRisk());
+        filterParam.setAppRegionFiltered(param.getAppRegionFiltered());
+        filterParam.setRegionCode(param.getRegionCode());
         return filterParam;
     }
 
@@ -26,6 +32,11 @@ public class FilterParamFactory {
         filterParam.setAppType(param.getAppType());
         filterParam.setMid(param.getMid());
         filterParam.setUid(param.getUid());
+        // 风险过滤
+        filterParam.setRiskFilterFlag(param.getRiskFilterFlag());
+        filterParam.setVideosWithRisk(param.getVideosWithRisk());
+        filterParam.setAppRegionFiltered(param.getAppRegionFiltered());
+        filterParam.setRegionCode(param.getRegionCode());
         return filterParam;
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallParam.java

@@ -2,6 +2,9 @@ package com.tzld.piaoquan.recommend.server.service.recall;
 
 import lombok.Data;
 
+import java.util.List;
+import java.util.Map;
+
 /**
  * @author dyp
  */
@@ -18,5 +21,9 @@ public class RecallParam {
     private Long videoId;
     private String uid;
     private boolean specialRecommend;
+    private int expansionFactor;
+    private Boolean riskFilterFlag;
+    private Map<Integer, List<String>> appRegionFiltered;
+    private List<Long> videosWithRisk;
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractRegionRecallStrategy.java

@@ -107,7 +107,8 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
         }
 
         // 召回
-        int getSize = param.getSize() * 5;
+        // 风险过滤 扩大因子
+        int getSize = param.getSize() * param.getExpansionFactor();
         int freq = 0;
         String lastVideoId = "";
         List<Video> results = new ArrayList<>();
@@ -128,7 +129,7 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
                 lastVideoId = t.getValue();
                 videoMap.put(NumberUtils.toLong(t.getValue(), 0), t.getScore());
             }
-
+            filterService.forceTruncation = 20;
             FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet())));
             log.info("recall filterResult={}", JSONUtils.toJson(filterResult));
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/ParserUtils.java

@@ -0,0 +1,52 @@
+package com.tzld.piaoquan.recommend.server.util;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.List;
+public class ParserUtils {
+
+    public static Map<Integer, List<String>> parseJsonForRiskRule(String s) {
+        ObjectMapper objectMapper = new ObjectMapper();
+        Map<Integer, List<String>> dataNew = new HashMap<>();
+        try {
+            Map<String, List<String>> data = objectMapper.readValue(s, Map.class);
+            for (Map.Entry<String, List<String>> entry : data.entrySet()) {
+                int key = Integer.parseInt(entry.getKey());
+                List<String> value = entry.getValue();
+                dataNew.put(key, value);
+            }
+        } catch (JsonParseException | JsonMappingException e) {
+            System.err.println("Error parsing JSON: " + e.getMessage());
+        } catch (IOException e) {
+            System.err.println("IO Exception: " + e.getMessage());
+        }
+
+        return dataNew;
+    }
+
+    public static List<Long> parseJsonForRiskVideos(String s) {
+        List<Long> dataNew = new ArrayList<>();
+
+        try {
+            String[] data = s.split(",");
+            for (String item : data) {
+                Long value = Long.parseLong(item);
+                dataNew.add(value);
+            }
+        } catch (NumberFormatException e) {
+            System.err.println("Error parsing string: " + e.getMessage());
+        }
+
+        return dataNew;
+    }
+
+    public static void main(String[] args) {
+        String json = "{\"VLOG\": [\"北京\"]}";
+
+    }
+}