feat:修改SecurityStrategy过滤逻辑

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/BlacklistContainer.java

@@ -0,0 +1,285 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
+import lombok.Data;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Nonnull;
+import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * 黑名单列表相关的容器。主要实现以下几个功能
+ * <ul>
+ *     <ol>1. 判断用户属于哪个类型的黑名单</ol>
+ *     <ol>2. 根据用户类型判断视频对于该用户是否有风险</ol>
+ *     <ol>3. 根据用户类型过滤掉视频列表中对该用户有风险的视频</ol>
+ * </ul>
+ */
+@Component
+public class BlacklistContainer {
+
+
+    private static final Logger LOG = LoggerFactory.getLogger(BlacklistContainer.class);
+
+    public static final String USER_TYPE_SUB_TYPE_CONNECTOR = ":";
+
+    private static final int USER_REDIS_KEY_PARTITION_COUNT = 100;
+
+    /**
+     * 用户访问黑名单 Redis Key
+     */
+    private static final String USER_VISIO_BLACKLIST_HASH_KEY = "visio:blacklist:user:";
+
+    /**
+     * IP访问黑名单 Redis Key
+     */
+    private static final String IP_VISIO_BLACKLIST_HASH_KEY = "visio:blacklist:ip";
+
+    @Autowired
+    @Qualifier("longVideoRedisTemplate")
+    private RedisTemplate<String, String> longVideoRedisTemplate;
+
+    @ApolloJsonValue("${content.security.generalization.user.condition.config:}")
+    private Map<String, GeneralizationUserConfig> generalizationUserConditionConfig;
+
+    @Resource
+    private WxVideoTagRelRepository wxVideoTagRelRepository;
+
+    /**
+     * 不同类型的用户要过滤掉的标签列表配置
+     * <br >
+     * <p>
+     * Key的格式为: {userType}:{userSubType}
+     * <br>
+     * userType枚举值: 1-竞品用户, 2-微信人员, 3-网安
+     * <br >
+     * userSubType枚举值: 1-精准用户, 2-泛化用户
+     */
+    @ApolloJsonValue("${content.security.filter.config:{}}")
+    private Map<String, TagFilterConfig> tagFilterConfigMap;
+
+    @ApolloJsonValue("${content.security.user.type.priority.config:{}}")
+    private Map<String, List<String>> userTypePriorityConfigMap;
+
+    /**
+     * 保存Tag标签与视频列表的映射
+     * <br>
+     * Key为TagId,  Value为对应的视频ID列表
+     */
+    private static Map<Long, Set<Long>> videoTagCache = new ConcurrentHashMap<>();
+
+    /**
+     * 黑名单本地二级缓存，一级缓存为Redis缓存。此处直接读取即可
+     * <br />
+     * Redis缓存由longvideo服务写入
+     * <br>
+     * com.weiqu.video.service.filter.impl.BlacklistFilterImpl#refreshUidCache
+     * <br>
+     * com.weiqu.video.service.filter.impl.BlacklistFilterImpl#refreshIPCache
+     */
+    private final LoadingCache<String, Map<String, String>> blacklistCache = CacheBuilder.newBuilder()
+            .expireAfterWrite(10, TimeUnit.MINUTES)
+            .build(new CacheLoader<String, Map<String, String>>() {
+                @Override
+                public Map<String, String> load(@Nonnull String key) {
+                    Map<Object, Object> map = longVideoRedisTemplate.opsForHash().entries(key);
+                    if (MapUtils.isEmpty(map)) {
+                        return new HashMap<>();
+                    }
+                    return map.entrySet().stream().collect(
+                            Collectors.toMap(
+                                    entry -> entry.getKey().toString(),
+                                    entry -> entry.getValue().toString(),
+                                    (v1, v2) -> v1
+                            ));
+                }
+            });
+
+    @PostConstruct
+    public void init() {
+        refreshVideoTagCache();
+    }
+
+    @Scheduled(cron = "0 0/5 0/1 * * ? ")
+    public void cronSync() {
+        refreshVideoTagCache();
+    }
+
+    public void refreshVideoTagCache() {
+        LOG.info("同步本地标签ID与视频列表的缓存任务开始");
+        Map<Long, Set<Long>> tmpMap = new ConcurrentHashMap<>();
+
+        if (MapUtils.isNotEmpty(tagFilterConfigMap)) {
+
+            // 获取所有的标签ID列表
+            Set<Long> tagIdSet = new HashSet<>();
+            for (Map.Entry<String, TagFilterConfig> entry : tagFilterConfigMap.entrySet()) {
+                TagFilterConfig tagFilterConfig = entry.getValue();
+                if (Objects.isNull(tagFilterConfig)) {
+                    continue;
+                }
+                if (CollectionUtils.isNotEmpty(tagFilterConfig.getRecommendExcludeTag())) {
+                    tagIdSet.addAll(tagFilterConfig.getRecommendExcludeTag());
+                }
+                if (CollectionUtils.isNotEmpty(tagFilterConfig.getDetailExcludeTag())) {
+                    tagIdSet.addAll(tagFilterConfig.getDetailExcludeTag());
+                }
+            }
+
+            // 获取标签ID对应的视频ID列表
+            for (Long tagId : tagIdSet) {
+                List<WxVideoTagRel> wxVideoTagRels = wxVideoTagRelRepository.findAllByTagId(tagId);
+                Set<Long> videoIdSet = wxVideoTagRels.stream().map(WxVideoTagRel::getVideoId).collect(Collectors.toSet());
+                tmpMap.put(tagId, videoIdSet);
+            }
+        }
+        videoTagCache = tmpMap;
+        LOG.info("同步本地标签ID与视频列表的缓存任务结束");
+    }
+
+    public List<Long> filterUnsafeVideoByUser(List<Long> videoIds, String uid, Long hotSceneType, String cityCode, String clientIP) {
+        if (CollectionUtils.isEmpty(videoIds)) {
+            return videoIds;
+        }
+
+        String userType = this.matchUserBlacklistTypeEnum(uid, hotSceneType, cityCode, clientIP);
+        Collection<Long> tagIdSet = this.findExcludeTagIds(userType);
+        if (CollectionUtils.isEmpty(tagIdSet)) {
+            return videoIds;
+        }
+
+        return videoIds.stream().filter(videoId -> {
+            if (videoTagAnyMatch(videoId, tagIdSet)) {
+                LOG.error("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}",
+                        uid, userType, videoId, hotSceneType, cityCode, clientIP);
+                return false;
+            }
+            return true;
+        }).collect(Collectors.toList());
+    }
+
+    private String matchUserBlacklistTypeEnum(String uid, Long hotSceneType, String cityCode, String clientIP) {
+        try {
+            LOG.info("计算用户黑名单类型，判断参数: uid={}, hotSceneType={}, cityCode={}, clientIP={}", uid, hotSceneType, cityCode, clientIP);
+            String key = this.calcUserRedisKey(uid);
+            Map<String, String> uidBlacklistMap = blacklistCache.get(key);
+            if (uidBlacklistMap.containsKey(uid)) {
+                String userType = uidBlacklistMap.get(uid);
+                LOG.info("用户 {} 在UID黑名单中命中 {}", uid, userType);
+                return userType;
+            }
+
+            Map<String, String> ipBlacklistMap = blacklistCache.get(IP_VISIO_BLACKLIST_HASH_KEY);
+            if (ipBlacklistMap.containsKey(clientIP)) {
+                String userType = ipBlacklistMap.get(clientIP);
+                LOG.info("用户 {} 在IP黑名单中命中 {}, 参数为: clientIP为: {}", uid, userType, clientIP);
+                return userType;
+            }
+
+            return this.matchGeneralizationUserType(uid, cityCode, hotSceneType);
+        } catch (
+                Exception e) {
+            LOG.error("blacklist filter isSafeVideoByUid error: ", e);
+        }
+        return null;
+    }
+
+    private String matchGeneralizationUserType(String uid, String cityCode, Long hotSceneType) {
+        if (MapUtils.isNotEmpty(generalizationUserConditionConfig) && generalizationUserConditionConfig.containsKey(cityCode)) {
+            GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(cityCode);
+            if (CollectionUtils.isNotEmpty(userConfig.getExcludeHotSceneType())) {
+                if (!userConfig.getExcludeHotSceneType().contains(hotSceneType)) {
+                    String userType = userConfig.getUserType() + USER_TYPE_SUB_TYPE_CONNECTOR + userConfig.getUserSubType();
+                    LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: cityCode={}, hotSceneType={}", uid, userType, cityCode, hotSceneType);
+                    return userType;
+                }
+            }
+        }
+        return null;
+    }
+
+    /**
+     * 匹配videoId的标签包含tagIds中的任意一个
+     *
+     * @param videoId 视频ID
+     * @param tagIds  标签ID列表
+     * @return true-匹配，false-不匹配
+     */
+    private boolean videoTagAnyMatch(Long videoId, Collection<Long> tagIds) {
+        if (MapUtils.isEmpty(videoTagCache) || CollectionUtils.isEmpty(tagIds)) {
+            return false;
+        }
+        for (Long tagId : tagIds) {
+            Set<Long> videoIds = videoTagCache.get(tagId);
+            if (CollectionUtils.isNotEmpty(videoIds) && videoIds.contains(videoId)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private String calcUserRedisKey(String uidStr) {
+        long uid = 0L;
+        try {
+            uid = Long.parseLong(uidStr);
+        } catch (
+                Exception e) {
+            LOG.error("calcUserRedisKey error: ", e);
+        }
+        return USER_VISIO_BLACKLIST_HASH_KEY + (uid % USER_REDIS_KEY_PARTITION_COUNT);
+    }
+
+    private Collection<Long> findExcludeTagIds(String userType) {
+        if (StringUtils.isBlank(userType) || MapUtils.isEmpty(tagFilterConfigMap)) {
+            return Collections.emptySet();
+        }
+
+        TagFilterConfig tagFilterConfig = tagFilterConfigMap.get(userType);
+        if (Objects.isNull(tagFilterConfig)) {
+            return Collections.emptySet();
+        }
+        return tagFilterConfig.getRecommendExcludeTag();
+    }
+
+    @Data
+    private static class GeneralizationUserConfig {
+        Set<Long> excludeHotSceneType;
+        String userType;
+        String userSubType;
+
+    }
+
+    @Data
+    private static class TagFilterConfig {
+        /**
+         * 推荐场景下要过滤掉的标签
+         */
+        Set<Long> recommendExcludeTag;
+        /**
+         * 详情场景下要过滤掉的标签
+         */
+        Set<Long> detailExcludeTag;
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/SecurityStrategy.java

@@ -17,9 +17,9 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 /**
  * @author dyp
@@ -39,6 +39,9 @@ public class SecurityStrategy implements FilterStrategy {
     @Autowired
     private WxVideoTagRelRepository wxVideoTagRelRepository;
 
+    @Resource
+    private BlacklistContainer blacklistContainer;
+
     // 内存持久保存不淘汰
     private LoadingCache<String, Set<Long>> videoCache = CacheBuilder.newBuilder()
             .maximumSize(100)
@@ -86,25 +89,29 @@ public class SecurityStrategy implements FilterStrategy {
             return param.getVideoIds();
         }
 
-        if (CollectionUtils.isEmpty(excludeScenes)
-                || !CommonCollectionUtils.contains(excludeScenes, param.getHotSceneType())) {
-
-            if (MapUtils.isEmpty(videoFilterCityTagIdMap)
-                    || !videoFilterCityTagIdMap.containsKey(param.getCityCode())) {
-                return param.getVideoIds();
-            }
+        // if (CollectionUtils.isEmpty(excludeScenes)
+        //         || !CommonCollectionUtils.contains(excludeScenes, param.getHotSceneType())) {
+        //
+        //     if (MapUtils.isEmpty(videoFilterCityTagIdMap)
+        //             || !videoFilterCityTagIdMap.containsKey(param.getCityCode())) {
+        //         return param.getVideoIds();
+        //     }
+        //
+        //     Set<Long> filterVideos = videoCache.getUnchecked(param.getCityCode());
+        //     if (CollectionUtils.isEmpty(filterVideos)) {
+        //         return param.getVideoIds();
+        //     }
+        //
+        //     List<Long> result = param.getVideoIds().stream()
+        //             .filter(l -> !filterVideos.contains(l))
+        //             .collect(Collectors.toList());
+        //
+        //     return result;
+        // }
+        // return param.getVideoIds();
 
-            Set<Long> filterVideos = videoCache.getUnchecked(param.getCityCode());
-            if (CollectionUtils.isEmpty(filterVideos)) {
-                return param.getVideoIds();
-            }
+        return blacklistContainer.filterUnsafeVideoByUser(param.getVideoIds(), param.getUid(),
+                param.getHotSceneType(), param.getCityCode(), param.getClientIp());
 
-            List<Long> result = param.getVideoIds().stream()
-                    .filter(l -> !filterVideos.contains(l))
-                    .collect(Collectors.toList());
-
-            return result;
-        }
-        return param.getVideoIds();
     }
 }