|
|
@@ -0,0 +1,361 @@
|
|
|
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
|
|
|
+
|
|
|
+import com.aliyun.openservices.aliyun.log.producer.LogProducer;
|
|
|
+import com.aliyun.openservices.aliyun.log.producer.Producer;
|
|
|
+import com.aliyun.openservices.aliyun.log.producer.ProducerConfig;
|
|
|
+import com.aliyun.openservices.aliyun.log.producer.ProjectConfig;
|
|
|
+import com.aliyun.openservices.log.common.LogItem;
|
|
|
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
+import com.google.common.cache.CacheBuilder;
|
|
|
+import com.google.common.cache.CacheLoader;
|
|
|
+import com.google.common.cache.LoadingCache;
|
|
|
+import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
|
|
|
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
|
|
|
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
|
|
|
+import lombok.Data;
|
|
|
+import org.apache.commons.collections4.CollectionUtils;
|
|
|
+import org.apache.commons.collections4.MapUtils;
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
+import org.slf4j.Logger;
|
|
|
+import org.slf4j.LoggerFactory;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.beans.factory.annotation.Qualifier;
|
|
|
+import org.springframework.beans.factory.annotation.Value;
|
|
|
+import org.springframework.data.redis.core.RedisTemplate;
|
|
|
+import org.springframework.scheduling.annotation.Scheduled;
|
|
|
+import org.springframework.stereotype.Component;
|
|
|
+
|
|
|
+import javax.annotation.Nonnull;
|
|
|
+import javax.annotation.PostConstruct;
|
|
|
+import javax.annotation.Resource;
|
|
|
+import java.util.*;
|
|
|
+import java.util.concurrent.ConcurrentHashMap;
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+
|
|
|
+/**
|
|
|
+ * 黑名单列表相关的容器。主要实现以下几个功能
|
|
|
+ * <ul>
|
|
|
+ * <ol>1. 判断用户属于哪个类型的黑名单</ol>
|
|
|
+ * <ol>2. 根据用户类型判断视频对于该用户是否有风险</ol>
|
|
|
+ * <ol>3. 根据用户类型过滤掉视频列表中对该用户有风险的视频</ol>
|
|
|
+ * </ul>
|
|
|
+ */
|
|
|
+@Component
|
|
|
+public class BlacklistContainer {
|
|
|
+
|
|
|
+
|
|
|
+ private static final Logger LOG = LoggerFactory.getLogger(BlacklistContainer.class);
|
|
|
+
|
|
|
+ public static final String USER_TYPE_SUB_TYPE_CONNECTOR = ":";
|
|
|
+
|
|
|
+ private static final int USER_REDIS_KEY_PARTITION_COUNT = 100;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 用户访问黑名单 Redis Key
|
|
|
+ */
|
|
|
+ private static final String USER_VISIO_BLACKLIST_HASH_KEY = "visio:blacklist:user:";
|
|
|
+
|
|
|
+ /**
|
|
|
+ * IP访问黑名单 Redis Key
|
|
|
+ */
|
|
|
+ private static final String IP_VISIO_BLACKLIST_HASH_KEY = "visio:blacklist:ip";
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ @Qualifier("longVideoRedisTemplate")
|
|
|
+ private RedisTemplate<String, String> longVideoRedisTemplate;
|
|
|
+
|
|
|
+ @Resource
|
|
|
+ private WxVideoTagRelRepository wxVideoTagRelRepository;
|
|
|
+
|
|
|
+ @ApolloJsonValue("${content.security.generalization.user.condition.config:{}}")
|
|
|
+ private Map<String, GeneralizationUserConfig> generalizationUserConditionConfig;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 不同类型的用户要过滤掉的标签列表配置
|
|
|
+ * <br >
|
|
|
+ * <p>
|
|
|
+ * Key的格式为: {userType}:{userSubType}
|
|
|
+ * <br>
|
|
|
+ * userType枚举值: 1-竞品用户, 2-微信人员, 3-网安
|
|
|
+ * <br >
|
|
|
+ * userSubType枚举值: 1-精准用户, 2-泛化用户
|
|
|
+ */
|
|
|
+ @ApolloJsonValue("${content.security.filter.config:{}}")
|
|
|
+ private Map<String, TagFilterConfig> tagFilterConfigMap;
|
|
|
+
|
|
|
+ @Value("${aliyun.log.endpoint}")
|
|
|
+ private String endpoint;
|
|
|
+ @Value("${aliyun.log.accessKeyId}")
|
|
|
+ private String accessKeyId;
|
|
|
+ @Value("${aliyun.log.accessKeySecret}")
|
|
|
+ private String accessKeySecret;
|
|
|
+
|
|
|
+ @Value("${aliyun.blacklist.filter.log.project}")
|
|
|
+ private String project;
|
|
|
+ @Value("${aliyun.blacklist.filter.log.store}")
|
|
|
+ private String logStore;
|
|
|
+ @Value("${spring.profiles.active}")
|
|
|
+ private String activeProfile;
|
|
|
+ private Producer producer;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 保存Tag标签与视频列表的映射
|
|
|
+ * <br>
|
|
|
+ * Key为TagId, Value为对应的视频ID列表
|
|
|
+ */
|
|
|
+ private static Map<Long, Set<Long>> videoTagCache = new ConcurrentHashMap<>();
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 黑名单本地二级缓存，一级缓存为Redis缓存。此处直接读取即可
|
|
|
+ * <br />
|
|
|
+ * Redis缓存由longvideo服务写入
|
|
|
+ * <br>
|
|
|
+ * com.weiqu.video.service.filter.impl.BlacklistFilterImpl#refreshUidCache
|
|
|
+ * <br>
|
|
|
+ * com.weiqu.video.service.filter.impl.BlacklistFilterImpl#refreshIPCache
|
|
|
+ */
|
|
|
+ private final LoadingCache<String, Map<String, String>> blacklistCache = CacheBuilder.newBuilder()
|
|
|
+ .expireAfterWrite(10, TimeUnit.MINUTES)
|
|
|
+ .build(new CacheLoader<String, Map<String, String>>() {
|
|
|
+ @Override
|
|
|
+ public Map<String, String> load(@Nonnull String key) {
|
|
|
+ Map<Object, Object> map = longVideoRedisTemplate.opsForHash().entries(key);
|
|
|
+ if (MapUtils.isEmpty(map)) {
|
|
|
+ return new HashMap<>();
|
|
|
+ }
|
|
|
+ return map.entrySet().stream().collect(
|
|
|
+ Collectors.toMap(
|
|
|
+ entry -> entry.getKey().toString(),
|
|
|
+ entry -> entry.getValue().toString(),
|
|
|
+ (v1, v2) -> v1
|
|
|
+ ));
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ @PostConstruct
|
|
|
+ public void init() {
|
|
|
+ LOG.info("generalizationUserConditionConfig: {}", generalizationUserConditionConfig);
|
|
|
+ LOG.info("tagFilterConfigMap: {}", tagFilterConfigMap);
|
|
|
+ refreshVideoTagCache();
|
|
|
+ initLogProducer();
|
|
|
+ }
|
|
|
+
|
|
|
+ @Scheduled(cron = "0 0/5 * * * ? ")
|
|
|
+ public void cronSync() {
|
|
|
+ refreshVideoTagCache();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void initLogProducer() {
|
|
|
+ LOG.info("BlacklistContainer.initLogProducer: project={}, endpoint={}, accessKeyId={}, accessKeySecret={}, logStore={}",
|
|
|
+ project, endpoint, accessKeyId, accessKeySecret, logStore);
|
|
|
+ ProducerConfig producerConfig = new ProducerConfig();
|
|
|
+ producer = new LogProducer(producerConfig);
|
|
|
+ producer.putProjectConfig(new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret));
|
|
|
+ }
|
|
|
+
|
|
|
+ public void refreshVideoTagCache() {
|
|
|
+ LOG.info("同步本地标签ID与视频列表的缓存任务开始");
|
|
|
+ Map<Long, Set<Long>> tmpMap = new ConcurrentHashMap<>();
|
|
|
+
|
|
|
+ if (MapUtils.isNotEmpty(tagFilterConfigMap)) {
|
|
|
+
|
|
|
+ // 获取所有的标签ID列表
|
|
|
+ Set<Long> tagIdSet = new HashSet<>();
|
|
|
+ for (Map.Entry<String, TagFilterConfig> entry : tagFilterConfigMap.entrySet()) {
|
|
|
+ TagFilterConfig tagFilterConfig = entry.getValue();
|
|
|
+ if (Objects.isNull(tagFilterConfig)) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (CollectionUtils.isNotEmpty(tagFilterConfig.getRecommendExcludeTag())) {
|
|
|
+ tagIdSet.addAll(tagFilterConfig.getRecommendExcludeTag());
|
|
|
+ }
|
|
|
+ if (CollectionUtils.isNotEmpty(tagFilterConfig.getDetailExcludeTag())) {
|
|
|
+ tagIdSet.addAll(tagFilterConfig.getDetailExcludeTag());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 获取标签ID对应的视频ID列表
|
|
|
+ for (Long tagId : tagIdSet) {
|
|
|
+ List<WxVideoTagRel> wxVideoTagRels = wxVideoTagRelRepository.findAllByTagId(tagId);
|
|
|
+ Set<Long> videoIdSet = wxVideoTagRels.stream().map(WxVideoTagRel::getVideoId).collect(Collectors.toSet());
|
|
|
+ LOG.info("同步本地标签ID与视频列表缓存任务 -- tagId: {}, videoIdSize: {}", tagId, tagIdSet.size());
|
|
|
+ tmpMap.put(tagId, videoIdSet);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ videoTagCache = tmpMap;
|
|
|
+
|
|
|
+ LOG.info("同步本地标签ID与视频列表的缓存任务结束");
|
|
|
+ }
|
|
|
+
|
|
|
+ public List<Long> filterUnsafeVideoByUser(List<Long> videoIds, String uid, Long hotSceneType, String cityCode, String clientIP) {
|
|
|
+ if (CollectionUtils.isEmpty(videoIds)) {
|
|
|
+ return videoIds;
|
|
|
+ }
|
|
|
+
|
|
|
+ String userType = this.matchUserBlacklistTypeEnum(uid, hotSceneType, cityCode, clientIP);
|
|
|
+ Collection<Long> tagIdSet = this.findExcludeTagIds(userType);
|
|
|
+ if (CollectionUtils.isEmpty(tagIdSet)) {
|
|
|
+ return videoIds;
|
|
|
+ }
|
|
|
+
|
|
|
+ return videoIds.stream().filter(videoId -> {
|
|
|
+ if (videoTagAnyMatch(videoId, tagIdSet)) {
|
|
|
+ LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}",
|
|
|
+ uid, userType, videoId, hotSceneType, cityCode, clientIP);
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ return true;
|
|
|
+ }).collect(Collectors.toList());
|
|
|
+ }
|
|
|
+
|
|
|
+ private String matchUserBlacklistTypeEnum(String uid, Long hotSceneType, String cityCode, String clientIP) {
|
|
|
+ try {
|
|
|
+ LOG.info("计算用户黑名单类型，判断参数: uid={}, hotSceneType={}, cityCode={}, clientIP={}", uid, hotSceneType, cityCode, clientIP);
|
|
|
+ String key = this.calcUserRedisKey(uid);
|
|
|
+ Map<String, String> uidBlacklistMap = blacklistCache.get(key);
|
|
|
+ if (uidBlacklistMap.containsKey(uid)) {
|
|
|
+ String userType = uidBlacklistMap.get(uid);
|
|
|
+ this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "UID");
|
|
|
+ LOG.info("用户 {} 在UID黑名单中命中 {}", uid, userType);
|
|
|
+ return userType;
|
|
|
+ }
|
|
|
+
|
|
|
+ Map<String, String> ipBlacklistMap = blacklistCache.get(IP_VISIO_BLACKLIST_HASH_KEY);
|
|
|
+ if (ipBlacklistMap.containsKey(clientIP)) {
|
|
|
+ String userType = ipBlacklistMap.get(clientIP);
|
|
|
+ this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "IP");
|
|
|
+ LOG.info("用户 {} 在IP黑名单中命中 {}, 参数为: clientIP为: {}", uid, userType, clientIP);
|
|
|
+ return userType;
|
|
|
+ }
|
|
|
+
|
|
|
+ String userType = this.matchGeneralizationUserType(uid, cityCode, hotSceneType);
|
|
|
+ this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "RegionAndHotSceneType");
|
|
|
+ return userType;
|
|
|
+ } catch (
|
|
|
+ Exception e) {
|
|
|
+ LOG.error("blacklist filter isSafeVideoByUid error: ", e);
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ private String matchGeneralizationUserType(String uid, String cityCode, Long hotSceneType) {
|
|
|
+ if (MapUtils.isNotEmpty(generalizationUserConditionConfig) && generalizationUserConditionConfig.containsKey(cityCode)) {
|
|
|
+ GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(cityCode);
|
|
|
+ if (CollectionUtils.isNotEmpty(userConfig.getExcludeHotSceneType())) {
|
|
|
+ if (!userConfig.getExcludeHotSceneType().contains(hotSceneType)) {
|
|
|
+ String userType = userConfig.getUserType() + USER_TYPE_SUB_TYPE_CONNECTOR + userConfig.getUserSubType();
|
|
|
+ LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: cityCode={}, hotSceneType={}", uid, userType, cityCode, hotSceneType);
|
|
|
+ return userType;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 匹配videoId的标签包含tagIds中的任意一个
|
|
|
+ *
|
|
|
+ * @param videoId 视频ID
|
|
|
+ * @param tagIds 标签ID列表
|
|
|
+ * @return true-匹配，false-不匹配
|
|
|
+ */
|
|
|
+ private boolean videoTagAnyMatch(Long videoId, Collection<Long> tagIds) {
|
|
|
+ if (MapUtils.isEmpty(videoTagCache) || CollectionUtils.isEmpty(tagIds)) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ for (Long tagId : tagIds) {
|
|
|
+ Set<Long> videoIds = videoTagCache.get(tagId);
|
|
|
+ if (CollectionUtils.isNotEmpty(videoIds) && videoIds.contains(videoId)) {
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ private String calcUserRedisKey(String uidStr) {
|
|
|
+ long uid = 0L;
|
|
|
+ try {
|
|
|
+ uid = Long.parseLong(uidStr);
|
|
|
+ } catch (
|
|
|
+ Exception e) {
|
|
|
+ LOG.error("calcUserRedisKey error: ", e);
|
|
|
+ }
|
|
|
+ return USER_VISIO_BLACKLIST_HASH_KEY + (uid % USER_REDIS_KEY_PARTITION_COUNT);
|
|
|
+ }
|
|
|
+
|
|
|
+ private Collection<Long> findExcludeTagIds(String userType) {
|
|
|
+ if (StringUtils.isBlank(userType) || MapUtils.isEmpty(tagFilterConfigMap)) {
|
|
|
+ return Collections.emptySet();
|
|
|
+ }
|
|
|
+
|
|
|
+ TagFilterConfig tagFilterConfig = tagFilterConfigMap.get(userType);
|
|
|
+ if (Objects.isNull(tagFilterConfig)) {
|
|
|
+ return Collections.emptySet();
|
|
|
+ }
|
|
|
+ return tagFilterConfig.getRecommendExcludeTag();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String fullUserType, String blacklistType) {
|
|
|
+ try {
|
|
|
+ String[] split = fullUserType.split(USER_TYPE_SUB_TYPE_CONNECTOR);
|
|
|
+ this.filterLogUpload(uid, cityCode, hotSceneType, clientIp, split[0], split[1], blacklistType);
|
|
|
+ } catch (
|
|
|
+ Exception e) {
|
|
|
+ LOG.error("filterLogUpload error: ", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String userType, String userSubType, String blacklistType) {
|
|
|
+ try {
|
|
|
+ Map<String, String> logMap = new HashMap<>();
|
|
|
+ logMap.put("uid", uid);
|
|
|
+ logMap.put("cityCode", cityCode);
|
|
|
+ logMap.put("hotSceneType", hotSceneType.toString());
|
|
|
+ logMap.put("clientIp", clientIp);
|
|
|
+ logMap.put("userType", userType);
|
|
|
+ logMap.put("userSubType", userSubType);
|
|
|
+ logMap.put("env", activeProfile);
|
|
|
+ logMap.put("blacklistType", blacklistType);
|
|
|
+ logMap.put("uploadService", "recommend-server");
|
|
|
+
|
|
|
+ LogItem logItem = new LogItem();
|
|
|
+ logMap.forEach(logItem::PushBack);
|
|
|
+
|
|
|
+ ThreadPoolFactory.logPool().execute(() -> {
|
|
|
+ try {
|
|
|
+ producer.send(project, logStore, logItem);
|
|
|
+ } catch (
|
|
|
+ Exception e) {
|
|
|
+ LOG.error("log send error: ", e);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ } catch (
|
|
|
+ Exception e) {
|
|
|
+ LOG.error("blacklist filter upload log error: ", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Data
|
|
|
+ private static class GeneralizationUserConfig {
|
|
|
+ Set<Long> excludeHotSceneType;
|
|
|
+ String userType;
|
|
|
+ String userSubType;
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ @Data
|
|
|
+ private static class TagFilterConfig {
|
|
|
+ /**
|
|
|
+ * 推荐场景下要过滤掉的标签
|
|
|
+ */
|
|
|
+ Set<Long> recommendExcludeTag;
|
|
|
+ /**
|
|
|
+ * 详情场景下要过滤掉的标签
|
|
|
+ */
|
|
|
+ Set<Long> detailExcludeTag;
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+}
|
