|
|
@@ -1,408 +0,0 @@
|
|
|
-package com.tzld.piaoquan.recommend.server.service.filter.strategy;
|
|
|
-
|
|
|
-import com.aliyun.openservices.aliyun.log.producer.LogProducer;
|
|
|
-import com.aliyun.openservices.aliyun.log.producer.Producer;
|
|
|
-import com.aliyun.openservices.aliyun.log.producer.ProducerConfig;
|
|
|
-import com.aliyun.openservices.aliyun.log.producer.ProjectConfig;
|
|
|
-import com.aliyun.openservices.log.common.LogItem;
|
|
|
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
-import com.google.common.cache.CacheBuilder;
|
|
|
-import com.google.common.cache.CacheLoader;
|
|
|
-import com.google.common.cache.LoadingCache;
|
|
|
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
|
|
|
-import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
|
|
|
-import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
|
|
|
-import lombok.Data;
|
|
|
-import org.apache.commons.collections4.CollectionUtils;
|
|
|
-import org.apache.commons.collections4.MapUtils;
|
|
|
-import org.apache.commons.lang3.StringUtils;
|
|
|
-import org.slf4j.Logger;
|
|
|
-import org.slf4j.LoggerFactory;
|
|
|
-import org.springframework.beans.factory.annotation.Autowired;
|
|
|
-import org.springframework.beans.factory.annotation.Qualifier;
|
|
|
-import org.springframework.beans.factory.annotation.Value;
|
|
|
-import org.springframework.data.redis.core.RedisTemplate;
|
|
|
-import org.springframework.scheduling.annotation.Scheduled;
|
|
|
-import org.springframework.stereotype.Component;
|
|
|
-
|
|
|
-import javax.annotation.Nonnull;
|
|
|
-import javax.annotation.PostConstruct;
|
|
|
-import javax.annotation.Resource;
|
|
|
-import java.util.*;
|
|
|
-import java.util.concurrent.ConcurrentHashMap;
|
|
|
-import java.util.concurrent.TimeUnit;
|
|
|
-import java.util.stream.Collectors;
|
|
|
-
|
|
|
-/**
|
|
|
- * 黑名单列表相关的容器。主要实现以下几个功能
|
|
|
- * <ul>
|
|
|
- * <ol>1. 判断用户属于哪个类型的黑名单</ol>
|
|
|
- * <ol>2. 根据用户类型判断视频对于该用户是否有风险</ol>
|
|
|
- * <ol>3. 根据用户类型过滤掉视频列表中对该用户有风险的视频</ol>
|
|
|
- * </ul>
|
|
|
- */
|
|
|
-@Component
|
|
|
-public class BlacklistContainer {
|
|
|
-
|
|
|
-
|
|
|
- private static final Logger LOG = LoggerFactory.getLogger(BlacklistContainer.class);
|
|
|
-
|
|
|
- public static final String CONNECTOR_STR = ":";
|
|
|
-
|
|
|
- private static final int USER_REDIS_KEY_PARTITION_COUNT = 10;
|
|
|
-
|
|
|
- /**
|
|
|
- * 用户访问黑名单 Redis Key
|
|
|
- */
|
|
|
- private static final String USER_VISIO_BLACKLIST_HASH_KEY = "visio:blacklist:user:";
|
|
|
-
|
|
|
- /**
|
|
|
- * IP访问黑名单 Redis Key
|
|
|
- */
|
|
|
- private static final String IP_VISIO_BLACKLIST_HASH_KEY = "visio:blacklist:ip";
|
|
|
-
|
|
|
- @Autowired
|
|
|
- @Qualifier("longVideoRedisTemplate")
|
|
|
- private RedisTemplate<String, String> longVideoRedisTemplate;
|
|
|
-
|
|
|
- @Resource
|
|
|
- private WxVideoTagRelRepository wxVideoTagRelRepository;
|
|
|
-
|
|
|
- @ApolloJsonValue("${content.security.generalization.user.condition.config:{}}")
|
|
|
- private Map<String, GeneralizationUserConfig> generalizationUserConditionConfig;
|
|
|
-
|
|
|
- /**
|
|
|
- * 不同类型的用户要过滤掉的标签列表配置
|
|
|
- * <br >
|
|
|
- * <p>
|
|
|
- * Key的格式为: {userType}:{userSubType}
|
|
|
- * <br>
|
|
|
- * userType枚举值: 1-竞品用户, 2-微信人员, 3-网安
|
|
|
- * <br >
|
|
|
- * userSubType枚举值: 1-精准用户, 2-泛化用户
|
|
|
- */
|
|
|
- @ApolloJsonValue("${content.security.filter.config:{}}")
|
|
|
- private Map<String, TagFilterConfig> tagFilterConfigMap;
|
|
|
-
|
|
|
- @Value("${aliyun.log.endpoint}")
|
|
|
- private String endpoint;
|
|
|
- @Value("${aliyun.log.accessKeyId}")
|
|
|
- private String accessKeyId;
|
|
|
- @Value("${aliyun.log.accessKeySecret}")
|
|
|
- private String accessKeySecret;
|
|
|
-
|
|
|
- @Value("${aliyun.blacklist.filter.log.project}")
|
|
|
- private String project;
|
|
|
- @Value("${aliyun.blacklist.filter.log.store}")
|
|
|
- private String logStore;
|
|
|
- @Value("${spring.profiles.active}")
|
|
|
- private String activeProfile;
|
|
|
- private Producer producer;
|
|
|
-
|
|
|
- /**
|
|
|
- * 保存Tag标签与视频列表的映射
|
|
|
- * <br>
|
|
|
- * Key为TagId, Value为对应的视频ID列表
|
|
|
- */
|
|
|
- private static Map<Long, Set<Long>> videoTagCache = new ConcurrentHashMap<>();
|
|
|
-
|
|
|
- /**
|
|
|
- * 黑名单本地二级缓存，一级缓存为Redis缓存。此处直接读取即可
|
|
|
- * <br />
|
|
|
- * Redis缓存由longvideo服务写入
|
|
|
- * <br>
|
|
|
- * com.weiqu.video.service.filter.impl.BlacklistFilterImpl#refreshUidCache
|
|
|
- * <br>
|
|
|
- * com.weiqu.video.service.filter.impl.BlacklistFilterImpl#refreshIPCache
|
|
|
- */
|
|
|
- private final LoadingCache<String, Map<String, String>> blacklistCache = CacheBuilder.newBuilder()
|
|
|
- .expireAfterWrite(5, TimeUnit.MINUTES)
|
|
|
- .build(new CacheLoader<String, Map<String, String>>() {
|
|
|
- @Override
|
|
|
- public Map<String, String> load(@Nonnull String key) {
|
|
|
- Map<Object, Object> map = longVideoRedisTemplate.opsForHash().entries(key);
|
|
|
- if (MapUtils.isEmpty(map)) {
|
|
|
- return new HashMap<>();
|
|
|
- }
|
|
|
- return map.entrySet().stream().collect(
|
|
|
- Collectors.toMap(
|
|
|
- entry -> entry.getKey().toString(),
|
|
|
- entry -> entry.getValue().toString(),
|
|
|
- (v1, v2) -> v1
|
|
|
- ));
|
|
|
- }
|
|
|
- });
|
|
|
-
|
|
|
- @PostConstruct
|
|
|
- public void init() {
|
|
|
- LOG.info("generalizationUserConditionConfig: {}", generalizationUserConditionConfig);
|
|
|
- LOG.info("tagFilterConfigMap: {}", tagFilterConfigMap);
|
|
|
- refreshVideoTagCache();
|
|
|
- initLogProducer();
|
|
|
- }
|
|
|
-
|
|
|
- @Scheduled(cron = "0 0/5 * * * ? ")
|
|
|
- public void cronSync() {
|
|
|
- refreshVideoTagCache();
|
|
|
- }
|
|
|
-
|
|
|
- private void initLogProducer() {
|
|
|
- LOG.info("BlacklistContainer.initLogProducer: project={}, endpoint={}, accessKeyId={}, accessKeySecret={}, logStore={}",
|
|
|
- project, endpoint, accessKeyId, accessKeySecret, logStore);
|
|
|
- ProducerConfig producerConfig = new ProducerConfig();
|
|
|
- producer = new LogProducer(producerConfig);
|
|
|
- producer.putProjectConfig(new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret));
|
|
|
- }
|
|
|
-
|
|
|
- public void refreshVideoTagCache() {
|
|
|
- // LOG.info("同步本地标签ID与视频列表的缓存任务开始");
|
|
|
- Map<Long, Set<Long>> tmpMap = new ConcurrentHashMap<>();
|
|
|
-
|
|
|
- if (MapUtils.isNotEmpty(tagFilterConfigMap)) {
|
|
|
-
|
|
|
- // 获取所有的标签ID列表
|
|
|
- Set<Long> tagIdSet = new HashSet<>();
|
|
|
- for (Map.Entry<String, TagFilterConfig> entry : tagFilterConfigMap.entrySet()) {
|
|
|
- TagFilterConfig tagFilterConfig = entry.getValue();
|
|
|
- if (Objects.isNull(tagFilterConfig)) {
|
|
|
- continue;
|
|
|
- }
|
|
|
- if (CollectionUtils.isNotEmpty(tagFilterConfig.getRecommendExcludeTag())) {
|
|
|
- tagIdSet.addAll(tagFilterConfig.getRecommendExcludeTag());
|
|
|
- }
|
|
|
- if (CollectionUtils.isNotEmpty(tagFilterConfig.getDetailExcludeTag())) {
|
|
|
- tagIdSet.addAll(tagFilterConfig.getDetailExcludeTag());
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // 获取标签ID对应的视频ID列表
|
|
|
- for (Long tagId : tagIdSet) {
|
|
|
- List<WxVideoTagRel> wxVideoTagRels = wxVideoTagRelRepository.findAllByTagId(tagId);
|
|
|
- Set<Long> videoIdSet = wxVideoTagRels.stream().map(WxVideoTagRel::getVideoId).collect(Collectors.toSet());
|
|
|
- // LOG.info("同步本地标签ID与视频列表缓存任务 -- tagId: {}, videoIdSize: {}", tagId, videoIdSet.size());
|
|
|
- tmpMap.put(tagId, videoIdSet);
|
|
|
- }
|
|
|
- }
|
|
|
- videoTagCache = tmpMap;
|
|
|
-
|
|
|
- // LOG.info("同步本地标签ID与视频列表的缓存任务结束");
|
|
|
- }
|
|
|
-
|
|
|
- public List<Long> filterUnsafeVideoByUser(List<Long> videoIds, String uid, Long hotSceneType, String cityCode, String clientIP, String mid, String usedScene, Integer appType) {
|
|
|
- if (CollectionUtils.isEmpty(videoIds)) {
|
|
|
- return videoIds;
|
|
|
- }
|
|
|
-
|
|
|
- String userType = this.matchUserBlacklistTypeEnum(uid, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
|
|
|
- Collection<Long> tagIdSet = this.findRecommendExcludeTagIds(userType, appType);
|
|
|
- if (CollectionUtils.isEmpty(tagIdSet)) {
|
|
|
- return videoIds;
|
|
|
- }
|
|
|
-
|
|
|
- return videoIds.stream().filter(videoId -> {
|
|
|
- if (videoTagAnyMatch(videoId, tagIdSet)) {
|
|
|
- // LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}, mid={}, usedScene={}, appType={}",
|
|
|
- // uid, userType, videoId, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
|
|
|
- return false;
|
|
|
- }
|
|
|
- return true;
|
|
|
- }).collect(Collectors.toList());
|
|
|
- }
|
|
|
-
|
|
|
- public String matchUserBlacklistTypeEnum(String uid, Long hotSceneType, String cityCode, String clientIP, String mid,
|
|
|
- String usedScene, Integer appType) {
|
|
|
- try {
|
|
|
- // LOG.info("计算用户黑名单类型，判断参数: uid={}, hotSceneType={}, cityCode={}, clientIP={}, mid={}, usedScene={}, appType={}",
|
|
|
- // uid, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
|
|
|
- if (StringUtils.isNotBlank(uid)) {
|
|
|
- String key = this.calcUserRedisKey(uid);
|
|
|
- Map<String, String> uidBlacklistMap = blacklistCache.get(key);
|
|
|
- if (uidBlacklistMap.containsKey(uid)) {
|
|
|
- String userType = uidBlacklistMap.get(uid);
|
|
|
- this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "UID", mid, usedScene, appType);
|
|
|
- // LOG.info("用户 {} 在UID黑名单中命中 {}", uid, userType);
|
|
|
- return userType;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if (StringUtils.isNotBlank(clientIP)) {
|
|
|
- Map<String, String> ipBlacklistMap = blacklistCache.get(IP_VISIO_BLACKLIST_HASH_KEY);
|
|
|
- if (ipBlacklistMap.containsKey(clientIP)) {
|
|
|
- String userType = ipBlacklistMap.get(clientIP);
|
|
|
- this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "IP", mid, usedScene, appType);
|
|
|
- // LOG.info("用户 {} 在IP黑名单中命中 {}, 参数为: clientIP为: {}", uid, userType, clientIP);
|
|
|
- return userType;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- String userType = this.matchGeneralizationUserType(uid, cityCode, hotSceneType, appType);
|
|
|
- if (StringUtils.isNotBlank(userType)) {
|
|
|
- this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "RegionAndHotSceneType", mid, usedScene, appType);
|
|
|
- }
|
|
|
- return userType;
|
|
|
- } catch (
|
|
|
- Exception e) {
|
|
|
- LOG.error("blacklist filter isSafeVideoByUid error: ", e);
|
|
|
-
|
|
|
- }
|
|
|
- return null;
|
|
|
- }
|
|
|
-
|
|
|
- private String matchGeneralizationUserType(String uid, String cityCode, Long hotSceneType, Integer appType) {
|
|
|
- // 参数为空，则跳过
|
|
|
- if (StringUtils.isBlank(cityCode) && Objects.isNull(appType)) {
|
|
|
- return null;
|
|
|
- }
|
|
|
- // 配置为空，则跳过
|
|
|
- if (MapUtils.isEmpty(generalizationUserConditionConfig)) {
|
|
|
- return null;
|
|
|
- }
|
|
|
-
|
|
|
- // 优先找，城市+appType的配置
|
|
|
- if (StringUtils.isNotBlank(cityCode) && Objects.nonNull(appType)) {
|
|
|
- GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(cityCode + CONNECTOR_STR + appType);
|
|
|
- if (Objects.nonNull(userConfig) && userConfig.isExcludeHotSceneType(hotSceneType)) {
|
|
|
- // LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: appType = {}, cityCode={}, hotSceneType={}", uid, userConfig.fullUserType, appType, cityCode, hotSceneType);
|
|
|
- return userConfig.getFullUserType();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // 其次找，appType的配置
|
|
|
- if (Objects.nonNull(appType)) {
|
|
|
- GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(appType.toString());
|
|
|
- if (Objects.nonNull(userConfig) && userConfig.isExcludeHotSceneType(hotSceneType)) {
|
|
|
- // LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: appType={}, hotSceneType={}", uid, userConfig.fullUserType, appType, hotSceneType);
|
|
|
- return userConfig.getFullUserType();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // 最后找，城市的配置
|
|
|
- if (StringUtils.isNotBlank(cityCode)) {
|
|
|
- GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(cityCode);
|
|
|
- if (Objects.nonNull(userConfig) && userConfig.isExcludeHotSceneType(hotSceneType)) {
|
|
|
- // LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: cityCode={}, hotSceneType={}", uid, userConfig.fullUserType, cityCode, hotSceneType);
|
|
|
- return userConfig.getFullUserType();
|
|
|
- }
|
|
|
- }
|
|
|
- return null;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 匹配videoId的标签包含tagIds中的任意一个
|
|
|
- *
|
|
|
- * @param videoId 视频ID
|
|
|
- * @param tagIds 标签ID列表
|
|
|
- * @return true-匹配，false-不匹配
|
|
|
- */
|
|
|
- private boolean videoTagAnyMatch(Long videoId, Collection<Long> tagIds) {
|
|
|
- if (MapUtils.isEmpty(videoTagCache) || CollectionUtils.isEmpty(tagIds)) {
|
|
|
- return false;
|
|
|
- }
|
|
|
- for (Long tagId : tagIds) {
|
|
|
- Set<Long> videoIds = videoTagCache.get(tagId);
|
|
|
- if (CollectionUtils.isNotEmpty(videoIds) && videoIds.contains(videoId)) {
|
|
|
- return true;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- private String calcUserRedisKey(String uidStr) {
|
|
|
- long uid = 0L;
|
|
|
- try {
|
|
|
- uid = Long.parseLong(uidStr);
|
|
|
- } catch (
|
|
|
- Exception ignore) {
|
|
|
- }
|
|
|
- return USER_VISIO_BLACKLIST_HASH_KEY + (uid % USER_REDIS_KEY_PARTITION_COUNT);
|
|
|
- }
|
|
|
-
|
|
|
- private Collection<Long> findRecommendExcludeTagIds(String userType, Integer appType) {
|
|
|
- if (StringUtils.isBlank(userType) || MapUtils.isEmpty(tagFilterConfigMap)) {
|
|
|
- return Collections.emptySet();
|
|
|
- }
|
|
|
-
|
|
|
- // 查看单APP配置的
|
|
|
- if (StringUtils.isNotBlank(userType) && Objects.nonNull(appType)) {
|
|
|
- String key = userType + CONNECTOR_STR + appType;
|
|
|
- TagFilterConfig tagFilterConfig = tagFilterConfigMap.get(key);
|
|
|
- if (Objects.nonNull(tagFilterConfig)) {
|
|
|
- // LOG.info("命中过滤标签配置: {} == {}", key, tagFilterConfig.getRecommendExcludeTag());
|
|
|
- return tagFilterConfig.getRecommendExcludeTag();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if (StringUtils.isNotBlank(userType)) {
|
|
|
- TagFilterConfig tagFilterConfig = tagFilterConfigMap.get(userType);
|
|
|
- if (Objects.nonNull(tagFilterConfig)) {
|
|
|
- // LOG.info("命中过滤标签配置: {} == {}", userType, tagFilterConfig.getRecommendExcludeTag());
|
|
|
- return tagFilterConfig.getRecommendExcludeTag();
|
|
|
- }
|
|
|
- }
|
|
|
- return null;
|
|
|
- }
|
|
|
-
|
|
|
- private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String fullUserType, String blacklistType, String mid, String usedScene, Integer appType) {
|
|
|
- try {
|
|
|
- String[] split = fullUserType.split(CONNECTOR_STR);
|
|
|
- Map<String, String> logMap = new HashMap<>();
|
|
|
- logMap.put("uid", StringUtils.isNotBlank(uid) ? uid : "");
|
|
|
- logMap.put("mid", StringUtils.isNotBlank(mid) ? mid : "");
|
|
|
- logMap.put("cityCode", StringUtils.isNotBlank(cityCode) ? cityCode : "");
|
|
|
- logMap.put("hotSceneType", Objects.nonNull(hotSceneType) ? hotSceneType.toString() : "");
|
|
|
- logMap.put("clientIp", StringUtils.isNotBlank(clientIp) ? clientIp : "");
|
|
|
- logMap.put("usedScene", StringUtils.isNotBlank(usedScene) ? usedScene : "");
|
|
|
- logMap.put("appType", Objects.nonNull(appType) ? appType.toString() : "");
|
|
|
- logMap.put("userType", split[0]);
|
|
|
- logMap.put("userSubType", split[1]);
|
|
|
- logMap.put("env", activeProfile);
|
|
|
- logMap.put("blacklistType", blacklistType);
|
|
|
- logMap.put("uploadService", "recommend-server");
|
|
|
-
|
|
|
- LogItem logItem = new LogItem();
|
|
|
- logMap.forEach(logItem::PushBack);
|
|
|
-
|
|
|
- ThreadPoolFactory.logPool().execute(() -> {
|
|
|
- try {
|
|
|
- producer.send(project, logStore, logItem);
|
|
|
- } catch (
|
|
|
- Exception e) {
|
|
|
- LOG.error("log send error: ", e);
|
|
|
- }
|
|
|
- });
|
|
|
- } catch (
|
|
|
- Exception e) {
|
|
|
- LOG.error("blacklist filter upload log error: ", e);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- @Data
|
|
|
- private static class GeneralizationUserConfig {
|
|
|
- Set<Long> excludeHotSceneType;
|
|
|
- String userType;
|
|
|
- String userSubType;
|
|
|
- String fullUserType;
|
|
|
-
|
|
|
- public boolean isExcludeHotSceneType(Long hotSceneType) {
|
|
|
- return CollectionUtils.isEmpty(excludeHotSceneType)
|
|
|
- || !excludeHotSceneType.contains(hotSceneType);
|
|
|
- }
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- @Data
|
|
|
- private static class TagFilterConfig {
|
|
|
- /**
|
|
|
- * 推荐场景下要过滤掉的标签
|
|
|
- */
|
|
|
- Set<Long> recommendExcludeTag;
|
|
|
- /**
|
|
|
- * 详情场景下要过滤掉的标签
|
|
|
- */
|
|
|
- Set<Long> detailExcludeTag;
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
-}
|
