Merge branch 'feature_20240424_zhaohaipeng_content_security_v2' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/BlacklistContainer.java

@@ -49,7 +49,7 @@ public class BlacklistContainer {
 
     public static final String USER_TYPE_SUB_TYPE_CONNECTOR = ":";
 
-    private static final int USER_REDIS_KEY_PARTITION_COUNT = 100;
+    private static final int USER_REDIS_KEY_PARTITION_COUNT = 10;
 
     /**
      * 用户访问黑名单 Redis Key
@@ -188,12 +188,12 @@ public class BlacklistContainer {
         LOG.info("同步本地标签ID与视频列表的缓存任务结束");
     }
 
-    public List<Long> filterUnsafeVideoByUser(List<Long> videoIds, String uid, Long hotSceneType, String cityCode, String clientIP) {
+    public List<Long> filterUnsafeVideoByUser(List<Long> videoIds, String uid, Long hotSceneType, String cityCode, String clientIP, String mid) {
         if (CollectionUtils.isEmpty(videoIds)) {
             return videoIds;
         }
 
-        String userType = this.matchUserBlacklistTypeEnum(uid, hotSceneType, cityCode, clientIP);
+        String userType = this.matchUserBlacklistTypeEnum(uid, hotSceneType, cityCode, clientIP, mid);
         Collection<Long> tagIdSet = this.findExcludeTagIds(userType);
         if (CollectionUtils.isEmpty(tagIdSet)) {
             return videoIds;
@@ -201,23 +201,23 @@ public class BlacklistContainer {
 
         return videoIds.stream().filter(videoId -> {
             if (videoTagAnyMatch(videoId, tagIdSet)) {
-                LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}",
-                        uid, userType, videoId, hotSceneType, cityCode, clientIP);
+                LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}, mid={}",
+                        uid, userType, videoId, hotSceneType, cityCode, clientIP, mid);
                 return false;
             }
             return true;
         }).collect(Collectors.toList());
     }
 
-    private String matchUserBlacklistTypeEnum(String uid, Long hotSceneType, String cityCode, String clientIP) {
+    private String matchUserBlacklistTypeEnum(String uid, Long hotSceneType, String cityCode, String clientIP, String mid) {
         try {
-            LOG.info("计算用户黑名单类型，判断参数: uid={}, hotSceneType={}, cityCode={}, clientIP={}", uid, hotSceneType, cityCode, clientIP);
+            LOG.info("计算用户黑名单类型，判断参数: uid={}, hotSceneType={}, cityCode={}, clientIP={}, mid={}", uid, hotSceneType, cityCode, clientIP, mid);
             if (StringUtils.isNotBlank(uid)) {
                 String key = this.calcUserRedisKey(uid);
                 Map<String, String> uidBlacklistMap = blacklistCache.get(key);
                 if (uidBlacklistMap.containsKey(uid)) {
                     String userType = uidBlacklistMap.get(uid);
-                    this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "UID");
+                    this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "UID", mid);
                     LOG.info("用户 {} 在UID黑名单中命中 {}", uid, userType);
                     return userType;
                 }
@@ -227,7 +227,7 @@ public class BlacklistContainer {
                 Map<String, String> ipBlacklistMap = blacklistCache.get(IP_VISIO_BLACKLIST_HASH_KEY);
                 if (ipBlacklistMap.containsKey(clientIP)) {
                     String userType = ipBlacklistMap.get(clientIP);
-                    this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "IP");
+                    this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "IP", mid);
                     LOG.info("用户 {} 在IP黑名单中命中 {}, 参数为: clientIP为: {}", uid, userType, clientIP);
                     return userType;
                 }
@@ -236,7 +236,7 @@ public class BlacklistContainer {
 
             String userType = this.matchGeneralizationUserType(uid, cityCode, hotSceneType);
             if (StringUtils.isNotBlank(userType)) {
-                this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "RegionAndHotSceneType");
+                this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "RegionAndHotSceneType", mid);
             }
             return userType;
         } catch (
@@ -305,23 +305,24 @@ public class BlacklistContainer {
         return tagFilterConfig.getRecommendExcludeTag();
     }
 
-    private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String fullUserType, String blacklistType) {
+    private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String fullUserType, String blacklistType, String mid) {
         try {
             String[] split = fullUserType.split(USER_TYPE_SUB_TYPE_CONNECTOR);
-            this.filterLogUpload(uid, cityCode, hotSceneType, clientIp, split[0], split[1], blacklistType);
+            this.filterLogUpload(uid, cityCode, hotSceneType, clientIp, split[0], split[1], blacklistType, mid);
         } catch (
                 Exception e) {
             LOG.error("filterLogUpload error: ", e);
         }
     }
 
-    private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String userType, String userSubType, String blacklistType) {
+    private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String userType, String userSubType, String blacklistType, String mid) {
         try {
             Map<String, String> logMap = new HashMap<>();
-            logMap.put("uid", uid);
-            logMap.put("cityCode", cityCode);
-            logMap.put("hotSceneType", hotSceneType.toString());
-            logMap.put("clientIp", clientIp);
+            logMap.put("uid", StringUtils.isNotBlank(uid) ? uid : "");
+            logMap.put("mid", StringUtils.isNotBlank(mid) ? mid : "");
+            logMap.put("cityCode", StringUtils.isNotBlank(cityCode) ? cityCode : "");
+            logMap.put("hotSceneType", Objects.nonNull(hotSceneType) ? hotSceneType.toString() : "");
+            logMap.put("clientIp", StringUtils.isNotBlank(clientIp) ? clientIp : "");
             logMap.put("userType", userType);
             logMap.put("userSubType", userSubType);
             logMap.put("env", activeProfile);
@@ -330,7 +331,7 @@ public class BlacklistContainer {
 
             LogItem logItem = new LogItem();
             logMap.forEach(logItem::PushBack);
-            
+
             ThreadPoolFactory.logPool().execute(() -> {
                 try {
                     producer.send(project, logStore, logItem);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/SecurityStrategy.java

@@ -37,6 +37,9 @@ public class SecurityStrategy implements FilterStrategy {
     @Value("${securityGlobalSwitch:false}")
     private boolean securityGlobalSwitch;
 
+    @Value("${content.security.new.filter.switch:false}")
+    private boolean newContentSecurityFilterSwitch;
+
     @Autowired
     private WxVideoTagRelRepository wxVideoTagRelRepository;
 
@@ -72,6 +75,7 @@ public class SecurityStrategy implements FilterStrategy {
         for (String cityCode : videoFilterCityTagIdMap.keySet()) {
             videoCache.getUnchecked(cityCode);
         }
+        log.info("newContentSecurityFilterSwitch: {}", newContentSecurityFilterSwitch);
     }
 
     @Override
@@ -90,9 +94,14 @@ public class SecurityStrategy implements FilterStrategy {
             return param.getVideoIds();
         }
 
+        if (newContentSecurityFilterSwitch) {
+            return blacklistContainer.filterUnsafeVideoByUser(param.getVideoIds(), param.getUid(),
+                    param.getHotSceneType(), param.getCityCode(), param.getClientIp(), param.getMid());
+        }
+
         if (CollectionUtils.isEmpty(excludeScenes)
                 || !CommonCollectionUtils.contains(excludeScenes, param.getHotSceneType())) {
-
+            log.info("old filter cityCodeAndScene filter");
             if (MapUtils.isEmpty(videoFilterCityTagIdMap)
                     || !videoFilterCityTagIdMap.containsKey(param.getCityCode())) {
                 return param.getVideoIds();
@@ -111,8 +120,6 @@ public class SecurityStrategy implements FilterStrategy {
         }
         return param.getVideoIds();
 
-        // return blacklistContainer.filterUnsafeVideoByUser(param.getVideoIds(), param.getUid(),
-        //         param.getHotSceneType(), param.getCityCode(), param.getClientIp());
 
     }
 }