Merge branch 'feature_20240426_zhaohaipeng_content_security_v3' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/BlacklistContainer.java

@@ -47,7 +47,7 @@ public class BlacklistContainer {
 
     private static final Logger LOG = LoggerFactory.getLogger(BlacklistContainer.class);
 
-    public static final String USER_TYPE_SUB_TYPE_CONNECTOR = ":";
+    public static final String CONNECTOR_STR = ":";
 
     private static final int USER_REDIS_KEY_PARTITION_COUNT = 10;
 
@@ -188,21 +188,21 @@ public class BlacklistContainer {
         LOG.info("同步本地标签ID与视频列表的缓存任务结束");
     }
 
-    public List<Long> filterUnsafeVideoByUser(List<Long> videoIds, String uid, Long hotSceneType, String cityCode, String clientIP, String mid, String usedScene) {
+    public List<Long> filterUnsafeVideoByUser(List<Long> videoIds, String uid, Long hotSceneType, String cityCode, String clientIP, String mid, String usedScene, Integer appType) {
         if (CollectionUtils.isEmpty(videoIds)) {
             return videoIds;
         }
 
-        String userType = this.matchUserBlacklistTypeEnum(uid, hotSceneType, cityCode, clientIP, mid, usedScene);
-        Collection<Long> tagIdSet = this.findExcludeTagIds(userType);
+        String userType = this.matchUserBlacklistTypeEnum(uid, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
+        Collection<Long> tagIdSet = this.findRecommendExcludeTagIds(userType, appType);
         if (CollectionUtils.isEmpty(tagIdSet)) {
             return videoIds;
         }
 
         return videoIds.stream().filter(videoId -> {
             if (videoTagAnyMatch(videoId, tagIdSet)) {
-                LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}, mid={}",
-                        uid, userType, videoId, hotSceneType, cityCode, clientIP, mid);
+                LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}, mid={}, usedScene={}, appType={}",
+                        uid, userType, videoId, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
                 return false;
             }
             return true;
@@ -210,15 +210,16 @@ public class BlacklistContainer {
     }
 
     public String matchUserBlacklistTypeEnum(String uid, Long hotSceneType, String cityCode, String clientIP, String mid,
-                                             String usedScene) {
+                                             String usedScene, Integer appType) {
         try {
-            LOG.info("计算用户黑名单类型，判断参数: uid={}, hotSceneType={}, cityCode={}, clientIP={}, mid={}", uid, hotSceneType, cityCode, clientIP, mid);
+            LOG.info("计算用户黑名单类型，判断参数: uid={}, hotSceneType={}, cityCode={}, clientIP={}, mid={}, usedScene={}, appType={}",
+                    uid, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
             if (StringUtils.isNotBlank(uid)) {
                 String key = this.calcUserRedisKey(uid);
                 Map<String, String> uidBlacklistMap = blacklistCache.get(key);
                 if (uidBlacklistMap.containsKey(uid)) {
                     String userType = uidBlacklistMap.get(uid);
-                    this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "UID", mid, usedScene);
+                    this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "UID", mid, usedScene, appType);
                     LOG.info("用户 {} 在UID黑名单中命中 {}", uid, userType);
                     return userType;
                 }
@@ -228,36 +229,60 @@ public class BlacklistContainer {
                 Map<String, String> ipBlacklistMap = blacklistCache.get(IP_VISIO_BLACKLIST_HASH_KEY);
                 if (ipBlacklistMap.containsKey(clientIP)) {
                     String userType = ipBlacklistMap.get(clientIP);
-                    this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "IP", mid, usedScene);
+                    this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "IP", mid, usedScene, appType);
                     LOG.info("用户 {} 在IP黑名单中命中 {}, 参数为: clientIP为: {}", uid, userType, clientIP);
                     return userType;
                 }
             }
 
 
-            String userType = this.matchGeneralizationUserType(uid, cityCode, hotSceneType);
+            String userType = this.matchGeneralizationUserType(uid, cityCode, hotSceneType, appType);
             if (StringUtils.isNotBlank(userType)) {
-                this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "RegionAndHotSceneType", mid, usedScene);
+                this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "RegionAndHotSceneType", mid, usedScene, appType);
             }
             return userType;
         } catch (
                 Exception e) {
             LOG.error("blacklist filter isSafeVideoByUid error: ", e);
+
         }
         return null;
     }
 
-    private String matchGeneralizationUserType(String uid, String cityCode, Long hotSceneType) {
-        if (StringUtils.isNotBlank(cityCode) && Objects.nonNull(hotSceneType)) {
-            if (MapUtils.isNotEmpty(generalizationUserConditionConfig) && generalizationUserConditionConfig.containsKey(cityCode)) {
-                GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(cityCode);
-                if (CollectionUtils.isNotEmpty(userConfig.getExcludeHotSceneType())) {
-                    if (!userConfig.getExcludeHotSceneType().contains(hotSceneType)) {
-                        String userType = userConfig.getUserType() + USER_TYPE_SUB_TYPE_CONNECTOR + userConfig.getUserSubType();
-                        LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: cityCode={}, hotSceneType={}", uid, userType, cityCode, hotSceneType);
-                        return userType;
-                    }
-                }
+    private String matchGeneralizationUserType(String uid, String cityCode, Long hotSceneType, Integer appType) {
+        // 参数为空，则跳过
+        if (StringUtils.isBlank(cityCode) && Objects.isNull(appType)) {
+            return null;
+        }
+        // 配置为空，则跳过
+        if (MapUtils.isEmpty(generalizationUserConditionConfig)) {
+            return null;
+        }
+
+        // 优先找，城市+appType的配置
+        if (StringUtils.isNotBlank(cityCode) && Objects.nonNull(appType)) {
+            GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(cityCode + CONNECTOR_STR + appType);
+            if (Objects.nonNull(userConfig) && userConfig.isExcludeHotSceneType(hotSceneType)) {
+                LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: appType = {}, cityCode={}, hotSceneType={}", uid, userConfig.fullUserType, appType, cityCode, hotSceneType);
+                return userConfig.getFullUserType();
+            }
+        }
+
+        // 其次找，appType的配置
+        if (Objects.nonNull(appType)) {
+            GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(appType.toString());
+            if (Objects.nonNull(userConfig) && userConfig.isExcludeHotSceneType(hotSceneType)) {
+                LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: appType={}, hotSceneType={}", uid, userConfig.fullUserType, appType, hotSceneType);
+                return userConfig.getFullUserType();
+            }
+        }
+
+        // 最后找，城市的配置
+        if (StringUtils.isNotBlank(cityCode)) {
+            GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(cityCode);
+            if (Objects.nonNull(userConfig) && userConfig.isExcludeHotSceneType(hotSceneType)) {
+                LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: cityCode={}, hotSceneType={}", uid, userConfig.fullUserType, cityCode, hotSceneType);
+                return userConfig.getFullUserType();
             }
         }
         return null;
@@ -294,30 +319,34 @@ public class BlacklistContainer {
         return USER_VISIO_BLACKLIST_HASH_KEY + (uid % USER_REDIS_KEY_PARTITION_COUNT);
     }
 
-    private Collection<Long> findExcludeTagIds(String userType) {
+    private Collection<Long> findRecommendExcludeTagIds(String userType, Integer appType) {
         if (StringUtils.isBlank(userType) || MapUtils.isEmpty(tagFilterConfigMap)) {
             return Collections.emptySet();
         }
 
-        TagFilterConfig tagFilterConfig = tagFilterConfigMap.get(userType);
-        if (Objects.isNull(tagFilterConfig)) {
-            return Collections.emptySet();
+        // 查看单APP配置的
+        if (StringUtils.isNotBlank(userType) && Objects.nonNull(appType)) {
+            String key = userType + CONNECTOR_STR + appType;
+            TagFilterConfig tagFilterConfig = tagFilterConfigMap.get(key);
+            if (Objects.nonNull(tagFilterConfig)) {
+                LOG.info("命中过滤标签配置: {} == {}", key, tagFilterConfig.getRecommendExcludeTag());
+                return tagFilterConfig.getRecommendExcludeTag();
+            }
         }
-        return tagFilterConfig.getRecommendExcludeTag();
-    }
 
-    private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String fullUserType, String blacklistType, String mid, String usedScene) {
-        try {
-            String[] split = fullUserType.split(USER_TYPE_SUB_TYPE_CONNECTOR);
-            this.filterLogUpload(uid, cityCode, hotSceneType, clientIp, split[0], split[1], blacklistType, mid, usedScene);
-        } catch (
-                Exception e) {
-            LOG.error("filterLogUpload error: ", e);
+        if (StringUtils.isNotBlank(userType)) {
+            TagFilterConfig tagFilterConfig = tagFilterConfigMap.get(userType);
+            if (Objects.nonNull(tagFilterConfig)) {
+                LOG.info("命中过滤标签配置: {} == {}", userType, tagFilterConfig.getRecommendExcludeTag());
+                return tagFilterConfig.getRecommendExcludeTag();
+            }
         }
+        return null;
     }
 
-    private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String userType, String userSubType, String blacklistType, String mid, String usedScene) {
+    private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String fullUserType, String blacklistType, String mid, String usedScene, Integer appType) {
         try {
+            String[] split = fullUserType.split(CONNECTOR_STR);
             Map<String, String> logMap = new HashMap<>();
             logMap.put("uid", StringUtils.isNotBlank(uid) ? uid : "");
             logMap.put("mid", StringUtils.isNotBlank(mid) ? mid : "");
@@ -325,8 +354,9 @@ public class BlacklistContainer {
             logMap.put("hotSceneType", Objects.nonNull(hotSceneType) ? hotSceneType.toString() : "");
             logMap.put("clientIp", StringUtils.isNotBlank(clientIp) ? clientIp : "");
             logMap.put("usedScene", StringUtils.isNotBlank(usedScene) ? usedScene : "");
-            logMap.put("userType", userType);
-            logMap.put("userSubType", userSubType);
+            logMap.put("appType", Objects.nonNull(appType) ? appType.toString() : "");
+            logMap.put("userType", split[0]);
+            logMap.put("userSubType", split[1]);
             logMap.put("env", activeProfile);
             logMap.put("blacklistType", blacklistType);
             logMap.put("uploadService", "recommend-server");
@@ -353,6 +383,12 @@ public class BlacklistContainer {
         Set<Long> excludeHotSceneType;
         String userType;
         String userSubType;
+        String fullUserType;
+
+        public boolean isExcludeHotSceneType(Long hotSceneType) {
+            return CollectionUtils.isEmpty(excludeHotSceneType)
+                    || !excludeHotSceneType.contains(hotSceneType);
+        }
 
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/SecurityStrategy.java

@@ -95,8 +95,8 @@ public class SecurityStrategy implements FilterStrategy {
         }
 
         if (newContentSecurityFilterSwitch) {
-            return blacklistContainer.filterUnsafeVideoByUser(param.getVideoIds(), param.getUid(),
-                    param.getHotSceneType(), param.getCityCode(), param.getClientIp(), param.getMid(), "recommend");
+            return blacklistContainer.filterUnsafeVideoByUser(param.getVideoIds(), param.getUid(), param.getHotSceneType(),
+                    param.getCityCode(), param.getClientIp(), param.getMid(), "recommend", param.getAppType());
         }
 
         if (CollectionUtils.isEmpty(excludeScenes)

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -84,7 +84,7 @@ public class RecallService implements ApplicationContextAware {
             return strategies;
         }
         String matchUserBlacklistTypeEnum = blacklistContainer.matchUserBlacklistTypeEnum(param.getUid(), param.getHotSceneType(), param.getCityCode(),
-                param.getClientIp(), param.getMid(), "recommend-flow-pool");
+                param.getClientIp(), param.getMid(), "recommend-flow-pool", param.getAppType());
         boolean hitUserBlacklist = StringUtils.isNotBlank(matchUserBlacklistTypeEnum);
 
         String abCode = param.getAbCode();