Merge branch 'featrue/20240416/sunxy/addClientIpParam' of algorithm/recommend-server into master

recommend-server-client/pom.xml

@@ -10,7 +10,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>recommend-server-client</artifactId>
-    <version>1.1.3</version>
+    <version>1.1.4</version>
 
     <dependencies>
         <dependency>

recommend-server-client/src/main/java/com/tzld/piaoquan/recommend/server/gen/recommend/Recommend.java

@@ -56,7 +56,7 @@ public final class Recommend {
       "\n2com/tzld/piaoquan/recommend/server/rec" +
       "ommend.proto\032\031google/protobuf/any.proto\032" +
       "/com/tzld/piaoquan/recommend/server/comm" +
-      "on.proto\"\233\004\n\020RecommendRequest\022\022\n\nrequest" +
+      "on.proto\"\256\004\n\020RecommendRequest\022\022\n\nrequest" +
       "_id\030\001 \001(\t\022\013\n\003mid\030\002 \001(\t\022\013\n\003uid\030\003 \001(\t\022\014\n\004s" +
       "ize\030\004 \001(\005\022\020\n\010app_type\030\005 \001(\005\022\021\n\tcity_code" +
       "\030\006 \001(\t\022\025\n\rprovince_code\030\007 \001(\t\022\023\n\013ab_exp_" +
@@ -68,25 +68,26 @@ public final class Recommend {
       "InfoProto\022\025\n\rnew_exp_group\030\020 \001(\t\022\022\n\nsess" +
       "ion_id\030\021 \001(\t\022\026\n\016sub_session_id\030\022 \001(\t\022\023\n\013" +
       "page_source\030\023 \001(\t\022\023\n\013category_id\030\024 \001(\t\022\026" +
-      "\n\016hot_scene_type\030\025 \001(\003\032.\n\014EventIdEntry\022\013" +
-      "\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\"\177\n\020Machin" +
-      "eInfoProto\022\r\n\005brand\030\001 \001(\t\022\r\n\005model\030\002 \001(\t" +
-      "\022\020\n\010platform\030\003 \001(\t\022\023\n\013sdk_version\030\004 \001(\t\022" +
-      "\016\n\006system\030\005 \001(\t\022\026\n\016wechat_version\030\006 \001(\t\"" +
-      "H\n\021RecommendResponse\022\027\n\006result\030\001 \001(\0132\007.R" +
-      "esult\022\032\n\005video\030\002 \003(\0132\013.VideoProto\"\336\001\n\nVi" +
-      "deoProto\022\020\n\010video_id\030\001 \001(\003\022\021\n\trov_score\030" +
-      "\002 \001(\001\022\021\n\tpush_from\030\003 \001(\t\022\017\n\007ab_code\030\004 \001(" +
-      "\t\022\022\n\nsort_score\030\005 \001(\001\022\020\n\010position\030\006 \001(\005\022" +
-      "\021\n\tflow_pool\030\007 \001(\t\022\027\n\017is_in_flow_pool\030\010 " +
-      "\001(\005\022\014\n\004rand\030\t \001(\001\022\'\n\017push_from_index\030\n \003" +
-      "(\0132\016.PushFromIndex\"1\n\rPushFromIndex\022\021\n\tp" +
-      "ush_from\030\001 \001(\t\022\r\n\005index\030\002 \003(\t2\212\001\n\020Recomm" +
-      "endService\022:\n\021HomepageRecommend\022\021.Recomm" +
-      "endRequest\032\022.RecommendResponse\022:\n\021Releva" +
-      "ntRecommend\022\021.RecommendRequest\032\022.Recomme" +
-      "ndResponseB7\n0com.tzld.piaoquan.recommen" +
-      "d.server.gen.recommendP\001\210\001\001b\006proto3"
+      "\n\016hot_scene_type\030\025 \001(\003\022\021\n\tclient_ip\030\026 \001(" +
+      "\t\032.\n\014EventIdEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030" +
+      "\002 \001(\t:\0028\001\"\177\n\020MachineInfoProto\022\r\n\005brand\030\001" +
+      " \001(\t\022\r\n\005model\030\002 \001(\t\022\020\n\010platform\030\003 \001(\t\022\023\n" +
+      "\013sdk_version\030\004 \001(\t\022\016\n\006system\030\005 \001(\t\022\026\n\016we" +
+      "chat_version\030\006 \001(\t\"H\n\021RecommendResponse\022" +
+      "\027\n\006result\030\001 \001(\0132\007.Result\022\032\n\005video\030\002 \003(\0132" +
+      "\013.VideoProto\"\336\001\n\nVideoProto\022\020\n\010video_id\030" +
+      "\001 \001(\003\022\021\n\trov_score\030\002 \001(\001\022\021\n\tpush_from\030\003 " +
+      "\001(\t\022\017\n\007ab_code\030\004 \001(\t\022\022\n\nsort_score\030\005 \001(\001" +
+      "\022\020\n\010position\030\006 \001(\005\022\021\n\tflow_pool\030\007 \001(\t\022\027\n" +
+      "\017is_in_flow_pool\030\010 \001(\005\022\014\n\004rand\030\t \001(\001\022\'\n\017" +
+      "push_from_index\030\n \003(\0132\016.PushFromIndex\"1\n" +
+      "\rPushFromIndex\022\021\n\tpush_from\030\001 \001(\t\022\r\n\005ind" +
+      "ex\030\002 \003(\t2\212\001\n\020RecommendService\022:\n\021Homepag" +
+      "eRecommend\022\021.RecommendRequest\032\022.Recommen" +
+      "dResponse\022:\n\021RelevantRecommend\022\021.Recomme" +
+      "ndRequest\032\022.RecommendResponseB7\n0com.tzl" +
+      "d.piaoquan.recommend.server.gen.recommen" +
+      "dP\001\210\001\001b\006proto3"
     };
     descriptor = com.google.protobuf.Descriptors.FileDescriptor
       .internalBuildGeneratedFileFrom(descriptorData,
@@ -99,7 +100,7 @@ public final class Recommend {
     internal_static_RecommendRequest_fieldAccessorTable = new
       com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
         internal_static_RecommendRequest_descriptor,
-        new java.lang.String[] { "RequestId", "Mid", "Uid", "Size", "AppType", "CityCode", "ProvinceCode", "AbExpCode", "EventId", "VersionAuditStatus", "RecommendTraceId", "VideoId", "City", "Province", "MachineInfo", "NewExpGroup", "SessionId", "SubSessionId", "PageSource", "CategoryId", "HotSceneType", });
+        new java.lang.String[] { "RequestId", "Mid", "Uid", "Size", "AppType", "CityCode", "ProvinceCode", "AbExpCode", "EventId", "VersionAuditStatus", "RecommendTraceId", "VideoId", "City", "Province", "MachineInfo", "NewExpGroup", "SessionId", "SubSessionId", "PageSource", "CategoryId", "HotSceneType", "ClientIp", });
     internal_static_RecommendRequest_EventIdEntry_descriptor =
       internal_static_RecommendRequest_descriptor.getNestedTypes().get(0);
     internal_static_RecommendRequest_EventIdEntry_fieldAccessorTable = new

recommend-server-client/src/main/java/com/tzld/piaoquan/recommend/server/gen/recommend/RecommendRequest.java

@@ -30,6 +30,7 @@ private static final long serialVersionUID = 0L;
     subSessionId_ = "";
     pageSource_ = "";
     categoryId_ = "";
+    clientIp_ = "";
   }
 
   @java.lang.Override
@@ -201,6 +202,12 @@ private static final long serialVersionUID = 0L;
             hotSceneType_ = input.readInt64();
             break;
           }
+          case 178: {
+            java.lang.String s = input.readStringRequireUtf8();
+
+            clientIp_ = s;
+            break;
+          }
           default: {
             if (!parseUnknownField(
                 input, unknownFields, extensionRegistry, tag)) {
@@ -951,6 +958,44 @@ private static final long serialVersionUID = 0L;
     return hotSceneType_;
   }
 
+  public static final int CLIENT_IP_FIELD_NUMBER = 22;
+  private volatile java.lang.Object clientIp_;
+  /**
+   * <code>string client_ip = 22;</code>
+   * @return The clientIp.
+   */
+  @java.lang.Override
+  public java.lang.String getClientIp() {
+    java.lang.Object ref = clientIp_;
+    if (ref instanceof java.lang.String) {
+      return (java.lang.String) ref;
+    } else {
+      com.google.protobuf.ByteString bs = 
+          (com.google.protobuf.ByteString) ref;
+      java.lang.String s = bs.toStringUtf8();
+      clientIp_ = s;
+      return s;
+    }
+  }
+  /**
+   * <code>string client_ip = 22;</code>
+   * @return The bytes for clientIp.
+   */
+  @java.lang.Override
+  public com.google.protobuf.ByteString
+      getClientIpBytes() {
+    java.lang.Object ref = clientIp_;
+    if (ref instanceof java.lang.String) {
+      com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString.copyFromUtf8(
+              (java.lang.String) ref);
+      clientIp_ = b;
+      return b;
+    } else {
+      return (com.google.protobuf.ByteString) ref;
+    }
+  }
+
   private byte memoizedIsInitialized = -1;
   @java.lang.Override
   public final boolean isInitialized() {
@@ -1031,6 +1076,9 @@ private static final long serialVersionUID = 0L;
     if (hotSceneType_ != 0L) {
       output.writeInt64(21, hotSceneType_);
     }
+    if (!getClientIpBytes().isEmpty()) {
+      com.google.protobuf.GeneratedMessageV3.writeString(output, 22, clientIp_);
+    }
     unknownFields.writeTo(output);
   }
 
@@ -1121,6 +1169,9 @@ private static final long serialVersionUID = 0L;
       size += com.google.protobuf.CodedOutputStream
         .computeInt64Size(21, hotSceneType_);
     }
+    if (!getClientIpBytes().isEmpty()) {
+      size += com.google.protobuf.GeneratedMessageV3.computeStringSize(22, clientIp_);
+    }
     size += unknownFields.getSerializedSize();
     memoizedSize = size;
     return size;
@@ -1181,6 +1232,8 @@ private static final long serialVersionUID = 0L;
         .equals(other.getCategoryId())) return false;
     if (getHotSceneType()
         != other.getHotSceneType()) return false;
+    if (!getClientIp()
+        .equals(other.getClientIp())) return false;
     if (!unknownFields.equals(other.unknownFields)) return false;
     return true;
   }
@@ -1242,6 +1295,8 @@ private static final long serialVersionUID = 0L;
     hash = (37 * hash) + HOT_SCENE_TYPE_FIELD_NUMBER;
     hash = (53 * hash) + com.google.protobuf.Internal.hashLong(
         getHotSceneType());
+    hash = (37 * hash) + CLIENT_IP_FIELD_NUMBER;
+    hash = (53 * hash) + getClientIp().hashCode();
     hash = (29 * hash) + unknownFields.hashCode();
     memoizedHashCode = hash;
     return hash;
@@ -1442,6 +1497,8 @@ private static final long serialVersionUID = 0L;
 
       hotSceneType_ = 0L;
 
+      clientIp_ = "";
+
       return this;
     }
 
@@ -1499,6 +1556,7 @@ private static final long serialVersionUID = 0L;
       result.pageSource_ = pageSource_;
       result.categoryId_ = categoryId_;
       result.hotSceneType_ = hotSceneType_;
+      result.clientIp_ = clientIp_;
       onBuilt();
       return result;
     }
@@ -1629,6 +1687,10 @@ private static final long serialVersionUID = 0L;
       if (other.getHotSceneType() != 0L) {
         setHotSceneType(other.getHotSceneType());
       }
+      if (!other.getClientIp().isEmpty()) {
+        clientIp_ = other.clientIp_;
+        onChanged();
+      }
       this.mergeUnknownFields(other.unknownFields);
       onChanged();
       return this;
@@ -3194,6 +3256,82 @@ private static final long serialVersionUID = 0L;
       onChanged();
       return this;
     }
+
+    private java.lang.Object clientIp_ = "";
+    /**
+     * <code>string client_ip = 22;</code>
+     * @return The clientIp.
+     */
+    public java.lang.String getClientIp() {
+      java.lang.Object ref = clientIp_;
+      if (!(ref instanceof java.lang.String)) {
+        com.google.protobuf.ByteString bs =
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        clientIp_ = s;
+        return s;
+      } else {
+        return (java.lang.String) ref;
+      }
+    }
+    /**
+     * <code>string client_ip = 22;</code>
+     * @return The bytes for clientIp.
+     */
+    public com.google.protobuf.ByteString
+        getClientIpBytes() {
+      java.lang.Object ref = clientIp_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        clientIp_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    /**
+     * <code>string client_ip = 22;</code>
+     * @param value The clientIp to set.
+     * @return This builder for chaining.
+     */
+    public Builder setClientIp(
+        java.lang.String value) {
+      if (value == null) {
+    throw new NullPointerException();
+  }
+  
+      clientIp_ = value;
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>string client_ip = 22;</code>
+     * @return This builder for chaining.
+     */
+    public Builder clearClientIp() {
+      
+      clientIp_ = getDefaultInstance().getClientIp();
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>string client_ip = 22;</code>
+     * @param value The bytes for clientIp to set.
+     * @return This builder for chaining.
+     */
+    public Builder setClientIpBytes(
+        com.google.protobuf.ByteString value) {
+      if (value == null) {
+    throw new NullPointerException();
+  }
+  checkByteStringIsUtf8(value);
+      
+      clientIp_ = value;
+      onChanged();
+      return this;
+    }
     @java.lang.Override
     public final Builder setUnknownFields(
         final com.google.protobuf.UnknownFieldSet unknownFields) {

recommend-server-client/src/main/java/com/tzld/piaoquan/recommend/server/gen/recommend/RecommendRequestOrBuilder.java

@@ -278,4 +278,16 @@ public interface RecommendRequestOrBuilder extends
    * @return The hotSceneType.
    */
   long getHotSceneType();
+
+  /**
+   * <code>string client_ip = 22;</code>
+   * @return The clientIp.
+   */
+  java.lang.String getClientIp();
+  /**
+   * <code>string client_ip = 22;</code>
+   * @return The bytes for clientIp.
+   */
+  com.google.protobuf.ByteString
+      getClientIpBytes();
 }

recommend-server-client/src/main/proto/com/tzld/piaoquan/recommend/server/recommend.proto

@@ -30,6 +30,7 @@ message RecommendRequest {
   string page_source = 19;
   string category_id = 20;
   int64 hot_scene_type = 21;
+  string client_ip = 22;
 }
 
 message MachineInfoProto {

recommend-server-service/pom.xml

@@ -153,7 +153,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-server-client</artifactId>
-            <version>1.1.3</version>
+            <version>1.1.4</version>
         </dependency>
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/Application.java

@@ -11,6 +11,7 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.http.converter.protobuf.ProtobufHttpMessageConverter;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 
 /**
@@ -31,6 +32,7 @@ import org.springframework.http.converter.protobuf.ProtobufHttpMessageConverter;
 })
 @EnableEurekaClient
 @EnableAspectJAutoProxy
+@EnableScheduling
 public class Application {
     public static void main(String[] args) {
         SpringApplication.run(Application.class, args);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/RecommendParam.java

@@ -60,5 +60,7 @@ public class RecommendParam {
 
     private Long hotSceneType;
 
+    private String clientIp;
+
 }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -439,6 +439,7 @@ public class RecommendService {
         }
 
         param.setHotSceneType(request.getHotSceneType());
+        param.setClientIp(request.getClientIp());
 
         return param;
     }
@@ -528,6 +529,7 @@ public class RecommendService {
         recallParam.setAbCode(param.getAbCode());
 
         recallParam.setHotSceneType(param.getHotSceneType());
+        recallParam.setClientIp(param.getClientIp());
 
         return recallParam;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/ViewedService.java

@@ -46,7 +46,7 @@ public class ViewedService {
 
     // TODO 如果过滤失败，那么认为所有视频都被过滤掉
     public List<Long> filterViewedVideo(int appType, String mid, String uid, List<Long> videoIds, String cityCode,
-                                        Set<String> abExpCodes, Long hotSceneType) {
+                                        Set<String> abExpCodes, Long hotSceneType, String clientIp) {
 
         List<Integer> viewedTypes = new ArrayList<>(viewedTypesMap.getOrDefault(appType, defaultViewedTypes));
         if (CommonCollectionUtils.contains(abExpCodes, securityAbExpCode)) {
@@ -65,6 +65,7 @@ public class ViewedService {
             param.put("videoIds", videoIds);
             param.put("cityCode", cityCode);
             param.put("hotSenceType", hotSceneType);
+            param.put("clientIp", clientIp);
             param.put("abExpCodes", abExpCodes);
             List<Integer> recommendStatus = new ArrayList<>();
             recommendStatus.add(-6);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -1,7 +1,6 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.base.Stopwatch;
 import com.google.common.collect.Lists;
 import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.service.PreViewedService;
@@ -119,7 +118,7 @@ public abstract class AbstractFilterService {
         for (final List<Long> ids : chunks) {
             Future<List<Long>> future = pool.submit(() ->
                     viewedService.filterViewedVideo(param.getAppType(), param.getMid(), param.getUid(), ids, param.getCityCode(),
-                            param.getAbExpCodes(), param.getHotSceneType()));
+                            param.getAbExpCodes(), param.getHotSceneType(), param.getClientIp()));
             futures.add(future);
         }
         try {
@@ -150,7 +149,7 @@ public abstract class AbstractFilterService {
             return videoIds;
         }
         return viewedService.filterViewedVideo(param.getAppType(), param.getMid(), param.getUid(), videoIds, param.getCityCode(),
-                param.getAbExpCodes(), param.getHotSceneType());
+                param.getAbExpCodes(), param.getHotSceneType(), param.getClientIp());
 
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterParam.java

@@ -36,4 +36,6 @@ public class FilterParam {
 
     private Long hotSceneType;
 
+    private String clientIp;
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/BlacklistContainer.java

@@ -0,0 +1,361 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.aliyun.openservices.aliyun.log.producer.LogProducer;
+import com.aliyun.openservices.aliyun.log.producer.Producer;
+import com.aliyun.openservices.aliyun.log.producer.ProducerConfig;
+import com.aliyun.openservices.aliyun.log.producer.ProjectConfig;
+import com.aliyun.openservices.log.common.LogItem;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
+import lombok.Data;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Nonnull;
+import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * 黑名单列表相关的容器。主要实现以下几个功能
+ * <ul>
+ *     <ol>1. 判断用户属于哪个类型的黑名单</ol>
+ *     <ol>2. 根据用户类型判断视频对于该用户是否有风险</ol>
+ *     <ol>3. 根据用户类型过滤掉视频列表中对该用户有风险的视频</ol>
+ * </ul>
+ */
+@Component
+public class BlacklistContainer {
+
+
+    private static final Logger LOG = LoggerFactory.getLogger(BlacklistContainer.class);
+
+    public static final String USER_TYPE_SUB_TYPE_CONNECTOR = ":";
+
+    private static final int USER_REDIS_KEY_PARTITION_COUNT = 100;
+
+    /**
+     * 用户访问黑名单 Redis Key
+     */
+    private static final String USER_VISIO_BLACKLIST_HASH_KEY = "visio:blacklist:user:";
+
+    /**
+     * IP访问黑名单 Redis Key
+     */
+    private static final String IP_VISIO_BLACKLIST_HASH_KEY = "visio:blacklist:ip";
+
+    @Autowired
+    @Qualifier("longVideoRedisTemplate")
+    private RedisTemplate<String, String> longVideoRedisTemplate;
+
+    @Resource
+    private WxVideoTagRelRepository wxVideoTagRelRepository;
+
+    @ApolloJsonValue("${content.security.generalization.user.condition.config:{}}")
+    private Map<String, GeneralizationUserConfig> generalizationUserConditionConfig;
+
+    /**
+     * 不同类型的用户要过滤掉的标签列表配置
+     * <br >
+     * <p>
+     * Key的格式为: {userType}:{userSubType}
+     * <br>
+     * userType枚举值: 1-竞品用户, 2-微信人员, 3-网安
+     * <br >
+     * userSubType枚举值: 1-精准用户, 2-泛化用户
+     */
+    @ApolloJsonValue("${content.security.filter.config:{}}")
+    private Map<String, TagFilterConfig> tagFilterConfigMap;
+
+    @Value("${aliyun.log.endpoint}")
+    private String endpoint;
+    @Value("${aliyun.log.accessKeyId}")
+    private String accessKeyId;
+    @Value("${aliyun.log.accessKeySecret}")
+    private String accessKeySecret;
+
+    @Value("${aliyun.blacklist.filter.log.project}")
+    private String project;
+    @Value("${aliyun.blacklist.filter.log.store}")
+    private String logStore;
+    @Value("${spring.profiles.active}")
+    private String activeProfile;
+    private Producer producer;
+
+    /**
+     * 保存Tag标签与视频列表的映射
+     * <br>
+     * Key为TagId,  Value为对应的视频ID列表
+     */
+    private static Map<Long, Set<Long>> videoTagCache = new ConcurrentHashMap<>();
+
+    /**
+     * 黑名单本地二级缓存，一级缓存为Redis缓存。此处直接读取即可
+     * <br />
+     * Redis缓存由longvideo服务写入
+     * <br>
+     * com.weiqu.video.service.filter.impl.BlacklistFilterImpl#refreshUidCache
+     * <br>
+     * com.weiqu.video.service.filter.impl.BlacklistFilterImpl#refreshIPCache
+     */
+    private final LoadingCache<String, Map<String, String>> blacklistCache = CacheBuilder.newBuilder()
+            .expireAfterWrite(10, TimeUnit.MINUTES)
+            .build(new CacheLoader<String, Map<String, String>>() {
+                @Override
+                public Map<String, String> load(@Nonnull String key) {
+                    Map<Object, Object> map = longVideoRedisTemplate.opsForHash().entries(key);
+                    if (MapUtils.isEmpty(map)) {
+                        return new HashMap<>();
+                    }
+                    return map.entrySet().stream().collect(
+                            Collectors.toMap(
+                                    entry -> entry.getKey().toString(),
+                                    entry -> entry.getValue().toString(),
+                                    (v1, v2) -> v1
+                            ));
+                }
+            });
+
+    @PostConstruct
+    public void init() {
+        LOG.info("generalizationUserConditionConfig: {}", generalizationUserConditionConfig);
+        LOG.info("tagFilterConfigMap: {}", tagFilterConfigMap);
+        refreshVideoTagCache();
+        initLogProducer();
+    }
+
+    @Scheduled(cron = "0 0/5 * * * ? ")
+    public void cronSync() {
+        refreshVideoTagCache();
+    }
+
+    private void initLogProducer() {
+        LOG.info("BlacklistContainer.initLogProducer: project={}, endpoint={}, accessKeyId={}, accessKeySecret={}, logStore={}",
+                project, endpoint, accessKeyId, accessKeySecret, logStore);
+        ProducerConfig producerConfig = new ProducerConfig();
+        producer = new LogProducer(producerConfig);
+        producer.putProjectConfig(new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret));
+    }
+
+    public void refreshVideoTagCache() {
+        LOG.info("同步本地标签ID与视频列表的缓存任务开始");
+        Map<Long, Set<Long>> tmpMap = new ConcurrentHashMap<>();
+
+        if (MapUtils.isNotEmpty(tagFilterConfigMap)) {
+
+            // 获取所有的标签ID列表
+            Set<Long> tagIdSet = new HashSet<>();
+            for (Map.Entry<String, TagFilterConfig> entry : tagFilterConfigMap.entrySet()) {
+                TagFilterConfig tagFilterConfig = entry.getValue();
+                if (Objects.isNull(tagFilterConfig)) {
+                    continue;
+                }
+                if (CollectionUtils.isNotEmpty(tagFilterConfig.getRecommendExcludeTag())) {
+                    tagIdSet.addAll(tagFilterConfig.getRecommendExcludeTag());
+                }
+                if (CollectionUtils.isNotEmpty(tagFilterConfig.getDetailExcludeTag())) {
+                    tagIdSet.addAll(tagFilterConfig.getDetailExcludeTag());
+                }
+            }
+
+            // 获取标签ID对应的视频ID列表
+            for (Long tagId : tagIdSet) {
+                List<WxVideoTagRel> wxVideoTagRels = wxVideoTagRelRepository.findAllByTagId(tagId);
+                Set<Long> videoIdSet = wxVideoTagRels.stream().map(WxVideoTagRel::getVideoId).collect(Collectors.toSet());
+                LOG.info("同步本地标签ID与视频列表缓存任务 -- tagId: {}, videoIdSize: {}", tagId, tagIdSet.size());
+                tmpMap.put(tagId, videoIdSet);
+            }
+        }
+        videoTagCache = tmpMap;
+
+        LOG.info("同步本地标签ID与视频列表的缓存任务结束");
+    }
+
+    public List<Long> filterUnsafeVideoByUser(List<Long> videoIds, String uid, Long hotSceneType, String cityCode, String clientIP) {
+        if (CollectionUtils.isEmpty(videoIds)) {
+            return videoIds;
+        }
+
+        String userType = this.matchUserBlacklistTypeEnum(uid, hotSceneType, cityCode, clientIP);
+        Collection<Long> tagIdSet = this.findExcludeTagIds(userType);
+        if (CollectionUtils.isEmpty(tagIdSet)) {
+            return videoIds;
+        }
+
+        return videoIds.stream().filter(videoId -> {
+            if (videoTagAnyMatch(videoId, tagIdSet)) {
+                LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}",
+                        uid, userType, videoId, hotSceneType, cityCode, clientIP);
+                return false;
+            }
+            return true;
+        }).collect(Collectors.toList());
+    }
+
+    private String matchUserBlacklistTypeEnum(String uid, Long hotSceneType, String cityCode, String clientIP) {
+        try {
+            LOG.info("计算用户黑名单类型，判断参数: uid={}, hotSceneType={}, cityCode={}, clientIP={}", uid, hotSceneType, cityCode, clientIP);
+            String key = this.calcUserRedisKey(uid);
+            Map<String, String> uidBlacklistMap = blacklistCache.get(key);
+            if (uidBlacklistMap.containsKey(uid)) {
+                String userType = uidBlacklistMap.get(uid);
+                this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "UID");
+                LOG.info("用户 {} 在UID黑名单中命中 {}", uid, userType);
+                return userType;
+            }
+
+            Map<String, String> ipBlacklistMap = blacklistCache.get(IP_VISIO_BLACKLIST_HASH_KEY);
+            if (ipBlacklistMap.containsKey(clientIP)) {
+                String userType = ipBlacklistMap.get(clientIP);
+                this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "IP");
+                LOG.info("用户 {} 在IP黑名单中命中 {}, 参数为: clientIP为: {}", uid, userType, clientIP);
+                return userType;
+            }
+
+            String userType = this.matchGeneralizationUserType(uid, cityCode, hotSceneType);
+            this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "RegionAndHotSceneType");
+            return userType;
+        } catch (
+                Exception e) {
+            LOG.error("blacklist filter isSafeVideoByUid error: ", e);
+        }
+        return null;
+    }
+
+    private String matchGeneralizationUserType(String uid, String cityCode, Long hotSceneType) {
+        if (MapUtils.isNotEmpty(generalizationUserConditionConfig) && generalizationUserConditionConfig.containsKey(cityCode)) {
+            GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(cityCode);
+            if (CollectionUtils.isNotEmpty(userConfig.getExcludeHotSceneType())) {
+                if (!userConfig.getExcludeHotSceneType().contains(hotSceneType)) {
+                    String userType = userConfig.getUserType() + USER_TYPE_SUB_TYPE_CONNECTOR + userConfig.getUserSubType();
+                    LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: cityCode={}, hotSceneType={}", uid, userType, cityCode, hotSceneType);
+                    return userType;
+                }
+            }
+        }
+        return null;
+    }
+
+    /**
+     * 匹配videoId的标签包含tagIds中的任意一个
+     *
+     * @param videoId 视频ID
+     * @param tagIds  标签ID列表
+     * @return true-匹配，false-不匹配
+     */
+    private boolean videoTagAnyMatch(Long videoId, Collection<Long> tagIds) {
+        if (MapUtils.isEmpty(videoTagCache) || CollectionUtils.isEmpty(tagIds)) {
+            return false;
+        }
+        for (Long tagId : tagIds) {
+            Set<Long> videoIds = videoTagCache.get(tagId);
+            if (CollectionUtils.isNotEmpty(videoIds) && videoIds.contains(videoId)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private String calcUserRedisKey(String uidStr) {
+        long uid = 0L;
+        try {
+            uid = Long.parseLong(uidStr);
+        } catch (
+                Exception e) {
+            LOG.error("calcUserRedisKey error: ", e);
+        }
+        return USER_VISIO_BLACKLIST_HASH_KEY + (uid % USER_REDIS_KEY_PARTITION_COUNT);
+    }
+
+    private Collection<Long> findExcludeTagIds(String userType) {
+        if (StringUtils.isBlank(userType) || MapUtils.isEmpty(tagFilterConfigMap)) {
+            return Collections.emptySet();
+        }
+
+        TagFilterConfig tagFilterConfig = tagFilterConfigMap.get(userType);
+        if (Objects.isNull(tagFilterConfig)) {
+            return Collections.emptySet();
+        }
+        return tagFilterConfig.getRecommendExcludeTag();
+    }
+
+    private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String fullUserType, String blacklistType) {
+        try {
+            String[] split = fullUserType.split(USER_TYPE_SUB_TYPE_CONNECTOR);
+            this.filterLogUpload(uid, cityCode, hotSceneType, clientIp, split[0], split[1], blacklistType);
+        } catch (
+                Exception e) {
+            LOG.error("filterLogUpload error: ", e);
+        }
+    }
+
+    private void filterLogUpload(String uid, String cityCode, Long hotSceneType, String clientIp, String userType, String userSubType, String blacklistType) {
+        try {
+            Map<String, String> logMap = new HashMap<>();
+            logMap.put("uid", uid);
+            logMap.put("cityCode", cityCode);
+            logMap.put("hotSceneType", hotSceneType.toString());
+            logMap.put("clientIp", clientIp);
+            logMap.put("userType", userType);
+            logMap.put("userSubType", userSubType);
+            logMap.put("env", activeProfile);
+            logMap.put("blacklistType", blacklistType);
+            logMap.put("uploadService", "recommend-server");
+
+            LogItem logItem = new LogItem();
+            logMap.forEach(logItem::PushBack);
+            
+            ThreadPoolFactory.logPool().execute(() -> {
+                try {
+                    producer.send(project, logStore, logItem);
+                } catch (
+                        Exception e) {
+                    LOG.error("log send error: ", e);
+                }
+            });
+        } catch (
+                Exception e) {
+            LOG.error("blacklist filter upload log error: ", e);
+        }
+    }
+
+    @Data
+    private static class GeneralizationUserConfig {
+        Set<Long> excludeHotSceneType;
+        String userType;
+        String userSubType;
+
+    }
+
+    @Data
+    private static class TagFilterConfig {
+        /**
+         * 推荐场景下要过滤掉的标签
+         */
+        Set<Long> recommendExcludeTag;
+        /**
+         * 详情场景下要过滤掉的标签
+         */
+        Set<Long> detailExcludeTag;
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/SecurityStrategy.java

@@ -17,9 +17,9 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 /**
  * @author dyp
@@ -39,6 +39,9 @@ public class SecurityStrategy implements FilterStrategy {
     @Autowired
     private WxVideoTagRelRepository wxVideoTagRelRepository;
 
+    @Resource
+    private BlacklistContainer blacklistContainer;
+
     // 内存持久保存不淘汰
     private LoadingCache<String, Set<Long>> videoCache = CacheBuilder.newBuilder()
             .maximumSize(100)
@@ -86,25 +89,29 @@ public class SecurityStrategy implements FilterStrategy {
             return param.getVideoIds();
         }
 
-        if (CollectionUtils.isEmpty(excludeScenes)
-                || !CommonCollectionUtils.contains(excludeScenes, param.getHotSceneType())) {
-
-            if (MapUtils.isEmpty(videoFilterCityTagIdMap)
-                    || !videoFilterCityTagIdMap.containsKey(param.getCityCode())) {
-                return param.getVideoIds();
-            }
+        // if (CollectionUtils.isEmpty(excludeScenes)
+        //         || !CommonCollectionUtils.contains(excludeScenes, param.getHotSceneType())) {
+        //
+        //     if (MapUtils.isEmpty(videoFilterCityTagIdMap)
+        //             || !videoFilterCityTagIdMap.containsKey(param.getCityCode())) {
+        //         return param.getVideoIds();
+        //     }
+        //
+        //     Set<Long> filterVideos = videoCache.getUnchecked(param.getCityCode());
+        //     if (CollectionUtils.isEmpty(filterVideos)) {
+        //         return param.getVideoIds();
+        //     }
+        //
+        //     List<Long> result = param.getVideoIds().stream()
+        //             .filter(l -> !filterVideos.contains(l))
+        //             .collect(Collectors.toList());
+        //
+        //     return result;
+        // }
+        // return param.getVideoIds();
 
-            Set<Long> filterVideos = videoCache.getUnchecked(param.getCityCode());
-            if (CollectionUtils.isEmpty(filterVideos)) {
-                return param.getVideoIds();
-            }
+        return blacklistContainer.filterUnsafeVideoByUser(param.getVideoIds(), param.getUid(),
+                param.getHotSceneType(), param.getCityCode(), param.getClientIp());
 
-            List<Long> result = param.getVideoIds().stream()
-                    .filter(l -> !filterVideos.contains(l))
-                    .collect(Collectors.toList());
-
-            return result;
-        }
-        return param.getVideoIds();
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/FilterParamFactory.java

@@ -28,6 +28,7 @@ public class FilterParamFactory {
         filterParam.setCityCode(param.getCityCode());
         filterParam.setAbCode(param.getAbCode());
         filterParam.setHotSceneType(param.getHotSceneType());
+        filterParam.setClientIp(param.getClientIp());
         return filterParam;
     }
 
@@ -48,6 +49,7 @@ public class FilterParamFactory {
         filterParam.setCityCode(param.getCityCode());
         filterParam.setAbCode(param.getAbCode());
         filterParam.setHotSceneType(param.getHotSceneType());
+        filterParam.setClientIp(param.getClientIp());
         return filterParam;
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallParam.java

@@ -41,4 +41,6 @@ public class RecallParam {
 
     private Long hotSceneType;
 
+    private String clientIp;
+
 }

recommend-server-service/src/main/resources/application-dev.yml

@@ -114,6 +114,11 @@ aliyun:
     log:
       project: recommend-server-test
       logStore: timer
+  blacklist:
+    filter:
+      log:
+        project: wqsd-video-test
+        store: video_blacklist_security_filter_log
 
 logging:
   file:

recommend-server-service/src/main/resources/application-pre.yml

@@ -112,4 +112,9 @@ aliyun:
   timer:
     log:
       project: recommend-server
-      logStore: timer
+      logStore: timer
+  blacklist:
+    filter:
+      log:
+        project: wqsd-video
+        store: video_blacklist_security_filter_log

recommend-server-service/src/main/resources/application-prod.yml

@@ -112,4 +112,9 @@ aliyun:
   timer:
     log:
       project: recommend-server
-      logStore: timer
+      logStore: timer
+  blacklist:
+    filter:
+      log:
+        project: wqsd-video
+        store: video_blacklist_security_filter_log

recommend-server-service/src/main/resources/application-test.yml

@@ -113,4 +113,9 @@ aliyun:
   timer:
     log:
       project: recommend-server-test
-      logStore: timer
+      logStore: timer
+  blacklist:
+    filter:
+      log:
+        project: wqsd-video-test
+        store: video_blacklist_security_filter_log