user upload filter exclude

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/VideoSourceTypeStrategy.java

@@ -1,16 +1,21 @@
 package com.tzld.piaoquan.recommend.server.service.filter.strategy;
 
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.repository.WxVideoStatus;
 import com.tzld.piaoquan.recommend.server.repository.WxVideoStatusRepository;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
 import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
 import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.math.NumberUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -22,6 +27,7 @@ import org.springframework.data.redis.core.SessionCallback;
 import org.springframework.data.redis.core.ValueOperations;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.PostConstruct;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -36,9 +42,11 @@ public class VideoSourceTypeStrategy implements FilterStrategy {
     @Qualifier("redisTemplate")
     private RedisTemplate<String, String> redisTemplate;
 
+
     @Autowired
     private WxVideoStatusRepository wxVideoStatusRepository;
 
+
     private final String keyFormat = "video:uid:%s";
 
     public static final String NOT_USER_UPLOAD_USER_KEY = "not:user:upload:user";
@@ -58,6 +66,33 @@ public class VideoSourceTypeStrategy implements FilterStrategy {
                 }
             });
 
+    @Autowired
+    private WxVideoTagRelRepository repository;
+
+    @ApolloJsonValue("${user.upload.exclude.tagids:[]}")
+    private List<Long> userUploadExcludeTagIds;
+
+    private String userUploadExcludeVideoCacheKey = "user.upload.exclude.video";
+    // 内存持久保存不淘汰
+    private LoadingCache<String, Set<Long>> userUploadExcludeVideoCache = CacheBuilder.newBuilder()
+            .maximumSize(10)
+            .refreshAfterWrite(60, TimeUnit.SECONDS)
+            .expireAfterWrite(60, TimeUnit.SECONDS)
+            .expireAfterAccess(60, TimeUnit.SECONDS)
+            .build(new CacheLoader<String, Set<Long>>() {
+                @Override
+                public Set<Long> load(String key) {
+                    List<WxVideoTagRel> rels = repository.findAllByTagIdIn(userUploadExcludeTagIds);
+                    return CommonCollectionUtils.toSet(rels, WxVideoTagRel::getVideoId);
+                }
+            });
+
+    @PostConstruct
+    public void init() {
+        Set<Long> data = userUploadExcludeVideoCache.getUnchecked(userUploadExcludeVideoCacheKey);
+        log.info("userUploadExcludeVideoCache size {}", data.size());
+    }
+
     @Override
     public List<Long> filter(FilterParam param) {
 
@@ -114,17 +149,12 @@ public class VideoSourceTypeStrategy implements FilterStrategy {
         }
 
         Set<String> notUserUploadUserIds = notUserUploadUserCache.getUnchecked(NOT_USER_UPLOAD_USER_KEY);
+        Set<Long> data = userUploadExcludeVideoCache.getUnchecked(userUploadExcludeVideoCacheKey);
 
         List<Long> videoIds = param.getVideoIds().stream()
-                .filter(l -> vid2UidMap.containsKey(l)
-                        && notUserUploadUserIds.contains(vid2UidMap.get(l)))
+                .filter(l -> data.contains(l)
+                        || (vid2UidMap.containsKey(l) && notUserUploadUserIds.contains(vid2UidMap.get(l))))
                 .collect(Collectors.toList());
-        log.info("VideoSourceTypeStrategy \t param={} \t vid2Uid={} \t before={} \t " +
-                        "after={}",
-                JSONUtils.toJson(param),
-                JSONUtils.toJson(vid2UidMap),
-                JSONUtils.toJson(param.getVideoIds()),
-                JSONUtils.toJson(videoIds));
 
         return videoIds;
     }

recommend-server-service/src/main/resources/application-dev.yml

@@ -100,7 +100,7 @@ spring:
       idle-timeout: 30000
   jpa:
     hibernate:
-      ddl-auto: validate
+      ddl-auto: none
     database: mysql
 xxl:
   job:

recommend-server-service/src/main/resources/application-pre.yml

@@ -99,7 +99,7 @@ spring:
       idle-timeout: 30000
   jpa:
     hibernate:
-      ddl-auto: validate
+      ddl-auto: none
     database: mysql
 xxl:
   job:

recommend-server-service/src/main/resources/application-prod.yml

@@ -99,7 +99,7 @@ spring:
       idle-timeout: 30000
   jpa:
     hibernate:
-      ddl-auto: validate
+      ddl-auto: none
     database: mysql
 xxl:
   job:

recommend-server-service/src/main/resources/application-test.yml

@@ -100,7 +100,7 @@ spring:
       idle-timeout: 30000
   jpa:
     hibernate:
-      ddl-auto: validate
+      ddl-auto: none
     database: mysql
 xxl:
   job: