Merge branch 'feature_filter' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/AllowListStrategy.java

@@ -1,5 +1,8 @@
 package com.tzld.piaoquan.recommend.server.service.filter.strategy;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 import com.google.common.collect.Lists;
 import com.google.common.hash.Hashing;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
@@ -12,7 +15,9 @@ import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.PostConstruct;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author dyp
@@ -24,15 +29,39 @@ public class AllowListStrategy implements FilterStrategy {
     @Qualifier("longVideoRedisTemplate")
     private RedisTemplate<String, String> redisTemplate;
 
-    // private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_PREFIX = "movie:videoid:allowlist:";
+    private LoadingCache<Integer, Set<String>> allowVideoCache = CacheBuilder.newBuilder()
+            .maximumSize(1000000) // TODO 评估容量
+            .refreshAfterWrite(600, TimeUnit.SECONDS)
+            .expireAfterWrite(600, TimeUnit.SECONDS)
+            .expireAfterAccess(600, TimeUnit.SECONDS)
+            .build(new CacheLoader<Integer, Set<String>>() {
+                @Override
+                public Set<String> load(Integer idx) {
+                    String key = VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX + idx;
+                    Set<String> result = redisTemplate.opsForSet().members(key);
+                    if (CollectionUtils.isEmpty(result)) {
+                        return Collections.emptySet();
+                    }
+                    return result;
+                }
+            });
+
     private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX = "movie:videoid:allowSet:";
-    // private static final String VIDEO_ALLOW_LIST_BITMAP_KEY = "movie.store.mp.allowlist.videoid.bitmap";
     private static final String RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY = "mp:religion:allowlist:videoid:bitmap";
-//    @Value("${movie.videoid.allowlist.compatible:1}")
-//    private Integer mvalCompatible;
+
+
+    @PostConstruct
+    public void init() {
+        // 预加载本地缓存
+        for (int i = 0; i < 100; i++) {
+            int finalI = i;
+            allowVideoCache.getUnchecked(finalI);
+            // ThreadPoolFactory.defaultPool().submit(() -> allowVideoCache.getUnchecked(finalI));
+        }
+        log.info("allowVideoCache size {} ", allowVideoCache.size());
+    }
 
     @Override
-    // TODO 未找到优化方法 暂时保留原代码
     public List<Long> filter(FilterParam param) {
         if (param == null
                 || CollectionUtils.isEmpty(param.getVideoIds())) {
@@ -74,22 +103,19 @@ public class AllowListStrategy implements FilterStrategy {
     }
 
     private boolean isMemberOfVideoAllowList(Long videoId) {
+        // TODO 优化
+        // 1. 每个idx 2000条数据 总共20w 约1.6M 可放入本地内存
+        // 2. 部分视频已下推荐，可清理
         if (Objects.isNull(videoId)) {
             return false;
         }
         try {
-            int newIdx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 100;
-            String newPrefix = VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX + newIdx;
-            Boolean result = redisTemplate.opsForSet().isMember(newPrefix, String.valueOf(videoId));
-//            // 兼容旧 key apollo配置0 所以删除
-//            if (Objects.equals(1, mvalCompatible) && !result) {
-//                int idx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 10;
-//                result = redisTemplate.opsForValue().getBit(VIDEO_ALLOW_LIST_BITMAP_KEY_PREFIX + idx, videoId);
-//            }
-//            if (Objects.equals(1, mvalCompatible) && !result) {
-//                result = redisTemplate.opsForValue().getBit(VIDEO_ALLOW_LIST_BITMAP_KEY, videoId);
-//            }
-            return result;
+            int idx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 100;
+            Set<String> allowList = allowVideoCache.get(idx);
+            if (CollectionUtils.isEmpty(allowList)) {
+                return false;
+            }
+            return allowList.contains(String.valueOf(videoId));
         } catch (Exception e) {
             log.error("isMemberOfVideoAllowList error {}", videoId, e);
         }