ADD：慢查询优化

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/repository/WxVideoTagRelRepository.java

@@ -1,5 +1,7 @@
 package com.tzld.piaoquan.recommend.server.repository;
 
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
@@ -10,4 +12,6 @@ public interface WxVideoTagRelRepository extends JpaRepository<WxVideoTagRel, Lo
     List<WxVideoTagRel> findAllByTagId(Long tagId);
 
     List<WxVideoTagRel> findAllByTagIdIn(List<Long> tagId);
+
+    Page<WxVideoTagRel> findAllByTagIdInOrderByCreateTimeAsc(List<Long> tagId, Pageable pageable);
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/BlacklistContainer.java

@@ -21,13 +21,20 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.redis.core.Cursor;
+import org.springframework.data.redis.core.HashOperations;
 import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.core.ScanOptions;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.Nonnull;
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
+import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
@@ -84,6 +91,9 @@ public class BlacklistContainer {
     @ApolloJsonValue("${content.security.filter.config:{}}")
     private Map<String, TagFilterConfig> tagFilterConfigMap;
 
+    @Value("${content.security.filter.queryTagPageSize:10000}")
+    private Integer queryTagPageSize;
+
     @Value("${aliyun.log.endpoint}")
     private String endpoint;
     @Value("${aliyun.log.accessKeyId}")
@@ -120,7 +130,7 @@ public class BlacklistContainer {
             .build(new CacheLoader<String, Map<String, String>>() {
                 @Override
                 public Map<String, String> load(@Nonnull String key) {
-                    Map<Object, Object> map = longVideoRedisTemplate.opsForHash().entries(key);
+                    Map<Object, Object> map = scanHashEntries(longVideoRedisTemplate, key);
                     if (MapUtils.isEmpty(map)) {
                         return new HashMap<>();
                     }
@@ -133,6 +143,22 @@ public class BlacklistContainer {
                 }
             });
 
+    private static Map<Object, Object> scanHashEntries(RedisTemplate<String, String> redisTemplate, String hashKey) {
+        HashOperations<String, String, String> hashOps = redisTemplate.opsForHash();
+        ScanOptions options = ScanOptions.scanOptions().count(5000).build();  // 每批处理 X 个元素
+
+        Map<Object, Object> result = new HashMap<>();
+        try (Cursor<Map.Entry<String, String>> cursor = hashOps.scan(hashKey, options)) {
+            while (cursor.hasNext()) {
+                Map.Entry<String, String> entry = cursor.next();
+                result.put(entry.getKey(), entry.getValue());
+            }
+        } catch (IOException e) {
+            LOG.error("scanHashEntries error: ", e);
+        }
+        return result;
+    }
+
     @PostConstruct
     public void init() {
         LOG.info("generalizationUserConditionConfig: {}", generalizationUserConditionConfig);
@@ -175,13 +201,39 @@ public class BlacklistContainer {
                 }
             }
 
-            // 获取标签ID对应的视频ID列表
-            for (Long tagId : tagIdSet) {
-                List<WxVideoTagRel> wxVideoTagRels = wxVideoTagRelRepository.findAllByTagId(tagId);
-                Set<Long> videoIdSet = wxVideoTagRels.stream().map(WxVideoTagRel::getVideoId).collect(Collectors.toSet());
-                // LOG.info("同步本地标签ID与视频列表缓存任务 -- tagId: {}, videoIdSize: {}", tagId, videoIdSet.size());
-                tmpMap.put(tagId, videoIdSet);
+//            // 获取标签ID对应的视频ID列表
+//            for (Long tagId : tagIdSet) {
+//                List<WxVideoTagRel> wxVideoTagRels = wxVideoTagRelRepository.findAllByTagId(tagId);
+//                Set<Long> videoIdSet = wxVideoTagRels.stream().map(WxVideoTagRel::getVideoId).collect(Collectors.toSet());
+//                // LOG.info("同步本地标签ID与视频列表缓存任务 -- tagId: {}, videoIdSize: {}", tagId, videoIdSet.size());
+//                tmpMap.put(tagId, videoIdSet);
+//            }
+            int page = 0; // 页码，从0开始
+            int maxFor = 100, forCount = 0;
+            List<WxVideoTagRel> totalList = new ArrayList<>();
+            while (true) {
+                if (forCount++ > maxFor) {
+                    break;
+                }
+                Pageable pageable = PageRequest.of(page, queryTagPageSize);
+                List<Long> tagIdList = new ArrayList<>(tagIdSet);
+                // 根据id排序
+                Page<WxVideoTagRel> videoTagRelPage = wxVideoTagRelRepository
+                        .findAllByTagIdInOrderByCreateTimeAsc(tagIdList, pageable);
+                if (CollectionUtils.isEmpty(videoTagRelPage.getContent())) {
+                    break;
+                }
+                totalList.addAll(videoTagRelPage.getContent());
+            }
+            if (CollectionUtils.isEmpty(totalList)) {
+                return;
             }
+            totalList.stream().collect(Collectors.groupingBy(WxVideoTagRel::getTagId))
+                    .forEach((tagId, wxVideoTagRelList) -> {
+                        Set<Long> videoIdSet = wxVideoTagRelList.stream().map(WxVideoTagRel::getVideoId).collect(Collectors.toSet());
+                        tmpMap.put(tagId, videoIdSet);
+                    });
+
         }
         videoTagCache = tmpMap;
 
@@ -200,12 +252,9 @@ public class BlacklistContainer {
         }
 
         return videoIds.stream().filter(videoId -> {
-            if (videoTagAnyMatch(videoId, tagIdSet)) {
-                // LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}, mid={}, usedScene={}, appType={}",
-                //         uid, userType, videoId, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
-                return false;
-            }
-            return true;
+            // LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}, mid={}, usedScene={}, appType={}",
+            //         uid, userType, videoId, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
+            return !videoTagAnyMatch(videoId, tagIdSet);
         }).collect(Collectors.toList());
     }