batch get feature

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/service/AbstractFeatureService.java

@@ -4,15 +4,20 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.feature.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.dao.DataAccessException;
+import org.springframework.data.redis.core.RedisOperations;
 import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.core.SessionCallback;
+import org.springframework.data.redis.core.ValueOperations;
 
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 
@@ -120,42 +125,90 @@ public abstract class AbstractFeatureService<K, V> {
 
     private Map<K, V> loadAll(Iterable<? extends K> keyIte) {
 
-        Map<K,V> featureMap = getFromCache(keyIte);
-        if (feature == null) {
-            feature = getFromSource(keyIte);
-            // TODO 可异步
-            saveToCache();
+        Iterator<? extends K> ite = keyIte.iterator();
+        List<K> keys = new ArrayList<>();
+        while (ite.hasNext()) {
+            keys.add(ite.next());
         }
-        return feature;
-    }
 
-    private V getFromCache(Iterable<? extends K> keyIte) {
 
-        Iterator<? extends K> ite = keyIte.iterator();
+        Map<K, V> cacheFeatureMap = getFromCache(keys);
+        if (cacheFeatureMap.size() == keys.size()) {
+            return cacheFeatureMap;
+        }
+
+        ite = keys.iterator();
         while (ite.hasNext()) {
-            keys.add(ite.next());
+            cacheFeatureMap.containsKey(ite.next());
+            ite.remove();
         }
-        String redisKey = cacheKey(key);
-        String value = redisTemplate.opsForValue().get(redisKey);
-        return JSONUtils.fromJson(value, typeToken, null);
+
+        Map<K, V> sourceFeatureMap = null;
+        if (CollectionUtils.isNotEmpty(keys)) {
+            sourceFeatureMap = getFromSource(keys);
+            // TODO 可异步
+            saveToCache(keys, sourceFeatureMap);
+        }
+
+
+        return CommonCollectionUtils.merge(cacheFeatureMap, sourceFeatureMap);
     }
 
-    private V getFromSource(Iterable<? extends K> keyIte) {
-        String sourceKey = cacheKey(key);
-        String value = tairTemplate.opsForValue().get(sourceKey);
-        return JSONUtils.fromJson(value, typeToken, null);
+    private Map<K, V> getFromCache(List<K> keys) {
+        List<String> redisKeys = CommonCollectionUtils.toList(keys, this::cacheKey);
+        List<String> values = redisTemplate.opsForValue().multiGet(redisKeys);
+        Map<K, V> result = new HashMap<>();
+        for (int i = 0; i < keys.size(); i++) {
+            if (values.get(i) != null) {
+                result.put(keys.get(i), JSONUtils.fromJson(values.get(i), typeToken, null));
+            }
+        }
+        return result;
     }
 
-    private void saveToCache(Map<K, V> map) {
-        String cacheKey = cacheKey(key);
-        String cacheValue = value == null
-                ? emptyData
-                : JSONUtils.toJson(value);
-        long expire = value == null
-                ? emptyDataExpire
-                : defaultExpire;
+    private Map<K, V> getFromSource(List<K> keys) {
+        List<String> redisKeys = CommonCollectionUtils.toList(keys, this::cacheKey);
+        List<String> values = tairTemplate.opsForValue().multiGet(redisKeys);
+        Map<K, V> result = new HashMap<>();
+        for (int i = 0; i < keys.size(); i++) {
+            if (values.get(i) != null) {
+                result.put(keys.get(i), JSONUtils.fromJson(values.get(i), typeToken, null));
+            }
+        }
+        return result;
+    }
+
+    private void saveToCache(List<K> keys, Map<K, V> map) {
+
+        Map<String, String> emptyDataMap = new HashMap<>();
+        Map<String, String> dataMap = new HashMap<>();
+        for (K key : keys) {
+            if (map.containsKey(key)) {
+                dataMap.put(cacheKey(key), JSONUtils.toJson(map.get(key)));
+            } else {
+                emptyDataMap.put(cacheKey(key), emptyData);
+            }
+        }
+
         // TODO 评估过期时间
-        redisTemplate.opsForValue().set(cacheKey, cacheValue, expire, TimeUnit.SECONDS);
+        redisTemplate.executePipelined(new SessionCallback<String>() {
+            @Override
+            public <A, B> String execute(RedisOperations<A, B> redisOperations) throws DataAccessException {
+                ValueOperations<String, String> operations = (ValueOperations<String, String>) redisOperations.opsForValue();
+                if (MapUtils.isNotEmpty(dataMap)) {
+                    dataMap.entrySet().forEach(e ->
+                            operations.set(e.getKey(), e.getValue(), defaultExpire, TimeUnit.SECONDS)
+                    );
+                }
+                if (MapUtils.isNotEmpty(emptyDataMap)) {
+                    emptyDataMap.entrySet().forEach(e ->
+                            operations.set(e.getKey(), e.getValue(), emptyDataExpire, TimeUnit.SECONDS)
+                    );
+                }
+                return null;
+            }
+        });
+
     }
 
     private String cacheKey(K key) {
@@ -179,11 +232,12 @@ public abstract class AbstractFeatureService<K, V> {
         redisTemplate.delete(cacheKey);
     }
 
-    protected List<V> getAll(List<K> videoIds) {
+    protected Map<K, V> getAll(List<K> videoIds) {
         try {
-            cache.getAll(videoIds);
+            return cache.getAll(videoIds);
         } catch (Exception e) {
-
+            log.error("");
         }
+        return Collections.emptyMap();
     }
 }

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/service/VideoFeatureService.java

@@ -10,11 +10,13 @@ import com.tzld.piaoquan.recommend.feature.model.feature.GetVideoFeatureResponse
 import com.tzld.piaoquan.recommend.feature.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.feature.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang.StringUtils;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
 import java.util.List;
+import java.util.Map;
 
 /**
  * @author dyp
@@ -70,8 +72,8 @@ public class VideoFeatureService extends AbstractFeatureService<String, VideoFea
         }
         List<String> videoIds = request.getVideoIdList();
 
-        List<VideoFeature> videoFeatures = super.getAll(videoIds);
-        if (CollectionUtils.isEmpty(videoFeatures)) {
+        Map<String, VideoFeature> videoFeatureMap = super.getAll(videoIds);
+        if (MapUtils.isEmpty(videoFeatureMap)) {
             return GetAllVideoFeatureResponse.newBuilder()
                     .setResult(Result.newBuilder().setCode(1))
                     .build();
@@ -79,7 +81,7 @@ public class VideoFeatureService extends AbstractFeatureService<String, VideoFea
 
         return GetAllVideoFeatureResponse.newBuilder()
                 .setResult(Result.newBuilder().setCode(1))
-                .addAllVideoFeature(CommonCollectionUtils.toList(videoFeatures, FeatureConverter::convert))
+                .addAllVideoFeature(CommonCollectionUtils.toList(videoFeatureMap.values(), FeatureConverter::convert))
                 .build();
     }
 }

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/util/CommonCollectionUtils.java

@@ -1,10 +1,9 @@
 package com.tzld.piaoquan.recommend.feature.util;
 
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
 
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -12,7 +11,7 @@ import java.util.stream.Collectors;
  * @author dyp
  */
 public class CommonCollectionUtils {
-    public static <T, R> List<R> toList(List<T> list, Function<T, R> map) {
+    public static <T, R> List<R> toList(Collection<T> list, Function<T, R> map) {
         if (CollectionUtils.isEmpty(list)) {
             return Collections.emptyList();
         }
@@ -25,4 +24,18 @@ public class CommonCollectionUtils {
         }
         return list.stream().collect(Collectors.toMap(keyFunc::apply, valueFunc::apply));
     }
+
+    public static <K, V> Map<K, V> merge(Map<K, V> map1, Map<K, V> map2) {
+        if (MapUtils.isEmpty(map1)) {
+            return map2;
+        }
+        if (MapUtils.isEmpty(map2)) {
+            return map1;
+        }
+
+        Map<K, V> map = new HashMap<>();
+        map.putAll(map1);
+        map.putAll(map2);
+        return map;
+    }
 }