召回改造

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV1.java

@@ -14,7 +14,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
-
+import java.util.stream.*;
 import java.util.*;
 
 import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
@@ -51,26 +51,34 @@ public class RegionRealtimeRecallStrategyV1 implements RecallStrategy {
             videoMap.put(v.getLeft(), v.getRight());
         }
 
-        FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
-        filterParam.setForceTruncation(10000);
+
         long t1 = new Long(System.currentTimeMillis());
-        FilterResult filterResult = filterService.filter(filterParam);
+        int chunkSize = 20;
+        List<List<Long>> groupedKeys = groupKeys(videoMap, chunkSize);
+        List<Long> videoids = new ArrayList<>();
+        for (List<Long> tmp : groupedKeys){
+            FilterParam filterParam = FilterParamFactory.create(param, tmp);
+            filterParam.setForceTruncation(10000);
+            FilterResult filterResult = filterService.filter(filterParam);
+            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())){
+                videoids.addAll(filterResult.getVideoIds());
+            }
+        }
         long t2 = new Long(System.currentTimeMillis());
         JSONObject obj = new JSONObject();
         obj.put("name", "RegionRealtimeRecallStrategyV1");
         obj.put("filter_time", t2-t1);
         log.info(obj.toString());
         List<Video> videosResult = new ArrayList<>();
-        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
-            filterResult.getVideoIds().stream().forEach(vid -> {
-                Video video = new Video();
-                video.setVideoId(vid);
-                video.setAbCode(param.getAbCode());
-                video.setRovScore(videoMap.get(vid));
-                video.setPushFrom(pushFrom());
-                videosResult.add(video);
-            });
-        }
+        videoids.stream().forEach(vid -> {
+            Video video = new Video();
+            video.setVideoId(vid);
+            video.setAbCode(param.getAbCode());
+            video.setRovScore(videoMap.get(vid));
+            video.setPushFrom(pushFrom());
+            videosResult.add(video);
+        });
+
         Collections.sort(videosResult, Comparator.comparingDouble(o -> -o.getRovScore()));
         return videosResult;
     }
@@ -80,4 +88,19 @@ public class RegionRealtimeRecallStrategyV1 implements RecallStrategy {
         return PUSH_FORM;
     }
 
+
+    public static List<List<Long>> groupKeys(Map<Long, Double> videoMap, int groupSize) {
+        List<List<Long>> result = new ArrayList<>();
+        List<Long> keys = new ArrayList<>(videoMap.keySet());
+
+        int size = keys.size();
+        for (int i = 0; i < size; i += groupSize) {
+            int endIndex = Math.min(i + groupSize, size);
+            List<Long> group = keys.subList(i, endIndex);
+            result.add(group);
+        }
+
+        return result;
+    }
+
 }