召回改造

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -47,7 +47,11 @@ public abstract class AbstractFilterService {
                 param.getAppType(), param.getRegionCode(), param.getAppRegionFiltered(), param.getVideosWithRisk(),
                 param.getVideoIds(), param.getForceTruncation());
         // log.info("filterByRiskVideos videoIds={}", JSONUtils.toJson(videoIds));
-        videoIds = filterByPreViewed(param.getAppType(), param.getMid(), videoIds);
+        if (param.isNotUsePreView()){
+           ;
+        }else {
+            videoIds = filterByPreViewed(param.getAppType(), param.getMid(), videoIds);
+        }
         // log.info("filterByPreViewed videoIds={}", JSONUtils.toJson(videoIds));
         if (param.isConcurrent()) {
             videoIds = filterByViewedConcurrent(param.getAppType(), param.getMid(), param.getUid(), videoIds);
@@ -65,10 +69,21 @@ public abstract class AbstractFilterService {
             return videoIds;
         }
 
+//        int chunkSize = 20;
+//        Collection<List<Long>> chunks = videoIds.stream()
+//                .collect(Collectors.groupingBy(it -> it / chunkSize))
+//                .values();
+
         int chunkSize = 20;
-        Collection<List<Long>> chunks = videoIds.stream()
-                .collect(Collectors.groupingBy(it -> it / chunkSize))
-                .values();
+        List<List<Long>> chunks = new ArrayList<>();
+        int size = videoIds.size();
+
+        for (int i = 0; i < size; i += chunkSize) {
+            int endIndex = Math.min(i + chunkSize, size);
+            List<Long> chunk = videoIds.subList(i, endIndex);
+            chunks.add(chunk);
+        }
+
 
         CountDownLatch cdl = new CountDownLatch(chunks.size());
         List<Future<List<Long>>> futures = new ArrayList<>();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterParam.java

@@ -25,5 +25,6 @@ public class FilterParam {
     private int forceTruncation = 1000;
     private Set<String> abExpCodes;
 
-    private boolean concurrent; // hardcode 临时解决过滤慢的问题
+    private boolean concurrent=false; // hardcode 临时解决过滤慢的问题
+    private boolean notUsePreView=false;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMerge.java

@@ -52,17 +52,26 @@ public class RankStrategy4RegionMerge extends RankService {
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        rovRecallRank.addAll(v1.subList(0, Math.min(10, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(5, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
         //-------------------地域 sim returnv2 融合+去重复-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
+
+        rovRecallRank.addAll(v1);
+        rovRecallRank.addAll(v2);
+        rovRecallRank.addAll(v3);
+        rovRecallRank.addAll(v4);
         rovRecallRank.addAll(v5);
         rovRecallRank.addAll(v6);
+
+//        rovRecallRank.addAll(v1.subList(0, Math.min(10, v1.size())));
+//        rovRecallRank.addAll(v2.subList(0, Math.min(5, v2.size())));
+//        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+//        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+//        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+//        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV1.java

@@ -53,34 +53,43 @@ public class RegionRealtimeRecallStrategyV1 implements RecallStrategy {
 
 
         long t1 = new Long(System.currentTimeMillis());
-        int chunkSize = 25;
-        List<List<Long>> groupedKeys = groupKeys(videoMap, chunkSize);
-        List<Long> videoids = new ArrayList<>();
-        for (List<Long> tmp : groupedKeys){
-            FilterParam filterParam = FilterParamFactory.create(param, tmp);
-            filterParam.setForceTruncation(10000);
-            FilterResult filterResult = filterService.filter(filterParam);
-            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())){
-                videoids.addAll(filterResult.getVideoIds());
-            }
-        }
+//        int chunkSize = 25;
+//        List<List<Long>> groupedKeys = groupKeys(videoMap, chunkSize);
+//        List<Long> videoids = new ArrayList<>();
+//        for (List<Long> tmp : groupedKeys){
+//            FilterParam filterParam = FilterParamFactory.create(param, tmp);
+//            filterParam.setForceTruncation(10000);
+//            filterParam.setConcurrent(true);
+//            filterParam.setUsePreView(false);
+//            FilterResult filterResult = filterService.filter(filterParam);
+//            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())){
+//                videoids.addAll(filterResult.getVideoIds());
+//            }
+//        }
+        FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(true);
+        FilterResult filterResult = filterService.filter(filterParam);
         long t2 = new Long(System.currentTimeMillis());
         JSONObject obj = new JSONObject();
         obj.put("name", "RegionRealtimeRecallStrategyV1");
         obj.put("filter_time", t2-t1);
         obj.put("provinceCn", provinceCn);
         obj.put("sizeOld", videoMap.size());
-        obj.put("sizeNew", videoids.size());
-        log.info(obj.toString());
         List<Video> videosResult = new ArrayList<>();
-        videoids.stream().forEach(vid -> {
-            Video video = new Video();
-            video.setVideoId(vid);
-            video.setAbCode(param.getAbCode());
-            video.setRovScore(videoMap.get(vid));
-            video.setPushFrom(pushFrom());
-            videosResult.add(video);
-        });
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            obj.put("sizeNew", filterResult.getVideoIds().size());
+            filterResult.getVideoIds().stream().forEach(vid -> {
+                Video video = new Video();
+                video.setVideoId(vid);
+                video.setAbCode(param.getAbCode());
+                video.setRovScore(videoMap.get(vid));
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+            });
+        }
+        log.info(obj.toString());
         Collections.sort(videosResult, Comparator.comparingDouble(o -> -o.getRovScore()));
         return videosResult;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV2.java

@@ -44,6 +44,8 @@ public class RegionRealtimeRecallStrategyV2 implements RecallStrategy {
 
         FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
         filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(true);
         FilterResult filterResult = filterService.filter(filterParam);
         List<Video> videosResult = new ArrayList<>();
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV3.java

@@ -52,6 +52,8 @@ public class RegionRealtimeRecallStrategyV3 implements RecallStrategy {
         long t1 = new Long(System.currentTimeMillis());
         FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
         filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(true);
         FilterResult filterResult = filterService.filter(filterParam);
         long t2 = new Long(System.currentTimeMillis());
         JSONObject obj = new JSONObject();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV4.java

@@ -43,6 +43,8 @@ public class RegionRealtimeRecallStrategyV4 implements RecallStrategy {
 
         FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
         filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(true);
         FilterResult filterResult = filterService.filter(filterParam);
         List<Video> videosResult = new ArrayList<>();
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/web/GlobalExceptionHandler.java

@@ -1,6 +1,7 @@
 package com.tzld.piaoquan.recommend.server.web;
 
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.exception.ExceptionUtils;
 import org.springframework.web.bind.annotation.ControllerAdvice;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.ResponseBody;
@@ -15,7 +16,7 @@ public class GlobalExceptionHandler {
     @ResponseBody
     public String handleException(HttpServletRequest hsr, Exception e) {
         if (e instanceof RuntimeException) {
-            log.error("request uri {} error", hsr.getRequestURI(), e);
+            log.error("request uri {} error:{}", hsr.getRequestURI(), ExceptionUtils.getFullStackTrace(e));
         }
         return "";
     }