Merge branch 'feature/zhangbo_rank' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -49,25 +49,25 @@ public abstract class AbstractFilterService {
                 param.getAppType(), param.getRegionCode(), param.getAppRegionFiltered(), param.getVideosWithRisk(),
                 param.getVideoIds(), param.getForceTruncation());
         // log.info("filterByRiskVideos videoIds={}", JSONUtils.toJson(videoIds));
-        JSONObject obj = new JSONObject();
-        obj.put("zhangbotest", "");
-        obj.put("concurrent", param.concurrent);
-        obj.put("notUsePreView", param.notUsePreView);
-        obj.put("size1", videoIds.size());
+//        JSONObject obj = new JSONObject();
+//        obj.put("zhangbotest", "");
+//        obj.put("concurrent", param.concurrent);
+//        obj.put("notUsePreView", param.notUsePreView);
+//        obj.put("size1", videoIds.size());
         if (param.isNotUsePreView()) {
             ;
         } else {
             videoIds = filterByPreViewed(param.getAppType(), param.getMid(), videoIds);
         }
-        obj.put("size2", videoIds.size());
+//        obj.put("size2", videoIds.size());
         // log.info("filterByPreViewed videoIds={}", JSONUtils.toJson(videoIds));
         if (param.isConcurrent()) {
             videoIds = filterByViewedConcurrent(param.getAppType(), param.getMid(), param.getUid(), videoIds);
         } else {
             videoIds = filterByViewed(param.getAppType(), param.getMid(), param.getUid(), videoIds);
         }
-        obj.put("size3", videoIds.size());
-        log.info(obj.toString());
+//        obj.put("size3", videoIds.size());
+//        log.info(obj.toString());
         // log.info("filterByViewed videoIds={}", JSONUtils.toJson(videoIds));
         return videoIds;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV1.java

@@ -26,10 +26,12 @@ import org.springframework.stereotype.Component;
 @Slf4j
 @Component
 public class RegionRealtimeRecallStrategyV1 implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
     @Autowired
     private RegionFilterService filterService;
     @Override
     public List<Video> recall(RecallParam param) {
+        long t0 = System.currentTimeMillis();
         // 1 获取省份key 放入参数map中
         String provinceCn = param.getProvince();
         if (provinceCn == null){
@@ -50,47 +52,48 @@ public class RegionRealtimeRecallStrategyV1 implements RecallStrategy {
         for (Pair<Long, Double> v: result){
             videoMap.put(v.getLeft(), v.getRight());
         }
-
-
-        long t1 = new Long(System.currentTimeMillis());
-//        int chunkSize = 25;
-//        List<List<Long>> groupedKeys = groupKeys(videoMap, chunkSize);
-//        List<Long> videoids = new ArrayList<>();
-//        for (List<Long> tmp : groupedKeys){
-//            FilterParam filterParam = FilterParamFactory.create(param, tmp);
-//            filterParam.setForceTruncation(10000);
-//            filterParam.setConcurrent(true);
-//            filterParam.setUsePreView(false);
-//            FilterResult filterResult = filterService.filter(filterParam);
-//            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())){
-//                videoids.addAll(filterResult.getVideoIds());
-//            }
-//        }
+        long t1 = System.currentTimeMillis();
+        // 3 召回内部过滤
         FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
         filterParam.setForceTruncation(10000);
         filterParam.setConcurrent(true);
         filterParam.setNotUsePreView(false);
         FilterResult filterResult = filterService.filter(filterParam);
-        long t2 = new Long(System.currentTimeMillis());
+        long t2 = System.currentTimeMillis();
         JSONObject obj = new JSONObject();
-        obj.put("name", "RegionRealtimeRecallStrategyV1");
+        obj.put("name", this.CLASS_NAME);
         obj.put("filter_time", t2-t1);
+        obj.put("recall_time", t1-t0);
         obj.put("provinceCn", provinceCn);
         obj.put("sizeOld", videoMap.size());
+        obj.put("mid", param.getMid());
+        obj.put("regionCode", param.getRegionCode());
+        obj.put("video_input", result.stream()
+                .map(pair -> String.valueOf(pair.getLeft()))
+                .collect(Collectors.joining(",")));
         List<Video> videosResult = new ArrayList<>();
+        List<Long> videosResultId = new ArrayList<>();
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
-            obj.put("sizeNew", filterResult.getVideoIds().size());
-            filterResult.getVideoIds().stream().forEach(vid -> {
+            filterResult.getVideoIds().forEach(vid -> {
                 Video video = new Video();
                 video.setVideoId(vid);
                 video.setAbCode(param.getAbCode());
                 video.setRovScore(videoMap.get(vid));
                 video.setPushFrom(pushFrom());
                 videosResult.add(video);
+                videosResultId.add(vid);
             });
         }
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
+//        obj.put("video_output", videosResult.stream()
+//                .map(v -> String.valueOf(v.getVideoId()))
+//                .collect(Collectors.joining(",")));
+        obj.put("video_filter", result.stream()
+                .filter(v -> !videosResultId.contains(v.getLeft()))
+                .map(v -> String.valueOf(v.getLeft()))
+                .collect(Collectors.joining(",")));
+        obj.put("sizeNew", videosResult.size());
         log.info(obj.toString());
-        Collections.sort(videosResult, Comparator.comparingDouble(o -> -o.getRovScore()));
         return videosResult;
     }
     public static final String PUSH_FORM = "recall_strategy_region_1h";

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV3.java

@@ -49,22 +49,14 @@ public class RegionRealtimeRecallStrategyV3 implements RecallStrategy {
         for (Pair<Long, Double> v: result){
             videoMap.put(v.getLeft(), v.getRight());
         }
-        long t1 = new Long(System.currentTimeMillis());
         FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
         filterParam.setForceTruncation(10000);
         filterParam.setConcurrent(true);
         filterParam.setNotUsePreView(false);
         FilterResult filterResult = filterService.filter(filterParam);
-        long t2 = new Long(System.currentTimeMillis());
-        JSONObject obj = new JSONObject();
-        obj.put("name", "RegionRealtimeRecallStrategyV3");
-        obj.put("provinceCn", provinceCn);
-        obj.put("sizeOld", videoMap.size());
-        obj.put("filter_time", t2-t1);
         List<Video> videosResult = new ArrayList<>();
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
-            obj.put("sizeNew", filterResult.getVideoIds().size());
-            filterResult.getVideoIds().stream().forEach(vid -> {
+            filterResult.getVideoIds().forEach(vid -> {
                 Video video = new Video();
                 video.setVideoId(vid);
                 video.setAbCode(param.getAbCode());
@@ -73,8 +65,7 @@ public class RegionRealtimeRecallStrategyV3 implements RecallStrategy {
                 videosResult.add(video);
             });
         }
-        log.info(obj.toString());
-        Collections.sort(videosResult, Comparator.comparingDouble(o -> -o.getRovScore()));
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
         return videosResult;
     }
     public static final String PUSH_FORM = "recall_strategy_region_24h";