homepage recommend

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -294,7 +294,7 @@ public class RecommendService {
         log.info("recallResult {}", recallResult);
 
         RankResult rankResult = rankService.rank(convertToRankParam(param, recallResult));
-        log.info("recallResult {}", rankResult);
+        log.info("rankResult {}", rankResult);
 
         if (rankResult == null || CollectionUtils.isEmpty(rankResult.getVideos())) {
             return Collections.emptyList();
@@ -311,6 +311,7 @@ public class RecommendService {
         // TODO 业务上有什么用处
         // 需要被标记的视频满足什么条件
         markFlowPoolVideo(param, videos);
+        log.info("markFlowPoolVideo videos {}", videos);
 
         return videos;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithLevelFilterService.java

@@ -1,6 +1,8 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolService;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -16,6 +18,7 @@ import java.util.stream.Collectors;
  * @author dyp
  */
 @Service
+@Slf4j
 public class FlowPoolWithLevelFilterService extends AbstractFilterService {
     @Autowired
     private FlowPoolService flowPoolService;
@@ -24,6 +27,7 @@ public class FlowPoolWithLevelFilterService extends AbstractFilterService {
         List<Long> videoIds = viewFilter(param);
 
         videoIds = filterDistributeCountAndDel(param, videoIds);
+        log.info("filterDistributeCountAndDel videoIds={}", JSONUtils.toJson(videoIds));
 
         return new FilterResult(videoIds);
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithLevelScoreFilterService.java

@@ -1,6 +1,8 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolService;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -16,6 +18,7 @@ import java.util.stream.Collectors;
  * @author dyp
  */
 @Service
+@Slf4j
 public class FlowPoolWithLevelScoreFilterService extends AbstractFilterService {
     @Autowired
     private FlowPoolService flowPoolService;
@@ -24,7 +27,7 @@ public class FlowPoolWithLevelScoreFilterService extends AbstractFilterService {
         List<Long> videoIds = viewFilter(param);
 
         videoIds = filterDistributeCountAndDel(param, videoIds);
-
+        log.info("filterDistributeCountAndDel videoIds={}", JSONUtils.toJson(videoIds));
         return new FilterResult(videoIds);
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithScoreFilterService.java

@@ -1,6 +1,8 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolService;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -16,6 +18,7 @@ import java.util.stream.Collectors;
  * @author dyp
  */
 @Service
+@Slf4j
 public class FlowPoolWithScoreFilterService extends AbstractFilterService {
     @Autowired
     private FlowPoolService flowPoolService;
@@ -24,7 +27,7 @@ public class FlowPoolWithScoreFilterService extends AbstractFilterService {
         List<Long> videoIds = viewFilter(param);
 
         videoIds = filterDistributeCountAndDel(param, videoIds);
-
+        log.info("filterDistributeCountAndDel videoIds={}", JSONUtils.toJson(videoIds));
         return new FilterResult(videoIds);
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/flowpool/FlowPoolService.java

@@ -10,10 +10,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Service;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 
@@ -194,8 +191,9 @@ public class FlowPoolService {
 
     private Map<Long, Integer> getDistributeCount(Map<Long, String> videoFlowPoolMap) {
         // 为了保证有序
-        List<Map.Entry<Long, String>> entries =
-                videoFlowPoolMap.entrySet().stream().sorted().collect(Collectors.toList());
+        List<Map.Entry<Long, String>> entries = videoFlowPoolMap.entrySet().stream()
+                .sorted(Comparator.comparingLong(e -> e.getKey()))
+                .collect(Collectors.toList());
 
         List<String> keys = entries.stream()
                 .map(v -> String.format(localDistributeCountFormat, v.getKey(), v.getValue()))

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -5,6 +5,8 @@ import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.commons.lang3.math.NumberUtils;
@@ -19,6 +21,7 @@ import java.util.stream.Collectors;
  * @author dyp
  */
 @Service
+@Slf4j
 public class RankService {
     @Autowired
     private RedisTemplate<String, String> redisTemplate;
@@ -32,10 +35,16 @@ public class RankService {
         }
 
         List<Video> rovRecallRank = mergeAndRankRovRecall(param);
+        log.info("mergeAndRankRovRecall rovRecallRank={}", JSONUtils.toJson(rovRecallRank));
         List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
+        log.info("mergeAndRankFlowPoolRecall flowPoolRank={}", JSONUtils.toJson(flowPoolRank));
 
         removeDuplicate(param, rovRecallRank, flowPoolRank);
 
+        log.info("removeDuplicate rovRecallRank={}, flowPoolRank={}",
+                JSONUtils.toJson(rovRecallRank),
+                JSONUtils.toJson(flowPoolRank));
+
         // 融合排序
         return mergeAndSort(param, rovRecallRank, flowPoolRank);
     }
@@ -188,10 +197,11 @@ public class RankService {
 
         double flowPoolP = getFlowPoolP(param);
         int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK() - 1;
+        int rovPoolIndex = param.getTopK();
 
         for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
             double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
             if (rand < flowPoolP) {
                 if (flowPoolIndex < flowPoolRank.size()) {
                     result.add(flowPoolRank.get(flowPoolIndex++));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -49,7 +49,7 @@ public class RecallService implements ApplicationContextAware {
         log.info("strategies {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies, o -> o.pushFrom())));
         CountDownLatch cdl = new CountDownLatch(strategies.size());
         List<Future<RecallResult.RecallData>> recallResultFutures = new ArrayList<>();
-        for (RecallStrategy strategy : strategies) {
+        for (final RecallStrategy strategy : strategies) {
             Future<RecallResult.RecallData> future = pool.submit(() -> {
                 List<Video> result = strategy.recall(param);
                 cdl.countDown();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractFlowPoolWithLevelRecallStrategy.java

@@ -6,6 +6,8 @@ import com.tzld.piaoquan.recommend.server.service.filter.FlowPoolWithLevelFilter
 import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.commons.lang3.math.NumberUtils;
@@ -21,6 +23,7 @@ import java.util.Map;
 /**
  * @author dyp
  */
+@Slf4j
 public abstract class AbstractFlowPoolWithLevelRecallStrategy implements RecallStrategy {
     @Autowired
     protected RedisTemplate<String, String> redisTemplate;
@@ -33,10 +36,11 @@ public abstract class AbstractFlowPoolWithLevelRecallStrategy implements RecallS
         Pair<String, String> flowPoolKeyAndLevel = flowPoolKeyAndLevel(param);
         String flowPoolKey = flowPoolKeyAndLevel.getLeft();
         String level = flowPoolKeyAndLevel.getRight();
-
         int getSize = param.getSize() * 5;
         List<Video> results = new ArrayList<>();
+        log.info("flowPoolKeyAndLevel={}", JSONUtils.toJson(flowPoolKeyAndLevel));
         List<String> data = redisTemplate.opsForSet().randomMembers(flowPoolKey, getSize);
+        log.info("randomMembers flowPoolKey={}, data={}", flowPoolKey, JSONUtils.toJson(data));
         if (CollectionUtils.isEmpty(data)) {
             return null;
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractFlowPoolWithLevelScoreRecallStrategy.java

@@ -6,6 +6,8 @@ import com.tzld.piaoquan.recommend.server.service.filter.FlowPoolWithLevelScoreF
 import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.math.NumberUtils;
 import org.apache.commons.lang3.tuple.Pair;
@@ -18,6 +20,7 @@ import java.util.*;
 /**
  * @author dyp
  */
+@Slf4j
 public abstract class AbstractFlowPoolWithLevelScoreRecallStrategy implements RecallStrategy {
     @Autowired
     protected RedisTemplate<String, String> redisTemplate;
@@ -33,7 +36,7 @@ public abstract class AbstractFlowPoolWithLevelScoreRecallStrategy implements Re
 
         int getSize = param.getSize() * 5;
         List<Video> results = new ArrayList<>();
-
+        log.info("flowPoolKeyAndLevel={}", JSONUtils.toJson(flowPoolKeyAndLevel));
         Set<ZSetOperations.TypedTuple<String>> data = redisTemplate.opsForZSet().reverseRangeWithScores(flowPoolKey, 0,
                 getSize - 1);
         if (CollectionUtils.isEmpty(data)) {
@@ -47,7 +50,9 @@ public abstract class AbstractFlowPoolWithLevelScoreRecallStrategy implements Re
             videoFlowPoolMap.put(NumberUtils.toLong(values[0], 0), values[1]);
             videoScoreMap.put(NumberUtils.toLong(values[0], 0), t.getScore());
         }
-
+        log.info("reverseRangeWithScores videoScoreMap={}, videoFlowPoolMap={}",
+                JSONUtils.toJson(videoScoreMap),
+                JSONUtils.toJson(videoFlowPoolMap));
         FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, videoFlowPoolMap));
 
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractFlowPoolWithScoreRecallStrategy.java

@@ -6,6 +6,8 @@ import com.tzld.piaoquan.recommend.server.service.filter.FlowPoolWithScoreFilter
 import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,6 +19,7 @@ import java.util.*;
 /**
  * @author dyp
  */
+@Slf4j
 public abstract class AbstractFlowPoolWithScoreRecallStrategy implements RecallStrategy {
     @Autowired
     protected RedisTemplate<String, String> redisTemplate;
@@ -30,7 +33,7 @@ public abstract class AbstractFlowPoolWithScoreRecallStrategy implements RecallS
 
         int getSize = param.getSize() * 5;
         List<Video> results = new ArrayList<>();
-
+        log.info("flowPoolKeyAndLevel={}", JSONUtils.toJson(flowPoolKey));
         Set<ZSetOperations.TypedTuple<String>> data = redisTemplate.opsForZSet().reverseRangeWithScores(flowPoolKey, 0,
                 getSize - 1);
         if (CollectionUtils.isEmpty(data)) {
@@ -44,7 +47,9 @@ public abstract class AbstractFlowPoolWithScoreRecallStrategy implements RecallS
             videoFlowPoolMap.put(NumberUtils.toLong(values[0], 0), values[1]);
             videoScoreMap.put(NumberUtils.toLong(values[0], 0), t.getScore());
         }
-
+        log.info("reverseRangeWithScores videoScoreMap={}, videoFlowPoolMap={}",
+                JSONUtils.toJson(videoScoreMap),
+                JSONUtils.toJson(videoFlowPoolMap));
         FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, videoFlowPoolMap));
 
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractVideoRecallStrategy.java

@@ -9,6 +9,7 @@ import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.math.NumberUtils;
@@ -20,6 +21,7 @@ import java.util.*;
 /**
  * @author dyp
  */
+@Slf4j
 public abstract class AbstractVideoRecallStrategy implements RecallStrategy {
     @Autowired
     protected RedisTemplate<String, String> redisTemplate;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithLevelRecallStrategy.java

@@ -4,6 +4,7 @@ import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConfigService
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.commons.lang3.tuple.Pair;
@@ -20,6 +21,7 @@ import static com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConsta
  * @author dyp
  */
 @Service
+@Slf4j
 public class FlowPoolWithLevelRecallStrategy extends AbstractFlowPoolWithLevelRecallStrategy {
 
     @Autowired
@@ -69,7 +71,7 @@ public class FlowPoolWithLevelRecallStrategy extends AbstractFlowPoolWithLevelRe
 
         // 4. 随机生成[0,1)之间数，返回相应概率区间的key
 
-        int random_p = RandomUtils.nextInt(0, 1);
+        double random_p = RandomUtils.nextDouble(0, 1);
         for (Map.Entry<String, LevelP> entry : level_p_mapping.entrySet()) {
             if (random_p >= entry.getValue().getMin()
                     && random_p >= entry.getValue().getMax()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithLevelScoreRecallStrategy.java

@@ -4,6 +4,7 @@ import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConfigService
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.commons.lang3.tuple.Pair;
@@ -20,6 +21,7 @@ import static com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConsta
  * @author dyp
  */
 @Service
+@Slf4j
 public class FlowPoolWithLevelScoreRecallStrategy extends AbstractFlowPoolWithLevelScoreRecallStrategy {
 
     @Autowired
@@ -68,7 +70,7 @@ public class FlowPoolWithLevelScoreRecallStrategy extends AbstractFlowPoolWithLe
 
         // 4. 随机生成[0,1)之间数，返回相应概率区间的key
 
-        int random_p = RandomUtils.nextInt(0, 1);
+        double random_p = RandomUtils.nextDouble(0, 1);
         for (Map.Entry<String, LevelP> entry : level_p_mapping.entrySet()) {
             if (random_p >= entry.getValue().getMin()
                     && random_p >= entry.getValue().getMax()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithScoreRecallStrategy.java

@@ -10,6 +10,7 @@ import static com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConsta
  * @author dyp
  */
 @Service
+
 public class FlowPoolWithScoreRecallStrategy extends AbstractFlowPoolWithScoreRecallStrategy {
 
     @Override

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/QuickFlowPoolWithLevelScoreRecallStrategy.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.recommend.server.service.recall.strategy;
 
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.stereotype.Service;
 
@@ -11,6 +12,7 @@ import static com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConsta
  * @author dyp
  */
 @Service
+@Slf4j
 public class QuickFlowPoolWithLevelScoreRecallStrategy extends AbstractFlowPoolWithLevelScoreRecallStrategy {
 
     @Override