Merge branch 'feature_log' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -117,7 +117,7 @@ public class RecommendService {
                 stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
         stopwatch.reset().start();
         updateCache(request, param, videos);
-        log.info("updateCache={}, updateCache cost={}", JSONUtils.toJson(videos), stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+//        log.info("updateCache={}, updateCache cost={}", JSONUtils.toJson(videos), stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
 
         // 更新position
         List<VideoProto> vps = new ArrayList<>();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/ViewedService.java

@@ -95,7 +95,7 @@ public class ViewedService {
         } catch (Exception e) {
             log.error("invoke http filterViewedVideo error", e);
         } finally {
-            log.info("filterViewedVideo cost={}", stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+            // log.info("filterViewedVideo cost={}", stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
             try {
                 if (chr != null) {
                     chr.close();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -31,16 +31,16 @@ public abstract class AbstractFilterService implements FilterService {
 
 
     protected List<Long> viewFilter(FilterParam param) {
-        log.info("filterParam={}", JSONUtils.toJson(param));
+        // log.info("filterParam={}", JSONUtils.toJson(param));
         // 风险过滤
         List<Long> videoIds = filterWithRiskVideo(param.getRiskFilterFlag(),
                 param.getAppType(), param.getRegionCode(), param.getAppRegionFiltered(), param.getVideosWithRisk(),
                 param.getVideoIds());
-        log.info("filterByRiskVideos videoIds={}", JSONUtils.toJson(videoIds));
+        // log.info("filterByRiskVideos videoIds={}", JSONUtils.toJson(videoIds));
         videoIds = filterByPreViewed(param.getAppType(), param.getMid(), videoIds);
-        log.info("filterByPreViewed videoIds={}", JSONUtils.toJson(videoIds));
+        // log.info("filterByPreViewed videoIds={}", JSONUtils.toJson(videoIds));
         videoIds = filterByViewed(param.getAppType(), param.getMid(), param.getUid(), videoIds);
-        log.info("filterByViewed videoIds={}", JSONUtils.toJson(videoIds));
+        // log.info("filterByViewed videoIds={}", JSONUtils.toJson(videoIds));
         return videoIds;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithLevelFilterService.java

@@ -27,7 +27,7 @@ public class FlowPoolWithLevelFilterService extends AbstractFilterService {
         List<Long> videoIds = viewFilter(param);
 
         videoIds = filterDistributeCountAndDel(param, videoIds);
-        log.info("filterDistributeCountAndDel videoIds={}", JSONUtils.toJson(videoIds));
+        // log.info("filterDistributeCountAndDel videoIds={}", JSONUtils.toJson(videoIds));
 
         return new FilterResult(videoIds);
     }
@@ -45,7 +45,7 @@ public class FlowPoolWithLevelFilterService extends AbstractFilterService {
                         v -> param.getFlowPoolMap().get(v)));
 
         Map<Long, Integer> distributeCountMap = flowPoolService.getDistributeCountWithLevel(flowPoolMap);
-        log.info("getDistributeCountWithLevel distributeCountMap={}", JSONUtils.toJson(distributeCountMap));
+        // log.info("getDistributeCountWithLevel distributeCountMap={}", JSONUtils.toJson(distributeCountMap));
 
         List<Long> remainVideoIds = new ArrayList<>();
         for (Long videoId : videoIds) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithLevelScoreFilterService.java

@@ -27,7 +27,7 @@ public class FlowPoolWithLevelScoreFilterService extends AbstractFilterService {
         List<Long> videoIds = viewFilter(param);
 
         videoIds = filterDistributeCountAndDel(param, videoIds);
-        log.info("filterDistributeCountAndDel videoIds={}", JSONUtils.toJson(videoIds));
+        // log.info("filterDistributeCountAndDel videoIds={}", JSONUtils.toJson(videoIds));
         return new FilterResult(videoIds);
     }
 
@@ -44,7 +44,7 @@ public class FlowPoolWithLevelScoreFilterService extends AbstractFilterService {
                         v -> param.getFlowPoolMap().get(v)));
 
         Map<Long, Integer> distributeCountMap = flowPoolService.getDistributeCountWithLevelScore(flowPoolMap);
-        log.info("getDistributeCountWithLevelScore distributeCountMap={}", JSONUtils.toJson(distributeCountMap));
+        // log.info("getDistributeCountWithLevelScore distributeCountMap={}", JSONUtils.toJson(distributeCountMap));
         List<Long> remainVideoIds = new ArrayList<>();
         for (Long videoId : videoIds) {
             if (distributeCountMap.get(videoId) != null

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithScoreFilterService.java

@@ -27,7 +27,7 @@ public class FlowPoolWithScoreFilterService extends AbstractFilterService {
         List<Long> videoIds = viewFilter(param);
 
         videoIds = filterDistributeCountAndDel(param, videoIds);
-        log.info("filterDistributeCountAndDel videoIds={}", JSONUtils.toJson(videoIds));
+        // log.info("filterDistributeCountAndDel videoIds={}", JSONUtils.toJson(videoIds));
         return new FilterResult(videoIds);
     }
 
@@ -44,7 +44,7 @@ public class FlowPoolWithScoreFilterService extends AbstractFilterService {
                         v -> param.getFlowPoolMap().get(v)));
 
         Map<Long, Integer> distributeCountMap = flowPoolService.getDistributeCountWithScore(flowPoolMap);
-        log.info("getDistributeCountWithScore distributeCountMap={}", JSONUtils.toJson(distributeCountMap));
+        // log.info("getDistributeCountWithScore distributeCountMap={}", JSONUtils.toJson(distributeCountMap));
         List<Long> remainVideoIds = new ArrayList<>();
         for (Long videoId : videoIds) {
             if (distributeCountMap.get(videoId) != null

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -70,18 +70,19 @@ public class RankService {
         }
 
         List<Video> rovRecallRank = mergeAndRankRovRecall(param);
-        log.info("mergeAndRankRovRecall rovRecallRank={}", JSONUtils.toJson(rovRecallRank));
+        // log.info("mergeAndRankRovRecall rovRecallRank={}", JSONUtils.toJson(rovRecallRank));
         List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
-        log.info("mergeAndRankFlowPoolRecall flowPoolRank={}", JSONUtils.toJson(flowPoolRank));
+        // log.info("mergeAndRankFlowPoolRecall flowPoolRank={}", JSONUtils.toJson(flowPoolRank));
 
         removeDuplicate(param, rovRecallRank, flowPoolRank);
 
-        log.info("removeDuplicate rovRecallRank={}, flowPoolRank={}",
-                JSONUtils.toJson(rovRecallRank),
-                JSONUtils.toJson(flowPoolRank));
+//        log.info("removeDuplicate rovRecallRank={}, flowPoolRank={}",
+//                JSONUtils.toJson(rovRecallRank),
+//                JSONUtils.toJson(flowPoolRank));
 
         // 融合排序
-        return mergeAndSort(param, rovRecallRank, flowPoolRank);    }
+        return mergeAndSort(param, rovRecallRank, flowPoolRank);
+    }
 
     public List<Video> mergeAndRankRovRecall(RankParam param) {
         // TODO ab test
@@ -135,8 +136,8 @@ public class RankService {
                     .map(t -> param.getRankKeyPrefix() + t.getVideoId())
                     .collect(Collectors.toList());
             List<String> videoScores = redisTemplate.opsForValue().multiGet(videoIdKeys);
-            log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-                    JSONUtils.toJson(videoScores));
+//            log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                    JSONUtils.toJson(videoScores));
             if (CollectionUtils.isNotEmpty(videoScores)
                     && videoScores.size() == rovRecallRank.size()) {
                 for (int i = 0; i < videoScores.size(); i++) {
@@ -144,11 +145,11 @@ public class RankService {
                 }
                 Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
             }
-        }else if(param.getAbCode().equals("60098")) {
+        } else if (param.getAbCode().equals("60098")) {
 
             int sizeNew = param.getSize();
             removeDuplicate(rovRecallRank);
-            rovRecallRank = rovRecallRank.size() <= sizeNew ? rovRecallRank: rovRecallRank.subList(0, sizeNew);
+            rovRecallRank = rovRecallRank.size() <= sizeNew ? rovRecallRank : rovRecallRank.subList(0, sizeNew);
 
             // merge sim recall 和 return recall
             rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
@@ -160,8 +161,8 @@ public class RankService {
                     .map(t -> param.getRankKeyPrefix() + t.getVideoId())
                     .collect(Collectors.toList());
             List<String> videoScores = redisTemplate.opsForValue().multiGet(videoIdKeys);
-            log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-                    JSONUtils.toJson(videoScores));
+//            log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                    JSONUtils.toJson(videoScores));
             if (CollectionUtils.isNotEmpty(videoScores)
                     && videoScores.size() == rovRecallRank.size()) {
                 for (int i = 0; i < videoScores.size(); i++) {
@@ -194,14 +195,14 @@ public class RankService {
 
     private List<Video> mergeAndRankRovRecallNew(RankParam param) {
         UserFeature userFeature = featureRemoteService.getUserFeature(param.getMid());
-        log.info("userFeature = {}", JSONUtils.toJson(userFeature));
+//        log.info("userFeature = {}", JSONUtils.toJson(userFeature));
 
         List<Video> recallVideos = param.getRecallResult().mergeRecallVideos();
         List<RankItem> rankItems = CommonCollectionUtils.toList(recallVideos, RankItem::new);
 
         List<Long> videoIds = CommonCollectionUtils.toListDistinct(recallVideos, Video::getVideoId);
         Map<Long, ItemFeature> videoFeatureMap = featureRemoteService.getVideoFeatureMap(videoIds);
-        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatureMap));
+//        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatureMap));
 
         for (RankItem rankItem : rankItems) {
             rankItem.setItemFeature(videoFeatureMap.get(rankItem.getVideoId()));
@@ -209,10 +210,10 @@ public class RankService {
 
         // TODO
         ScoreParam scoreParam = convert(param);
-        log.info("scoreParam {}", JSONUtils.toJson(scoreParam));
+        // log.info("scoreParam {}", JSONUtils.toJson(scoreParam));
         List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
                 .scoring(scoreParam, userFeature, rankItems);
-        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        // log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
 
         return CommonCollectionUtils.toList(rovRecallScore, i -> {
             // hard code 将排序分数 赋值给video的sortScore
@@ -390,5 +391,4 @@ public class RankService {
     }
 
 
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -46,7 +46,7 @@ public class RecallService implements ApplicationContextAware {
 
     public RecallResult recall(RecallParam param) {
         List<RecallStrategy> strategies = getRecallStrategy(param);
-        log.info("strategies {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies, o -> o.pushFrom())));
+        // log.info("strategies {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies, o -> o.pushFrom())));
         CountDownLatch cdl = new CountDownLatch(strategies.size());
         List<Future<RecallResult.RecallData>> recallResultFutures = new ArrayList<>();
         for (final RecallStrategy strategy : strategies) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractFlowPoolWithLevelRecallStrategy.java

@@ -38,9 +38,9 @@ public abstract class AbstractFlowPoolWithLevelRecallStrategy implements RecallS
         String level = flowPoolKeyAndLevel.getRight();
         int getSize = param.getSize() * 5;
         List<Video> results = new ArrayList<>();
-        log.info("flowPoolKeyAndLevel={}", JSONUtils.toJson(flowPoolKeyAndLevel));
+        // log.info("flowPoolKeyAndLevel={}", JSONUtils.toJson(flowPoolKeyAndLevel));
         List<String> data = redisTemplate.opsForSet().randomMembers(flowPoolKey, getSize);
-        log.info("randomMembers flowPoolKey={}, data={}", flowPoolKey, JSONUtils.toJson(data));
+        // log.info("randomMembers flowPoolKey={}, data={}", flowPoolKey, JSONUtils.toJson(data));
         if (CollectionUtils.isEmpty(data)) {
             return null;
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractFlowPoolWithLevelScoreRecallStrategy.java

@@ -36,7 +36,7 @@ public abstract class AbstractFlowPoolWithLevelScoreRecallStrategy implements Re
 
         int getSize = param.getSize() * 5;
         List<Video> results = new ArrayList<>();
-        log.info("flowPoolKeyAndLevel={}", JSONUtils.toJson(flowPoolKeyAndLevel));
+        // log.info("flowPoolKeyAndLevel={}", JSONUtils.toJson(flowPoolKeyAndLevel));
         Set<ZSetOperations.TypedTuple<String>> data = redisTemplate.opsForZSet().reverseRangeWithScores(flowPoolKey, 0,
                 getSize - 1);
         if (CollectionUtils.isEmpty(data)) {
@@ -50,9 +50,9 @@ public abstract class AbstractFlowPoolWithLevelScoreRecallStrategy implements Re
             videoFlowPoolMap.put(NumberUtils.toLong(values[0], 0), values[1]);
             videoScoreMap.put(NumberUtils.toLong(values[0], 0), t.getScore());
         }
-        log.info("abstractFlowPoolWithLevelScoreRecallStrategy videoScoreMap={}, videoFlowPoolMap={}",
-                JSONUtils.toJson(videoScoreMap),
-                JSONUtils.toJson(videoFlowPoolMap));
+//        log.info("abstractFlowPoolWithLevelScoreRecallStrategy videoScoreMap={}, videoFlowPoolMap={}",
+//                JSONUtils.toJson(videoScoreMap),
+//                JSONUtils.toJson(videoFlowPoolMap));
         FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, videoFlowPoolMap));
 
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractFlowPoolWithScoreRecallStrategy.java

@@ -33,7 +33,7 @@ public abstract class AbstractFlowPoolWithScoreRecallStrategy implements RecallS
 
         int getSize = param.getSize() * 5;
         List<Video> results = new ArrayList<>();
-        log.info("flowPoolKey={}", JSONUtils.toJson(flowPoolKey));
+        // log.info("flowPoolKey={}", JSONUtils.toJson(flowPoolKey));
         Set<ZSetOperations.TypedTuple<String>> data = redisTemplate.opsForZSet().reverseRangeWithScores(flowPoolKey, 0,
                 getSize - 1);
         if (CollectionUtils.isEmpty(data)) {
@@ -47,9 +47,9 @@ public abstract class AbstractFlowPoolWithScoreRecallStrategy implements RecallS
             videoFlowPoolMap.put(NumberUtils.toLong(values[0], 0), values[1]);
             videoScoreMap.put(NumberUtils.toLong(values[0], 0), t.getScore());
         }
-        log.info("AbstractFlowPoolWithScoreRecallStrategy videoScoreMap={}, videoFlowPoolMap={}",
-                JSONUtils.toJson(videoScoreMap),
-                JSONUtils.toJson(videoFlowPoolMap));
+//        log.info("AbstractFlowPoolWithScoreRecallStrategy videoScoreMap={}, videoFlowPoolMap={}",
+//                JSONUtils.toJson(videoScoreMap),
+//                JSONUtils.toJson(videoFlowPoolMap));
         FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, videoFlowPoolMap));
 
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractRegionRecallStrategy.java

@@ -131,7 +131,7 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
             }
             filterService.forceTruncation = 20;
             FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet())));
-            log.info("recall filterResult={}", JSONUtils.toJson(filterResult));
+            // log.info("recall filterResult={}", JSONUtils.toJson(filterResult));
 
             if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
                 filterResult.getVideoIds().stream().forEach(vid -> {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractVideoRecallStrategy.java

@@ -56,7 +56,7 @@ public abstract class AbstractVideoRecallStrategy implements RecallStrategy {
         List<Video> results = new ArrayList<>();
 
         FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, Lists.newArrayList(videoScoreMap.keySet())));
-        log.info("recall filterResult={}", JSONUtils.toJson(filterResult));
+        // log.info("recall filterResult={}", JSONUtils.toJson(filterResult));
 
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
             filterResult.getVideoIds().stream().forEach(vid -> {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithLevelRecallStrategy.java

@@ -45,14 +45,14 @@ public class FlowPoolWithLevelRecallStrategy extends AbstractFlowPoolWithLevelRe
                 availableLevels.add(lw);
             }
         }
-        log.info("flowPoolKeyAndLevel availableLevels={}", JSONUtils.toJson(availableLevels));
+        // log.info("flowPoolKeyAndLevel availableLevels={}", JSONUtils.toJson(availableLevels));
         if (CollectionUtils.isEmpty(availableLevels)) {
             return Pair.of("", "");
         }
 
         // 3. 根据可分发层级权重设置分发概率
         Collections.sort(availableLevels, Comparator.comparingDouble(LevelWeight::getWeight));
-        log.info("flowPoolKeyAndLevel sorted availableLevels={}", JSONUtils.toJson(availableLevels));
+        // log.info("flowPoolKeyAndLevel sorted availableLevels={}", JSONUtils.toJson(availableLevels));
 
         double weightSum = availableLevels.stream().mapToDouble(o -> o.getWeight()).sum();
         BigDecimal weightSumBD = new BigDecimal(weightSum);
@@ -75,7 +75,7 @@ public class FlowPoolWithLevelRecallStrategy extends AbstractFlowPoolWithLevelRe
 
         // 4. 随机生成[0,1)之间数，返回相应概率区间的key
         double random_p = RandomUtils.nextDouble(0, 1);
-        log.info("flowPoolKeyAndLevel level_p_mapping={}, random_p={}", JSONUtils.toJson(level_p_mapping), random_p);
+        // log.info("flowPoolKeyAndLevel level_p_mapping={}, random_p={}", JSONUtils.toJson(level_p_mapping), random_p);
         for (Map.Entry<String, LevelP> entry : level_p_mapping.entrySet()) {
             if (random_p >= entry.getValue().getMin()
                     && random_p <= entry.getValue().getMax()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithLevelScoreRecallStrategy.java

@@ -44,14 +44,14 @@ public class FlowPoolWithLevelScoreRecallStrategy extends AbstractFlowPoolWithLe
                 availableLevels.add(lw);
             }
         }
-        log.info("flowPoolKeyAndLevel availableLevels={}", JSONUtils.toJson(availableLevels));
+        // log.info("flowPoolKeyAndLevel availableLevels={}", JSONUtils.toJson(availableLevels));
         if (CollectionUtils.isEmpty(availableLevels)) {
             return Pair.of("", "");
         }
 
         // 3. 根据可分发层级权重设置分发概率
         Collections.sort(availableLevels, Comparator.comparingDouble(LevelWeight::getWeight));
-        log.info("flowPoolKeyAndLevel sorted availableLevels={}", JSONUtils.toJson(availableLevels));
+        // log.info("flowPoolKeyAndLevel sorted availableLevels={}", JSONUtils.toJson(availableLevels));
 
         double weightSum = availableLevels.stream().mapToDouble(o -> o.getWeight()).sum();
         BigDecimal weightSumBD = new BigDecimal(weightSum);
@@ -75,7 +75,7 @@ public class FlowPoolWithLevelScoreRecallStrategy extends AbstractFlowPoolWithLe
         // 4. 随机生成[0,1)之间数，返回相应概率区间的key
 
         double random_p = RandomUtils.nextDouble(0, 1);
-        log.info("flowPoolKeyAndLevel level_p_mapping={}, random_p={}", JSONUtils.toJson(level_p_mapping), random_p);
+        // log.info("flowPoolKeyAndLevel level_p_mapping={}, random_p={}", JSONUtils.toJson(level_p_mapping), random_p);
         for (Map.Entry<String, LevelP> entry : level_p_mapping.entrySet()) {
             if (random_p >= entry.getValue().getMin()
                     && random_p <= entry.getValue().getMax()) {