reco process

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/Application.java

@@ -24,6 +24,7 @@ import org.springframework.context.annotation.EnableAspectJAutoProxy;
 public class Application {
     public static void main(String[] args) {
         SpringApplication.run(Application.class, args);
+
     }
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -8,15 +8,17 @@ import java.util.Map;
 
 @Data
 public class RankItem implements Comparable<RankItem> {
-    public long videoid;
+    public long videoId;
     private double score; // 记录最终的score
+    private Video video;
 
     // 记录Item侧用到的特征
     private ItemFeature itemFeature;
 
     public RankItem(Video video) {
-        this.videoid = video.getVideoId() ;
-        this.score = 0.0 ;
+        this.videoId = video.getVideoId();
+        this.score = 0.0;
+        this.video = video;
     }
 
     private Map<String, Double> rankerScore;
@@ -38,6 +40,4 @@ public class RankItem implements Comparable<RankItem> {
     }
 
 
-
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/remote/FeatureRemoteService.java

@@ -7,10 +7,17 @@ import com.tzld.piaoquan.recommend.feature.model.feature.VideoFeatureProto;
 import com.tzld.piaoquan.recommend.server.common.base.ItemFeature;
 import com.tzld.piaoquan.recommend.server.common.base.UserActionFeature;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 
 /**
  * @author dyp
@@ -22,6 +29,7 @@ public class FeatureRemoteService {
     @Autowired
     private FeatureClient client;
 
+    // 有性能问题再增加localCache
     public UserFeature getUserFeature(String uid) {
         if (StringUtils.isBlank(uid)) {
             return null;
@@ -76,6 +84,15 @@ public class FeatureRemoteService {
         return convert(proto);
     }
 
+    public Map<Long, ItemFeature> getVideoFeatureMap(List<Long> videoIds) {
+        if (CollectionUtils.isEmpty(videoIds)) {
+            return Collections.emptyMap();
+        }
+        List<VideoFeatureProto> protoList = client.getAllVideoFeature(videoIds);
+
+        return CommonCollectionUtils.toMap(protoList, p -> NumberUtils.toLong(p.getVideoId(), 0L), this::convert);
+    }
+
     private ItemFeature convert(VideoFeatureProto proto) {
         if (proto == null) {
             return null;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -341,6 +341,8 @@ public class RecommendService {
         rankParam.setTopK(param.getTopK());
         rankParam.setRankKeyPrefix(param.getRankKeyPrefix());
         rankParam.setAppType(param.getAppType());
+        rankParam.setUid(param.getUid());
+        rankParam.setCityCode(param.getCityCode());
         return rankParam;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankParam.java

@@ -16,4 +16,6 @@ public class RankParam {
     private String abCode;
     private int appType;
     private boolean specialRecommend;
+    private String uid;
+    private String cityCode;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -1,17 +1,17 @@
 package com.tzld.piaoquan.recommend.server.service.rank;
 
 
-import com.typesafe.config.Config;
+import com.tzld.piaoquan.recommend.server.common.base.ItemFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.remote.FeatureRemoteService;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerPipeline;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
@@ -32,65 +32,10 @@ import java.util.stream.Collectors;
 public class RankService {
     @Autowired
     private RedisTemplate<String, String> redisTemplate;
-
-
-    // LR scores
-    public RankResult rank(RecommendRequest recommendRequest,
-                           RankParam param,
-                           RecallResult recallResult,
-                           UserFeature userFeature,
-                           List<RankItem> videoRankFeatures,
-                           Config config) {
-        if (param == null
-                || param.getRecallResult() == null
-                || CollectionUtils.isEmpty(param.getRecallResult().getData())) {
-            return null;
-        }
-
-        if (param.isSpecialRecommend()) {
-            Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
-                    .filter(d -> d.getPushFrom().equals(SpecialRecallStrategy.PUSH_FROM))
-                    .findFirst();
-            if (data.isPresent()
-                    && data.get() != null) {
-                return new RankResult(data.get().getVideos());
-            }
-            return null;
-        }
-
-        ScorerPipeline scorerPipeline = ScorerUtils.getScorerPipeline(config);
-        // TODO  merge 后返回待排序的video list
-        List<Video> recallVideos = recallResult.mergeRecallVideos();
-        // TODO 转化成rankitem
-        // convert List<Video> to List<RankItem>
-        List<RankItem> rankItem = new ArrayList<RankItem>();
-
-        List<RankItem> rovRecallRank  =
-                scorerPipeline.scoring(recommendRequest, param, userFeature, rankItem);
-
-        log.info("mergeAndRankRovRecall rovRecallRank={}", JSONUtils.toJson(rovRecallRank));
-        List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
-        log.info("mergeAndRankFlowPoolRecall flowPoolRank={}", JSONUtils.toJson(flowPoolRank));
-
-        removeDuplicate(param, rovRecallRank, flowPoolRank);
-        // convert List<RankItem> to List<Video>
-
-
-
-        log.info("removeDuplicate rovRecallRank={}, flowPoolRank={}",
-                JSONUtils.toJson(rovRecallRank),
-                JSONUtils.toJson(flowPoolRank));
-
-        // 融合排序
-        return mergeAndSort(param, rovRecallRank, flowPoolRank);
-
-
-    }
-
+    @Autowired
+    private FeatureRemoteService featureRemoteService;
 
     public RankResult rank(RankParam param) {
-
-
         if (param == null
                 || param.getRecallResult() == null
                 || CollectionUtils.isEmpty(param.getRecallResult().getData())) {
@@ -124,6 +69,17 @@ public class RankService {
     }
 
     private List<Video> mergeAndRankRovRecall(RankParam param) {
+        // TODO ab test
+        // TODO 抽象成Strategy
+        boolean hitTest = true;
+        if (hitTest) {
+            return mergeAndRankRovRecallNew(param);
+        } else {
+            return mergeAndRankRovRecallOld(param);
+        }
+    }
+
+    private List<Video> mergeAndRankRovRecallOld(RankParam param) {
         List<Video> rovRecallRank = new ArrayList<>();
         rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
         rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
@@ -181,6 +137,29 @@ public class RankService {
         return rovRecallRank;
     }
 
+    private List<Video> mergeAndRankRovRecallNew(RankParam param) {
+        UserFeature userFeature = featureRemoteService.getUserFeature(param.getUid());
+        List<Video> recallVideos = param.getRecallResult().mergeRecallVideos();
+        List<RankItem> rankItems = CommonCollectionUtils.toList(recallVideos, RankItem::new);
+
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(recallVideos, Video::getVideoId);
+        Map<Long, ItemFeature> videoFeatureMap = featureRemoteService.getVideoFeatureMap(videoIds);
+        for (RankItem rankItem : rankItems) {
+            rankItem.setItemFeature(videoFeatureMap.get(rankItem.getVideoId()));
+        }
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(param, userFeature, rankItems);
+        log.info("mergeAndRankRovRecall rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 复制给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
+
     private List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
         if (param.getAppType() == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
                 || param.getAppType() == AppTypeEnum.ZUI_JING_QI.getCode()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/AbstractScorer.java

@@ -63,9 +63,8 @@ public abstract class AbstractScorer {
         return scorerConfigInfo;
     }
 
-    public abstract List<RankItem> scoring(final RecommendRequest request,
-                                        final RankParam param,
-                                        final UserFeature userFeature,
-                                        final List<RankItem> rankItems);
+    public abstract List<RankItem> scoring(final RankParam param,
+                                           final UserFeature userFeature,
+                                           final List<RankItem> rankItems);
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerPipeline.java

@@ -2,12 +2,8 @@ package com.tzld.piaoquan.recommend.server.service.score;
 
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -15,11 +11,7 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 
 
 public class ScorerPipeline {
@@ -28,7 +20,7 @@ public class ScorerPipeline {
     public static final Logger LOGGER = LoggerFactory.getLogger(ScorerPipeline.class);
     public static final ExecutorService executorService = Executors.newFixedThreadPool(corePoolSize);
 
-    public List<AbstractScorer> scorers;
+    private List<AbstractScorer> scorers;
 
     public ScorerPipeline(List<AbstractScorer> scorers) {
         this.scorers = scorers;
@@ -36,12 +28,12 @@ public class ScorerPipeline {
 
     /**
      * scoring
+     *
      * @return
      */
-    public List<RankItem> scoring(final RecommendRequest recommendRequest,
-                               final RankParam param,
-                               final UserFeature userFeature,
-                               final List<RankItem> rankItems) {
+    public List<RankItem> scoring(final RankParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
         // check recall is empty?
         if (CollectionUtils.isEmpty(param.getRecallResult().getData())) {
             // log.error
@@ -69,7 +61,7 @@ public class ScorerPipeline {
             Callable<List<RankItem>> callable = new Callable<List<RankItem>>() {
                 @Override
                 public List<RankItem> call() throws Exception {
-                    return scorer.scoring(recommendRequest, param, userFeature, scoreRankerItems);
+                    return scorer.scoring(param, userFeature, scoreRankerItems);
                 }
             };
 
@@ -108,7 +100,7 @@ public class ScorerPipeline {
                 item.getRankerScore().put(scorerName, item.getScore());
             }
 
-           //
+            //
             long spentTime = System.currentTimeMillis() - startTime;
             LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
                     new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -8,13 +8,24 @@ import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 
-public class ScorerUtils {
+public final class ScorerUtils {
     private static Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
 
+    private static Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
+
+    public static String BASE_CONF = "feeds_score_config_baseline.conf";
+
+    private ScorerUtils() {
+        scorerPipelineCache.put(BASE_CONF, getScorerPipeline(BASE_CONF));
+    }
+
     /**
      * init load model
+     *
      * @param scorers
      */
     public static void initLoadModel(List<AbstractScorer> scorers) {
@@ -32,6 +43,19 @@ public class ScorerUtils {
         initLoadModel(scorers);
     }
 
+    public static ScorerPipeline getScorerPipeline(String configFile) {
+        // 不需要保证严格意义的单例
+        if (scorerPipelineCache.containsKey(configFile)) {
+            return scorerPipelineCache.get(configFile);
+        }
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(configFile);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        ScorerPipeline pipeline = new ScorerPipeline(scorers);
+        scorerPipelineCache.put(configFile, pipeline);
+        return pipeline;
+    }
+
     public static ScorerPipeline getScorerPipeline(Config mergeConfig) {
         ScorerConfig scorerConfig = new ScorerConfig();
         scorerConfig.load(mergeConfig);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareGBDTScorer.java

@@ -9,21 +9,12 @@ import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.score.feature.GBDTFeatureExtractorBase;
 import com.tzld.piaoquan.recommend.server.service.score.model.GBDTModel;
-import com.tzld.piaoquan.recommend.server.model.Video;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
+import java.util.*;
+import java.util.concurrent.*;
 
 public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
 
@@ -38,17 +29,16 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
     }
 
 
-    private RequestContext getRequestContext(RecommendRequest request) {
+    private RequestContext getRequestContext(RankParam param) {
 
         RequestContext requestContext = new RequestContext();
         return requestContext;
     }
 
     @Override
-    public List<RankItem> scoring(final RecommendRequest request,
-                               final RankParam param,
-                               final UserFeature userFeature,
-                               final List<RankItem> rankItems) {
+    public List<RankItem> scoring(final RankParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
         RecallResult recallResult = param.getRecallResult();
         if (recallResult.getData().size() == 0) {
             return rankItems;
@@ -61,10 +51,10 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
             LOGGER.error("not found model");
             return rankItems;
         }
-        RequestContext requestContext = getRequestContext(request);
+        RequestContext requestContext = getRequestContext(param);
 
         // 多Rank的rank打分
-        multipleGBDTScore(rankItems, model, userFeature, request, requestContext);
+        multipleGBDTScore(rankItems, model, userFeature, requestContext);
         Collections.sort(rankItems);
         LOGGER.debug("dwelltime ranker excute time: [{}]", System.currentTimeMillis() - startTime);
         return rankItems;
@@ -80,8 +70,8 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
      * @return
      */
     private double predictMultipleGBDTScore(final double oldScore, final double dwelltimeOrCompletion,
-                                         final double duration, final String itemId,
-                                         final RecommendRequest requestData, final UserFeature user) {
+                                            final double duration, final String itemId,
+                                            final RecommendRequest requestData, final UserFeature user) {
         double ctrScalePower = 0;
         double dwelltimeScalePower = 0;
 
@@ -91,7 +81,7 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
         }
         double newScore = Math.pow(ctrScore, ctrScalePower) * Math.pow(dwelltimeOrCompletion, dwelltimeScalePower);
         //值越大压制越弱
-        double alpha =  1.0;
+        double alpha = 1.0;
         double suppressRatio = (1 + alpha) / (Math.max(duration, 300.0) / 300 + alpha);
         newScore = newScore * suppressRatio;
         LOGGER.debug("Expected dwell time score [{}]: {} ,{}, {}--> {}", new Object[]{itemId, oldScore, dwelltimeOrCompletion, duration, newScore});
@@ -99,11 +89,11 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
     }
 
     private void gbdtBaseScore(final RankItem item,
-                                final GBDTModel model,
-                                final RequestContext requestContext,
-                                final Map<String, Double> userFeatures,
-                                final Map<String, Double> contextFeatures,
-                                final RecommendRequest requestData, final UserFeature user) {
+                               final GBDTModel model,
+                               final RequestContext requestContext,
+                               final Map<String, Double> userFeatures,
+                               final Map<String, Double> contextFeatures,
+                               final UserFeature user) {
         try {
 
             Map<String, Double> articleFeatures;
@@ -130,10 +120,9 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
 
 
     public void multipleGBDTScore(final List<RankItem> items,
-                                       final GBDTModel model,
-                                       final UserFeature user,
-                                       final RecommendRequest requestData,
-                                       final RequestContext requestContext) {
+                                  final GBDTModel model,
+                                  final UserFeature user,
+                                  final RequestContext requestContext) {
         final int size = items.size();
         if (size == 0) {
             return;
@@ -155,7 +144,7 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
                               @Override
                               public Object call() throws Exception {
                                   try {
-                                      gbdtBaseScore(items.get(fIndex), model, requestContext, userFeatures, contextFeatures, requestData, user);
+                                      gbdtBaseScore(items.get(fIndex), model, requestContext, userFeatures, contextFeatures, user);
                                   } catch (Exception e) {
                                       LOGGER.error("dwelltime exception: [{}] [{}]", items.get(fIndex), ExceptionUtils.getFullStackTrace(e));
                                   }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareLRScorer.java

@@ -5,13 +5,8 @@ package com.tzld.piaoquan.recommend.server.service.score;
 import com.tzld.piaoquan.recommend.server.common.base.*;
 import com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples;
 import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.score.feature.FeatureUsage;
-import com.tzld.piaoquan.recommend.server.service.score.feature.LRBytesFeatureExtractorBase;
 import com.tzld.piaoquan.recommend.server.service.score.feature.VlogShareLRFeatureExtractor;
 import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
 import org.apache.commons.lang.exception.ExceptionUtils;
@@ -43,8 +38,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
 
 
     @Override
-    public List<RankItem> scoring(final RecommendRequest request,
-                               final RankParam param,
+    public List<RankItem> scoring(final RankParam param,
                                final UserFeature userFeature,
                                final List<RankItem> rankItems) {
         long startTime = System.currentTimeMillis();
@@ -55,7 +49,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
         }
 
         List<RankItem> result = rankItems;
-        result = rankByJava(rankItems, request, userFeature);
+        result = rankByJava(rankItems, param, userFeature);
 
         LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
                 System.currentTimeMillis() - startTime);
@@ -64,16 +58,17 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
     }
 
     private List<RankItem> rankByJava(final List<RankItem> items,
-                                        final RecommendRequest request,
+                                        final RankParam param,
                                         final UserFeature user) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
 
         // init request context
-        String cityCode = request.getCityCode();
+        // TODO  应该用哪个？ city or province or region
+        String cityCode = param.getCityCode();
         RequestContext requestContext = new RequestContext();
-        requestContext.setCity(request.getCityCode());
+        requestContext.setCity(param.getCityCode());
 
         // userBytes
         UserBytesFeature userInfoBytes = null;
@@ -126,11 +121,11 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
                 LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoid(), ExceptionUtils.getFullStackTrace(e)});
+                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
             }
 
 
-            CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
+            CtrSamples.Builder samples =  CtrSamples.newBuilder();
             samples.setLr_samples(lrSamples);
             item.setSamples(samples);
         }
@@ -168,7 +163,8 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
                     try {
                         calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
                     } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoid, ExceptionUtils.getFullStackTrace(e));
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId,
+                                ExceptionUtils.getFullStackTrace(e));
                     }
                     return new Object();
                 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/CommonCollectionUtils.java

@@ -19,6 +19,13 @@ public class CommonCollectionUtils {
         return list.stream().map(map).collect(Collectors.toList());
     }
 
+    public static <T, R> List<R> toListDistinct(List<T> list, Function<T, R> map) {
+        if (CollectionUtils.isEmpty(list)) {
+            return Collections.emptyList();
+        }
+        return list.stream().map(map).distinct().collect(Collectors.toList());
+    }
+
     public static <T, K, V> Map<K, V> toMap(List<T> list, Function<T, K> keyFunc, Function<T, V> valueFunc) {
         if (CollectionUtils.isEmpty(list)) {
             return Collections.emptyMap();