write to redis & repair conflicts

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -10,13 +10,15 @@ import java.util.Map;
 public class RankItem implements Comparable<RankItem> {
     public long videoid;
     private double score; // 记录最终的score
+    private Video video;
 
     // 记录Item侧用到的特征
     private ItemFeature itemFeature;
 
     public RankItem(Video video) {
         this.videoid = video.getVideoId() ;
-        this.score = 0.0 ;
+        this.score = 0.0;
+        this.video = video;
     }
 
     private Map<String, Double> rankerScore;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/remote/FeatureRemoteService.java

@@ -7,10 +7,19 @@ import com.tzld.piaoquan.recommend.feature.model.feature.VideoFeatureProto;
 import com.tzld.piaoquan.recommend.server.common.base.ItemFeature;
 import com.tzld.piaoquan.recommend.server.common.base.UserActionFeature;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+
 
 /**
  * @author dyp
@@ -22,6 +31,7 @@ public class FeatureRemoteService {
     @Autowired
     private FeatureClient client;
 
+    // 有性能问题再增加localCache
     public UserFeature getUserFeature(String uid) {
         if (StringUtils.isBlank(uid)) {
             return null;
@@ -75,6 +85,14 @@ public class FeatureRemoteService {
         return convert(proto);
     }
 
+    public Map<Long, ItemFeature> getVideoFeatureMap(List<Long> videoIds) {
+        if (CollectionUtils.isEmpty(videoIds)) {
+            return Collections.emptyMap();
+        }
+        List<VideoFeatureProto> protoList = client.getAllVideoFeature(videoIds);
+        return CommonCollectionUtils.toMap(protoList, p -> NumberUtils.toLong(p.getVideoId(), 0L), this::convert);
+    }
+
     private ItemFeature convert(VideoFeatureProto proto) {
         if (proto == null) {
             return null;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -2,7 +2,9 @@ package com.tzld.piaoquan.recommend.server.service.rank;
 
 
 import com.typesafe.config.Config;
+import com.tzld.piaoquan.recommend.server.common.base.ItemFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.base.RequestContext;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
@@ -41,6 +43,7 @@ public class RankService {
     // LR scores
     public RankResult rank(RecommendRequest recommendRequest,
                            RankParam param,
+                           RequestContext requestContext,
                            RecallResult recallResult,
                            UserFeature userFeature,
                            List<RankItem> videoRankFeatures,
@@ -70,7 +73,7 @@ public class RankService {
         List<RankItem> rankItem = new ArrayList<RankItem>();
 
         List<RankItem> rovRecallRank  =
-                scorerPipeline.scoring(recommendRequest, param, userFeature, rankItem);
+                scorerPipeline.scoring(requestContext, param, userFeature, rankItem);
 
         log.info("mergeAndRankRovRecall rovRecallRank={}", JSONUtils.toJson(rovRecallRank));
         List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
@@ -220,11 +223,15 @@ public class RankService {
         List<Long> videoIds = CommonCollectionUtils.toListDistinct(recallVideos, Video::getVideoId);
         Map<Long, ItemFeature> videoFeatureMap = featureRemoteService.getVideoFeatureMap(videoIds);
         for (RankItem rankItem : rankItems) {
-            rankItem.setItemFeature(videoFeatureMap.get(rankItem.getVideoId()));
+            rankItem.setItemFeature(videoFeatureMap.get(rankItem.getVideoid()));
         }
 
+        // TODO
+        // 初始化 RecommendRequest, RequenstContxt
+        RequestContext requestContext = new RequestContext();
+
         List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(param, userFeature, rankItems);
+                .scoring(requestContext, param, userFeature, rankItems);
         log.info("mergeAndRankRovRecall rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
 
         return CommonCollectionUtils.toList(rovRecallScore, i -> {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -107,24 +107,28 @@ public class RecallService implements ApplicationContextAware {
                 || param.getAbCode().equals("60088")
                 || param.getAbCode().equals("60089")
                 || param.getAbCode().equals("60090")
-                || param.getAbCode().equals("60091")
-                || param.getAbCode().equals("60092")
-                || param.getAbCode().equals("60093")
-                || param.getAbCode().equals("60094")) {
+                || param.getAbCode().equals("60091")) {
             strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
         } else if (param.getAbCode().equals("60068")
                 || param.getAbCode().equals("60081")
 <<<<<<< HEAD
+<<<<<<< HEAD
+=======
+>>>>>>> e9fc74f (write to redis & repair conflicts)
                 || param.getAbCode().equals("60084")
                 || param.getAbCode().equals("60092")
                 || param.getAbCode().equals("60093")
                 || param.getAbCode().equals("60094")
                 || param.getAbCode().equals("60096")
+<<<<<<< HEAD
                 || param.getAbCode().equals("60095")
                 || param.getAbCode().equals("60097")) {
 =======
                 || param.getAbCode().equals("60084")) {
 >>>>>>> 78c80c0 (offline write samples and to redis)
+=======
+                || param.getAbCode().equals("60095")) {
+>>>>>>> e9fc74f (write to redis & repair conflicts)
             strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
             strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/AbstractScorer.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.recommend.server.service.score;
 
 
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.base.RequestContext;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.model.Video;
@@ -63,7 +64,7 @@ public abstract class AbstractScorer {
         return scorerConfigInfo;
     }
 
-    public abstract List<RankItem> scoring(final RecommendRequest request,
+    public abstract List<RankItem> scoring(final RequestContext requestContext,
                                         final RankParam param,
                                         final UserFeature userFeature,
                                         final List<RankItem> rankItems);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerPipeline.java

@@ -1,6 +1,7 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.base.RequestContext;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.model.Video;
@@ -38,7 +39,7 @@ public class ScorerPipeline {
      * scoring
      * @return
      */
-    public List<RankItem> scoring(final RecommendRequest recommendRequest,
+    public List<RankItem> scoring(final RequestContext requestContext,
                                final RankParam param,
                                final UserFeature userFeature,
                                final List<RankItem> rankItems) {
@@ -69,7 +70,7 @@ public class ScorerPipeline {
             Callable<List<RankItem>> callable = new Callable<List<RankItem>>() {
                 @Override
                 public List<RankItem> call() throws Exception {
-                    return scorer.scoring(recommendRequest, param, userFeature, scoreRankerItems);
+                    return scorer.scoring(requestContext, param, userFeature, scoreRankerItems);
                 }
             };
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareGBDTScorer.java

@@ -36,7 +36,7 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
     }
 
     @Override
-    public List<RankItem> scoring(final RecommendRequest request,
+    public List<RankItem> scoring(final RequestContext requestContext,
                                final RankParam param,
                                final UserFeature userFeature,
                                final List<RankItem> rankItems) {
@@ -52,10 +52,9 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
             LOGGER.error("not found model");
             return rankItems;
         }
-        RequestContext requestContext = getRequestContext(request);
 
         // 多Rank的rank打分
-        multipleGBDTScore(rankItems, model, userFeature, request, requestContext);
+        multipleGBDTScore(rankItems, model, userFeature, requestContext);
         Collections.sort(rankItems);
         LOGGER.debug("dwelltime ranker excute time: [{}]", System.currentTimeMillis() - startTime);
         return rankItems;
@@ -94,7 +93,7 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
                                 final RequestContext requestContext,
                                 final Map<String, Double> userFeatures,
                                 final Map<String, Double> contextFeatures,
-                                final RecommendRequest requestData, final UserFeature user) {
+                                final UserFeature user) {
         try {
 
             Map<String, Double> articleFeatures;
@@ -123,7 +122,6 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
     public void multipleGBDTScore(final List<RankItem> items,
                                        final GBDTModel model,
                                        final UserFeature user,
-                                       final RecommendRequest requestData,
                                        final RequestContext requestContext) {
         final int size = items.size();
         if (size == 0) {
@@ -146,7 +144,7 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
                               @Override
                               public Object call() throws Exception {
                                   try {
-                                      gbdtBaseScore(items.get(fIndex), model, requestContext, userFeatures, contextFeatures, requestData, user);
+                                      gbdtBaseScore(items.get(fIndex), model, requestContext, userFeatures, contextFeatures, user);
                                   } catch (Exception e) {
                                       LOGGER.error("dwelltime exception: [{}] [{}]", items.get(fIndex), ExceptionUtils.getFullStackTrace(e));
                                   }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareLRScorer.java

@@ -42,7 +42,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
 
 
     @Override
-    public List<RankItem> scoring(final RecommendRequest request,
+    public List<RankItem> scoring(final RequestContext requestContext,
                                final RankParam param,
                                final UserFeature userFeature,
                                final List<RankItem> rankItems) {
@@ -54,7 +54,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
         }
 
         List<RankItem> result = rankItems;
-        result = rankByJava(rankItems, request, userFeature);
+        result = rankByJava(rankItems, requestContext, userFeature);
 
         LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
                 System.currentTimeMillis() - startTime);
@@ -63,17 +63,12 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
     }
 
     private List<RankItem> rankByJava(final List<RankItem> items,
-                                        final RecommendRequest request,
+                                        final RequestContext requestContext,
                                         final UserFeature user) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
 
-        // init request context
-        String cityCode = request.getCityCode();
-        RequestContext requestContext = new RequestContext();
-        requestContext.setCity(request.getCityCode());
-
         // userBytes
         UserBytesFeature userInfoBytes = null;
         userInfoBytes = new UserBytesFeature(user);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/CommonCollectionUtils.java

@@ -19,6 +19,14 @@ public class CommonCollectionUtils {
         return list.stream().map(map).collect(Collectors.toList());
     }
 
+     public static <T, R> List<R> toListDistinct(List<T> list, Function<T, R> map) {
+        if (CollectionUtils.isEmpty(list)) {
+            return Collections.emptyList();
+        }
+        return list.stream().map(map).distinct().collect(Collectors.toList());
+    }
+
+
     public static <T, K, V> Map<K, V> toMap(List<T> list, Function<T, K> keyFunc, Function<T, V> valueFunc) {
         if (CollectionUtils.isEmpty(list)) {
             return Collections.emptyMap();