request context

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -8,7 +8,7 @@ import java.util.Map;
 
 @Data
 public class RankItem implements Comparable<RankItem> {
-    public long videoid;
+    public long videoId;
     private double score; // 记录最终的score
     private Video video;
 
@@ -16,7 +16,7 @@ public class RankItem implements Comparable<RankItem> {
     private ItemFeature itemFeature;
 
     public RankItem(Video video) {
-        this.videoid = video.getVideoId() ;
+        this.videoId = video.getVideoId();
         this.score = 0.0;
         this.video = video;
     }
@@ -40,6 +40,4 @@ public class RankItem implements Comparable<RankItem> {
     }
 
 
-
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RequestContext.java

@@ -1,7 +1,6 @@
 package com.tzld.piaoquan.recommend.server.common.base;
 
 
-import lombok.Data;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 
@@ -28,96 +27,81 @@ public class RequestContext {
 
     public void setApptype(String apptype) {
         this.apptype = apptype;
-        if(apptype == null)
+        if (apptype == null)
             this.apptype = "-1";
     }
 
     public void setMachineinfo_brand(String machineinfo_brand) {
         this.machineinfo_brand = machineinfo_brand;
-        if(machineinfo_brand == null)
+        if (machineinfo_brand == null)
             this.machineinfo_brand = "-1";
     }
 
     public void setMachineinfo_model(String machineinfo_model) {
         this.machineinfo_model = machineinfo_model;
-        if(machineinfo_model == null)
+        if (machineinfo_model == null)
             this.machineinfo_model = "-1";
     }
 
 
     public void setMachineinfo_wechatversion(String machineinfo_wechatversion) {
         this.machineinfo_wechatversion = machineinfo_wechatversion;
-        if(machineinfo_wechatversion == null)
+        if (machineinfo_wechatversion == null)
             this.machineinfo_wechatversion = "-1";
     }
 
 
     public void setMachineinfo_sdkversion(String machineinfo_sdkversion) {
         this.machineinfo_sdkversion = machineinfo_sdkversion;
-        if(machineinfo_sdkversion == null)
+        if (machineinfo_sdkversion == null)
             this.machineinfo_sdkversion = "-1";
     }
 
     public void setMachineinfo_platform(String machineinfo_platform) {
         this.machineinfo_platform = machineinfo_platform;
-        if(machineinfo_platform == null)
+        if (machineinfo_platform == null)
             this.machineinfo_platform = "-1";
     }
 
     public void setMachineinfo_system(String machineinfo_system) {
         this.machineinfo_system = machineinfo_system;
-        if(machineinfo_system == null)
+        if (machineinfo_system == null)
             this.machineinfo_system = "-1";
     }
 
 
-
     public void setHour(String hour) {
         this.hour = hour;
-        if(hour == null)
+        if (hour == null)
             this.hour = "-1";
     }
 
 
     public void setDay(String day) {
         this.day = day;
-        if(day == null)
+        if (day == null)
             this.day = "-1";
     }
 
     public void setWeek(String week) {
         this.week = week;
-        if(week == null)
+        if (week == null)
             this.week = "-1";
     }
 
 
     public void setRegion(String region) {
         this.region = region;
-        if(region == null)
+        if (region == null)
             this.region = "-1";
     }
 
 
     public void setCity(String city) {
         this.city = city;
-        if(city == null)
+        if (city == null)
             this.city = "-1";
     }
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/remote/FeatureRemoteService.java

@@ -20,7 +20,6 @@ import java.util.List;
 import java.util.Map;
 
 
-
 /**
  * @author dyp
  */
@@ -85,7 +84,6 @@ public class FeatureRemoteService {
         return convert(proto);
     }
 
-    /*
     public Map<Long, ItemFeature> getVideoFeatureMap(List<Long> videoIds) {
         if (CollectionUtils.isEmpty(videoIds)) {
             return Collections.emptyMap();
@@ -93,7 +91,7 @@ public class FeatureRemoteService {
         List<VideoFeatureProto> protoList = client.getAllVideoFeature(videoIds);
         return CommonCollectionUtils.toMap(protoList, p -> NumberUtils.toLong(p.getVideoId(), 0L), this::convert);
     }
-     */
+
 
     private ItemFeature convert(VideoFeatureProto proto) {
         if (proto == null) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.recommend.server.service;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.base.Stopwatch;
 import com.google.common.base.Strings;
+import com.tzld.piaoquan.recommend.server.common.base.RequestContext;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.gen.common.Result;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
@@ -320,8 +321,6 @@ public class RecommendService {
                 ? param.getCityCode()
                 : "";
         recallParam.setRegionCode(cityCodes.contains(cityCode) ? cityCode : provinceCode);
-        // hard code  如果region为空，python任务会按照-1处理
-
 
         recallParam.setRuleKey(param.getRuleKey());
         recallParam.setDataKey(param.getDataKey());
@@ -341,6 +340,11 @@ public class RecommendService {
         rankParam.setTopK(param.getTopK());
         rankParam.setRankKeyPrefix(param.getRankKeyPrefix());
         rankParam.setAppType(param.getAppType());
+
+        // TODO RequestContext
+        RequestContext context = new RequestContext();
+
+        rankParam.setRequestContext(context);
         return rankParam;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankParam.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.recommend.server.service.rank;
 
+import com.tzld.piaoquan.recommend.server.common.base.RequestContext;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import lombok.Data;
 
@@ -18,4 +19,5 @@ public class RankParam {
     private boolean specialRecommend;
     private String uid;
     private String cityCode;
+    private RequestContext requestContext;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -1,19 +1,15 @@
 package com.tzld.piaoquan.recommend.server.service.rank;
 
 
-import com.typesafe.config.Config;
 import com.tzld.piaoquan.recommend.server.common.base.ItemFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.base.RequestContext;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.remote.FeatureRemoteService;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerPipeline;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
@@ -154,30 +150,25 @@ public class RankService {
             }
         }
     }
+
     private List<Video> mergeAndRankRovRecallNew(RankParam param) {
         UserFeature userFeature = featureRemoteService.getUserFeature(param.getUid());
         List<Video> recallVideos = param.getRecallResult().mergeRecallVideos();
         List<RankItem> rankItems = CommonCollectionUtils.toList(recallVideos, RankItem::new);
 
         List<Long> videoIds = CommonCollectionUtils.toListDistinct(recallVideos, Video::getVideoId);
-       /*
         Map<Long, ItemFeature> videoFeatureMap = featureRemoteService.getVideoFeatureMap(videoIds);
         for (RankItem rankItem : rankItems) {
-            rankItem.setItemFeature(videoFeatureMap.get(rankItem.getVideoid()));
+            rankItem.setItemFeature(videoFeatureMap.get(rankItem.getVideoId()));
         }
 
-        */
-
         // TODO
-        // 初始化 RecommendRequest, RequenstContxt
-        RequestContext requestContext = new RequestContext();
-
         List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(requestContext, param, userFeature, rankItems);
+                .scoring(param, userFeature, rankItems);
         log.info("mergeAndRankRovRecall rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
 
         return CommonCollectionUtils.toList(rovRecallScore, i -> {
-            // hard code 将排序分数 复制给video的sortScore
+            // hard code 将排序分数 赋值给video的sortScore
             Video v = i.getVideo();
             v.setSortScore(i.getScore());
             return v;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallResult.java

@@ -1,12 +1,15 @@
 package com.tzld.piaoquan.recommend.server.service.recall;
 
 import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import org.apache.commons.collections4.CollectionUtils;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
 
 /**
  * @author dyp
@@ -26,15 +29,18 @@ public class RecallResult {
     }
 
     public List<Video> mergeRecallVideos() {
-        HashSet<Video>  rankItems = new HashSet<Video>();
-        for(int i=0; i < data.size(); i++){
-            for(int j=0; j < data.get(i).videos.size(); j++)
-            rankItems.add(data.get(i).videos.get(j));
+        if (CollectionUtils.isEmpty(data)) {
+            return Collections.emptyList();
         }
-        List<Video> tmpList = new ArrayList<>(rankItems.size());
-        Collections.addAll(Arrays.asList(rankItems.toArray()), tmpList);
-
-        return tmpList;
+        LinkedHashSet<Video> rankItems = new LinkedHashSet<>();
+        for (int i = 0; i < data.size(); i++) {
+            if (CollectionUtils.isEmpty(data.get(i).videos)) {
+                continue;
+            }
+            for (int j = 0; j < data.get(i).videos.size(); j++)
+                rankItems.add(data.get(i).videos.get(j));
+        }
+        return new ArrayList<>(rankItems);
     }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/AbstractScorer.java

@@ -2,12 +2,8 @@ package com.tzld.piaoquan.recommend.server.service.score;
 
 
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.base.RequestContext;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.score.model.Model;
 import com.tzld.piaoquan.recommend.server.service.score.model.ModelManager;
 import org.apache.commons.lang3.StringUtils;
@@ -64,9 +60,8 @@ public abstract class AbstractScorer {
         return scorerConfigInfo;
     }
 
-    public abstract List<RankItem> scoring(final RequestContext requestContext,
-                                        final RankParam param,
-                                        final UserFeature userFeature,
-                                        final List<RankItem> rankItems);
+    public abstract List<RankItem> scoring(final RankParam param,
+                                           final UserFeature userFeature,
+                                           final List<RankItem> rankItems);
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerPipeline.java

@@ -39,8 +39,7 @@ public class ScorerPipeline {
      * scoring
      * @return
      */
-    public List<RankItem> scoring(final RequestContext requestContext,
-                               final RankParam param,
+    public List<RankItem> scoring(final RankParam param,
                                final UserFeature userFeature,
                                final List<RankItem> rankItems) {
         // check recall is empty?
@@ -70,7 +69,7 @@ public class ScorerPipeline {
             Callable<List<RankItem>> callable = new Callable<List<RankItem>>() {
                 @Override
                 public List<RankItem> call() throws Exception {
-                    return scorer.scoring(requestContext, param, userFeature, scoreRankerItems);
+                    return scorer.scoring(param, userFeature, scoreRankerItems);
                 }
             };
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareGBDTScorer.java

@@ -36,10 +36,9 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
     }
 
     @Override
-    public List<RankItem> scoring(final RequestContext requestContext,
-                               final RankParam param,
-                               final UserFeature userFeature,
-                               final List<RankItem> rankItems) {
+    public List<RankItem> scoring(final RankParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
         RecallResult recallResult = param.getRecallResult();
         if (recallResult.getData().size() == 0) {
             return rankItems;
@@ -54,7 +53,7 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
         }
 
         // 多Rank的rank打分
-        multipleGBDTScore(rankItems, model, userFeature, requestContext);
+        multipleGBDTScore(rankItems, model, userFeature, param.getRequestContext());
         Collections.sort(rankItems);
         LOGGER.debug("dwelltime ranker excute time: [{}]", System.currentTimeMillis() - startTime);
         return rankItems;
@@ -70,8 +69,8 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
      * @return
      */
     private double predictMultipleGBDTScore(final double oldScore, final double dwelltimeOrCompletion,
-                                         final double duration, final String itemId,
-                                         final RecommendRequest requestData, final UserFeature user) {
+                                            final double duration, final String itemId,
+                                            final RecommendRequest requestData, final UserFeature user) {
         double ctrScalePower = 0;
         double dwelltimeScalePower = 0;
 
@@ -81,7 +80,7 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
         }
         double newScore = Math.pow(ctrScore, ctrScalePower) * Math.pow(dwelltimeOrCompletion, dwelltimeScalePower);
         //值越大压制越弱
-        double alpha =  1.0;
+        double alpha = 1.0;
         double suppressRatio = (1 + alpha) / (Math.max(duration, 300.0) / 300 + alpha);
         newScore = newScore * suppressRatio;
         LOGGER.debug("Expected dwell time score [{}]: {} ,{}, {}--> {}", new Object[]{itemId, oldScore, dwelltimeOrCompletion, duration, newScore});
@@ -89,11 +88,11 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
     }
 
     private void gbdtBaseScore(final RankItem item,
-                                final GBDTModel model,
-                                final RequestContext requestContext,
-                                final Map<String, Double> userFeatures,
-                                final Map<String, Double> contextFeatures,
-                                final UserFeature user) {
+                               final GBDTModel model,
+                               final RequestContext requestContext,
+                               final Map<String, Double> userFeatures,
+                               final Map<String, Double> contextFeatures,
+                               final UserFeature user) {
         try {
 
             Map<String, Double> articleFeatures;
@@ -120,9 +119,9 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
 
 
     public void multipleGBDTScore(final List<RankItem> items,
-                                       final GBDTModel model,
-                                       final UserFeature user,
-                                       final RequestContext requestContext) {
+                                  final GBDTModel model,
+                                  final UserFeature user,
+                                  final RequestContext requestContext) {
         final int size = items.size();
         if (size == 0) {
             return;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareLRScorer.java

@@ -1,17 +1,9 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
 
-
 import com.tzld.piaoquan.recommend.server.common.base.*;
-import com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples;
 import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import com.tzld.piaoquan.recommend.server.service.score.feature.FeatureUsage;
-import com.tzld.piaoquan.recommend.server.service.score.feature.LRBytesFeatureExtractorBase;
 import com.tzld.piaoquan.recommend.server.service.score.feature.VlogShareLRFeatureExtractor;
 import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
 import org.apache.commons.lang.exception.ExceptionUtils;
@@ -24,11 +16,11 @@ import java.util.List;
 import java.util.concurrent.*;
 
 
-public class VlogShareLRScorer extends BaseLRModelScorer{
+public class VlogShareLRScorer extends BaseLRModelScorer {
 
     private final static int CORE_POOL_SIZE = 64;
 
-    private static final int LOCAL_TIME_OUT= 150;
+    private static final int LOCAL_TIME_OUT = 150;
     private final static Logger LOGGER = LoggerFactory.getLogger(VlogShareLRScorer.class);
     private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
     private static final double defaultUserCtrGroupNumber = 10.0;
@@ -42,10 +34,9 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
 
 
     @Override
-    public List<RankItem> scoring(final RequestContext requestContext,
-                               final RankParam param,
-                               final UserFeature userFeature,
-                               final List<RankItem> rankItems) {
+    public List<RankItem> scoring(final RankParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
@@ -54,7 +45,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
         }
 
         List<RankItem> result = rankItems;
-        result = rankByJava(rankItems, requestContext, userFeature);
+        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
 
         LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
                 System.currentTimeMillis() - startTime);
@@ -63,8 +54,8 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
     }
 
     private List<RankItem> rankByJava(final List<RankItem> items,
-                                        final RequestContext requestContext,
-                                        final UserFeature user) {
+                                      final RequestContext requestContext,
+                                      final UserFeature user) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
@@ -109,7 +100,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
             lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
                     new RequestContextBytesFeature(requestContext));
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getUid()), item.getVideoid(),
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getUid()), item.getVideoId(),
                     ExceptionUtils.getFullStackTrace(e)});
         }
 
@@ -120,7 +111,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
                 LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoid(), ExceptionUtils.getFullStackTrace(e)});
+                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
             }
             // 增加实时特征后打开在线存储日志逻辑
             //
@@ -157,7 +148,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer{
                     try {
                         calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
                     } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoid, ExceptionUtils.getFullStackTrace(e));
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
                     }
                     return new Object();
                 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/LRModel.java

@@ -138,7 +138,7 @@ public class LRModel extends Model {
                 continue;
             }
 
-            putFeature(model, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
+            putFeature(model, new BigInteger(items[0].trim()).longValue(), Float.valueOf(items[1].trim()).floatValue());
             if (cnt++ < 10) {
                 LOGGER.debug("fea: " + items[0] + ", weight: " + items[1]);
             }