v2 20231203

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -0,0 +1,33 @@
+package com.tzld.piaoquan.recommend.server.common.base;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+
+public class RankItem implements Comparable<RankItem> {
+    public long videoid;
+    private double rankScore;
+
+
+    public RankItem(Video video) {
+        this.videoid = video.getVideoId() ;
+        this.rankScore = 0.0 ;
+    }
+
+
+    @Override
+    public int compareTo(RankItem o) {
+        if (o == null) {
+            return -1;
+        }
+        if (rankScore > o.rankScore) {
+            return -1;
+        } else if (rankScore < o.rankScore) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RequestContext.java

@@ -0,0 +1,9 @@
+package com.tzld.piaoquan.recommend.server.common.base;
+
+
+import lombok.Data;
+
+@Data
+public class RequestContext {
+    String cityCode;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RequestContextBytesFeature.java

@@ -0,0 +1,11 @@
+package com.tzld.piaoquan.recommend.server.common.base;
+
+
+
+public class RequestContextBytesFeature {
+
+
+    public RequestContextBytesFeature(RequestContext requestContext){
+
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/UserBytesFeature.java

@@ -0,0 +1,11 @@
+package com.tzld.piaoquan.recommend.server.common.base;
+import io.lettuce.core.StrAlgoArgs;
+import lombok.Data;
+
+@Data
+public class UserBytesFeature {
+    public final byte[]  sex;
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/UserFeature.java

@@ -1,8 +1,9 @@
 package com.tzld.piaoquan.recommend.server.common.base;
 
-public class UserFeature {
-    public final byte[] sex;
-
+import lombok.Data;
 
+@Data
+public class UserFeature {
+    private String sex;
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/VideoBytesFeature.java

@@ -0,0 +1,13 @@
+package com.tzld.piaoquan.recommend.server.common.base;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import lombok.Data;
+
+public class VideoBytesFeature {
+    private final byte[] videoId;
+
+    public VideoBytesFeature(RankItem video) {
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/VideoRankFeature.java

@@ -1,10 +0,0 @@
-package com.tzld.piaoquan.recommend.server.common.base;
-
-public class VideoRankFeature {
-    public final byte[] videoid;
-
-    public VideoRankFeature(String item) {
-        this.videoid = item.getBytes();
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/Video.java

@@ -22,4 +22,5 @@ public class Video {
 
     private double rand;
     private String lastVideoKey;
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -1,6 +1,11 @@
 package com.tzld.piaoquan.recommend.server.service.rank;
 
+
+import com.typesafe.config.Config;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
@@ -28,10 +33,63 @@ public class RankService {
     @Autowired
     private RedisTemplate<String, String> redisTemplate;
 
-    public RankResult rank(RankParam param) {
 
+    // LR scores
+    public RankResult rank(RecommendRequest recommendRequest,
+                           RankParam param,
+                           RecallResult recallResult,
+                           UserFeature userFeature,
+                           List<RankItem> videoRankFeatures,
+                           Config config) {
+        if (param == null
+                || param.getRecallResult() == null
+                || CollectionUtils.isEmpty(param.getRecallResult().getData())) {
+            return null;
+        }
+
+        if (param.isSpecialRecommend()) {
+            Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                    .filter(d -> d.getPushFrom().equals(SpecialRecallStrategy.PUSH_FROM))
+                    .findFirst();
+            if (data.isPresent()
+                    && data.get() != null) {
+                return new RankResult(data.get().getVideos());
+            }
+            return null;
+        }
+
+        ScorerPipeline scorerPipeline = ScorerUtils.getScorerPipeline(config);
+        // TODO  merge 后返回待排序的video list
+        List<Video> recallVideos = recallResult.mergeRecallVideos();
+        // TODO 转化成rankitem
+        // convert List<Video> to List<RankItem>
+        List<RankItem> rankItem = new ArrayList<RankItem>();
+
+        List<RankItem> rovRecallRank  =
+                scorerPipeline.scoring(recommendRequest, param, userFeature, rankItem);
+
+        log.info("mergeAndRankRovRecall rovRecallRank={}", JSONUtils.toJson(rovRecallRank));
+        List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
+        log.info("mergeAndRankFlowPoolRecall flowPoolRank={}", JSONUtils.toJson(flowPoolRank));
+
+        removeDuplicate(param, rovRecallRank, flowPoolRank);
+        // convert List<RankItem> to List<Video>
+
+
+
+        log.info("removeDuplicate rovRecallRank={}, flowPoolRank={}",
+                JSONUtils.toJson(rovRecallRank),
+                JSONUtils.toJson(flowPoolRank));
+
+        // 融合排序
+        return mergeAndSort(param, rovRecallRank, flowPoolRank);
+
+
+    }
+
+
+    public RankResult rank(RankParam param) {
 
-        ScorerPipeline scorerPipeline = ScorerUtils.getScorerPipeline();
 
         if (param == null
                 || param.getRecallResult() == null

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallResult.java

@@ -1,11 +1,12 @@
 package com.tzld.piaoquan.recommend.server.service.recall;
 
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.util.List;
+import java.util.*;
 
 /**
  * @author dyp
@@ -24,4 +25,16 @@ public class RecallResult {
         private List<Video> videos;
     }
 
+    public List<Video> mergeRecallVideos() {
+        HashSet<Video>  rankItems = new HashSet<Video>();
+        for(int i=0; i < data.size(); i++){
+            for(int j=0; j < data.get(i).videos.size(); j++)
+            rankItems.add(data.get(i).videos.get(j));
+        }
+        List<Video> tmpList = new ArrayList<>(rankItems.size());
+        Collections.addAll(Arrays.asList(rankItems.toArray()), tmpList);
+
+        return tmpList;
+    }
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/AbstractScorer.java

@@ -1,6 +1,9 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
 
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
@@ -60,6 +63,9 @@ public abstract class AbstractScorer {
         return scorerConfigInfo;
     }
 
-    public abstract List<Video> scoring(final RankParam param);
+    public abstract List<RankItem> scoring(final RecommendRequest request,
+                                        final RankParam param,
+                                        final UserFeature userFeature,
+                                        final List<RankItem> rankItems);
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerPipeline.java

@@ -1,7 +1,7 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.VideoRankFeature;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
@@ -38,14 +38,12 @@ public class ScorerPipeline {
      * scoring
      * @return
      */
-    public List<Video> scoring(final RecommendRequest recommendRequest,
+    public List<RankItem> scoring(final RecommendRequest recommendRequest,
                                final RankParam param,
-                               final RecallResult recallResult,
                                final UserFeature userFeature,
-                               final VideoRankFeature videoFeature
-                               ) {
+                               final List<RankItem> rankItems) {
         // check recall is empty?
-        if (CollectionUtils.isEmpty(recallResult.getData())) {
+        if (CollectionUtils.isEmpty(param.getRecallResult().getData())) {
             // log.error
         }
 
@@ -53,24 +51,23 @@ public class ScorerPipeline {
             // log.error()
         }
         long scoreStart = System.currentTimeMillis();
-        List<Video> items = new ArrayList<Video>();
         for (final AbstractScorer scorer : scorers) {
             if (!scorer.isEnable()) {
                 continue;
             }
 
-            final int beforeSize = items.size();
+            final int beforeSize = rankItems.size();
             final long startTime = System.currentTimeMillis();
 
             String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
             String[] scorerNames = fullScorerName.split("\\.");
             final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
 
-            final List<Video> scoreRankerItems = items;
-            Callable<List<Video>> callable = new Callable<List<Video>>() {
+            final List<RankItem> scoreRankerItems = rankItems;
+            Callable<List<RankItem>> callable = new Callable<List<Video>>() {
                 @Override
-                public List<Video> call() throws Exception {
-                    return scorer.scoring(param);
+                public List<RankItem> call() throws Exception {
+                    return scorer.scoring(recommendRequest, param, userFeature, rankItems);
                 }
             };
            //

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/FeedsShareLTRScorer.java → recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareGBDTScorer.java

@@ -1,15 +1,15 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
 
-
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.base.RequestContext;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.VideoRankFeature;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import com.tzld.piaoquan.recommend.server.service.score.feature.VlogShareFeatureExtractor;
+import com.tzld.piaoquan.recommend.server.service.score.feature.GBDTFeatureExtractorBase;
 import com.tzld.piaoquan.recommend.server.service.score.model.GBDTModel;
+import com.tzld.piaoquan.recommend.server.model.Video;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -25,52 +25,127 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
-
-public class FeedsShareLTRScorer extends BaseGBDTModelScorer{
+public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
 
     private final static int CORE_POOL_SIZE = 64;
     private final static int TIME_OUT = 150;
-    private final static Logger LOGGER = LoggerFactory.getLogger(FeedsShareLTRScorer.class);
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogShareGBDTScorer.class);
     private final static ExecutorService executorService = Executors.newFixedThreadPool(CORE_POOL_SIZE);
+    private static final String VIMAGE_VIDEO_REC_TAG = "小视频";
 
-    public FeedsShareLTRScorer(ScorerConfigInfo configInfo) {
-
+    public VlogShareGBDTScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
     }
 
+
     private RequestContext getRequestContext(RecommendRequest request) {
 
+        RequestContext requestContext = new RequestContext();
+        return requestContext;
     }
 
     @Override
-    public List<Video> scoring(final RecommendRequest recommendRequest,
+    public List<RankItem> scoring(final RecommendRequest request,
                                final RankParam param,
-                               final RecallResult recallResult,
                                final UserFeature userFeature,
-                               final VideoRankFeature videoFeature) {
+                               final List<RankItem> rankItems) {
+        RecallResult recallResult = param.getRecallResult();
+        if (recallResult.getData().size() == 0) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+
+        GBDTModel model = (GBDTModel) this.getModel();
+        if (model == null) {
+            LOGGER.error("not found model");
+            return rankItems;
+        }
+        RequestContext requestContext = getRequestContext(request);
 
-        feedsShareScore();
-        Collections.sort();
-        return items;
+        // 多Rank的rank打分
+        multipleGBDTScore(rankItems, model, userFeature, request, requestContext);
+        Collections.sort(rankItems);
+        LOGGER.debug("dwelltime ranker excute time: [{}]", System.currentTimeMillis() - startTime);
+        return rankItems;
     }
 
+    /**
+     * 分数转换
+     *
+     * @param oldScore
+     * @param dwelltimeOrCompletion
+     * @param duration
+     * @param itemId
+     * @return
+     */
+    private double predictMultipleGBDTScore(final double oldScore, final double dwelltimeOrCompletion,
+                                         final double duration, final String itemId,
+                                         final RecommendRequest requestData, final UserFeature user) {
+        double ctrScalePower = 0;
+        double dwelltimeScalePower = 0;
+
+        double ctrScore = oldScore;
+        if (oldScore <= 0.0) {
+            ctrScore = 0.01;
+        }
+        double newScore = Math.pow(ctrScore, ctrScalePower) * Math.pow(dwelltimeOrCompletion, dwelltimeScalePower);
+        //值越大压制越弱
+        double alpha =  1.0;
+        double suppressRatio = (1 + alpha) / (Math.max(duration, 300.0) / 300 + alpha);
+        newScore = newScore * suppressRatio;
+        LOGGER.debug("Expected dwell time score [{}]: {} ,{}, {}--> {}", new Object[]{itemId, oldScore, dwelltimeOrCompletion, duration, newScore});
+        return newScore;
+    }
+
+    private void gbdtBaseScore(final Video item,
+                                final GBDTModel model,
+                                final RequestContext requestContext,
+                                final Map<String, Double> userFeatures,
+                                final Map<String, Double> contextFeatures,
+                                final RecommendRequest requestData, final UserFeature user) {
+        try {
+
+            Map<String, Double> articleFeatures;
+            double freshness;
+            Map<String, Double> features;
+            features = GBDTFeatureExtractorBase.extractUserFeatures(user);
 
+            if (model != null) {
+                Map<String, Double> featureScoreMap = new HashMap<String, Double>();
+                double pro = model.score(features, featureScoreMap);
+                featureScoreMap.put("TOTAL_SCORE", pro);
+                pro = pro > 1 ? pro : 1;
+                Double rankScore = item.getRankScore();
+                LOGGER.debug("xgb score = {}, lgb score = {}", rankScore, pro);
+                double lgbWeight = 1.0;
+                pro = (rankScore + lgbWeight * pro) / (1.0 + lgbWeight);
+                item.setRankScore(pro);
+            }
 
-    public void feedsShareScore(final List<Video> items,
+        } catch (Exception e) {
+            LOGGER.error("Exception {},{}", requestContext, ExceptionUtils.getFullStackTrace(e));
+        }
+    }
+
+
+    public void multipleGBDTScore(final List<Video> items,
                                        final GBDTModel model,
                                        final UserFeature user,
-                                       final RecommendRequest requestData) {
+                                       final RecommendRequest requestData,
+                                       final RequestContext requestContext) {
         final int size = items.size();
         if (size == 0) {
             return;
         }
+
         final Map<String, Double> userFeatures;
         final Map<String, Double> contextFeatures;
 
-
         // context feature
-        contextFeatures = VlogShareFeatureExtractor.extractContextFeatures(requestContext);
+        contextFeatures = GBDTFeatureExtractorBase.extractRequestContextFeatures(requestContext);
         // user feature
-        userFeatures = VlogShareFeatureExtractor.extractUserFeatures();
+        userFeatures = GBDTFeatureExtractorBase.extractUserFeatures(user);
 
         // score item
         List<Callable<Object>> callables = new ArrayList<Callable<Object>>();
@@ -80,9 +155,9 @@ public class FeedsShareLTRScorer extends BaseGBDTModelScorer{
                               @Override
                               public Object call() throws Exception {
                                   try {
-                                      dwelltimeScore(items.get(fIndex), model, requestContext, userFeatures, contextFeatures, requestData, user);
+                                      gbdtBaseScore(items.get(fIndex), model, requestContext, userFeatures, contextFeatures, requestData, user);
                                   } catch (Exception e) {
-                                      LOGGER.error("dwelltime exception: [{}] [{}]", items.get(fIndex).getId(), ExceptionUtils.getFullStackTrace(e));
+                                      LOGGER.error("dwelltime exception: [{}] [{}]", items.get(fIndex), ExceptionUtils.getFullStackTrace(e));
                                   }
                                   return new Object();
                               }
@@ -103,7 +178,7 @@ public class FeedsShareLTRScorer extends BaseGBDTModelScorer{
                 try {
                     if (future != null && future.isDone() && !future.isCancelled() && future.get() != null) {
                     } else {
-                        LOGGER.debug("Canceled Dwelltime Score {}", requestContext.getId());
+                        LOGGER.debug("Canceled Dwelltime Score {}", requestContext);
                     }
                 } catch (Exception e) {
                     LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
@@ -111,65 +186,5 @@ public class FeedsShareLTRScorer extends BaseGBDTModelScorer{
             }
         }
     }
-
-
-
-
-    // GBDT计算score
-    private void shareScore(final RankerItem item,
-                                final GBDTModel model,
-                                final RequestContext requestContext,
-                                final Map<String, Double> userFeatures,
-                                final Map<String, Double> contextFeatures,
-                                final int debugLevel,
-                                final RecommendRequest requestData, final User user) {
-        try {
-            //judge null
-            if (item == null || item.getItemInfo() == null) {
-                return;
-            }
-
-            ArticleInfo articleInfo = (ArticleInfo) item.getItemInfo();
-
-            Map<String, Double> articleFeatures;
-            double freshness;
-            Map<String, Double> features;
-            articleFeatures = DurationFeatureExtractor.extractArticleFeatures(articleInfo);
-
-
-            if (model != null) {
-                Map<String, Double> featureScoreMap = new HashMap<String, Double>();
-                double pro = model.score(features, featureScoreMap, debugLevel);
-                featureScoreMap.put("TOTAL_SCORE", pro);
-
-                pro = pro > 1 ? pro : 1;
-                LOGGER.debug("xgb score = {}, lgb score = {}", item.getDwelltimeScore(), pro);
-
-                double lgbWeight = Configuration.getDouble("mivideo-recommend-service.dwelltime_ranker.lgb.weight", 1.0);
-                pro = (item.getDwelltimeScore() + lgbWeight * pro) / (1.0 + lgbWeight);
-                item.setDwelltimeScore(pro);
-                double duration = 0;
-                if(item.getItemInfo() != null) {
-                    duration = ((ArticleInfo) item.getItemInfo()).getDuration();
-                }
-                duration = Math.min(Math.max(duration, 1), 1800);
-                item.setScore(predictDwelltimeScore(item.getRecScore(), pro, duration, item.getId(), requestData, user));
-            }
-
-        } catch (Exception e) {
-            LOGGER.error("Exception {},{}", requestContext.getId(), ExceptionUtils.getFullStackTrace(e));
-        }
-    }
-
-
-
-
-
-
-
-
-
-
-
-
 }
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareLRScorer.java

@@ -0,0 +1,208 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+
+
+import com.tzld.piaoquan.recommend.server.common.base.*;
+import com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples;
+import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
+import com.tzld.piaoquan.recommend.server.service.score.feature.FeatureUsage;
+import com.tzld.piaoquan.recommend.server.service.score.feature.LRBytesFeatureExtractorBase;
+import com.tzld.piaoquan.recommend.server.service.score.feature.VlogShareLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.*;
+
+
+public class VlogShareLRScorer extends BaseLRModelScorer{
+
+    private final static int CORE_POOL_SIZE = 64;
+    private final static int TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogShareLRScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+    private static final FeatureUsage featureUsage = new FeatureUsage();
+    private static final double defaultUserCtrGroupNumber = 10.0;
+    private static final int enterFeedsScoreRatio = 10;
+    private static final int enterFeedsScoreNum = 20;
+
+
+    public VlogShareLRScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+
+    @Override
+    public List<RankItem> scoring(final RecommendRequest request,
+                               final RankParam param,
+                               final UserFeature userFeature,
+                               final List<RankItem> rankItems) {
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+        if (rankItems.size() == 0) {
+            return rankItems;
+        }
+
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(rankItems, request, userFeature);
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final List<RankItem> items,
+                                        final RecommendRequest request,
+                                        final UserFeature user) {
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        String cityCode = request.getCityCode();
+        RequestContext requestContext = new RequestContext();
+        requestContext.setCityCode(cityCode);
+
+
+        UserBytesFeature userInfoBytes = null;
+        userInfoBytes = new UserBytesFeature(user.getSex());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userInfoBytes, requestContext, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+
+    /**
+     * 计算 predict ctr
+     */
+    public double calcScore(final LRModel lrModel,
+                            final RankItem item,
+                            final UserBytesFeature userInfoBytes,
+                            final RequestContext requestContext) {
+
+        LRSamples lrSamples = null;
+        VlogShareLRFeatureExtractor bytesFeatureExtractor;
+        bytesFeatureExtractor = new VlogShareLRFeatureExtractor(featureUsage, 100.0, 1);
+
+        try {
+            VideoBytesFeature newsInfoBytes = new VideoBytesFeature(item);
+            lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
+                    new RequestContextBytesFeature(requestContext));
+        } catch (Exception e) {
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.imei), item.getId(),
+                    ExceptionUtils.getFullStackTrace(e)});
+        }
+
+
+        double pro = 0.0;
+        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
+            try {
+                pro = lrModel.score(lrSamples);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", new Object[]{
+                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+            }
+
+
+            CtrSamples samples = CtrSamples.newBuilder();
+            samples.setLr_samples(lrSamples);
+            item.setSamples(samples);
+        }
+
+        item.setScore(pro);
+        item.setRecScore(pro);
+        return pro;
+    }
+
+    /**
+     * 并行打分
+     *
+     */
+    private void multipleCtrScore(final RecommendRequest request,
+                                  final RankParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < rankItems.size(); index++) {
+            final int fIndex = index;
+            rankItems.get(fIndex).setRankScore(0.0);//原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userInfoBytes, requestContext, feedsContext);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).getId(), ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, 200, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", request.getRequestId(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{request.getRequestId(), rankItems.size(), cancel});
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/feature/BytesUtils.java

@@ -12,12 +12,9 @@ import java.util.List;
 public class BytesUtils {
     private static final byte[] SEPARATOR = "_".getBytes();
     private static final byte[] FEATURE_SEPARATOR = "#".getBytes();
-
     private static final int MAX_FEATURE_BYTES_LENGTH = 512;
     private static final long SEED = 11L;
-
     private BytesGroup[] groups;
-    private final int debugLevel;
 
     /**
      * 一个种特殊的List,在尝试写入null的时候回默默地扔掉.
@@ -38,9 +35,8 @@ public class BytesUtils {
         }
     }
 
-    public BytesUtils(BytesGroup[] groups, FeatureUsage usage, int debugLevel) {
+    public BytesUtils(BytesGroup[] groups, FeatureUsage usage) {
         this.groups = groups;
-        this.debugLevel = debugLevel;
         for (BytesGroup g : groups) {
             byte[] buffer = prepareBuffer(g.getName(), g.getNameBytes(), usage);
             groups[g.getId()].setBuffer(buffer);
@@ -59,13 +55,11 @@ public class BytesUtils {
 
     public BaseFeature baseFea(byte[] buffer, int length) {
         long hash = FeatureHash.MurmurHash64(buffer, 0, length, SEED);
-        String fea = new String(buffer, 0, length);
+
+        // debug中查看 String fea = new String(buffer, 0, length);
         // 初始化protobuf并赋值
         BaseFeature.Builder tmp = BaseFeature.newBuilder();
         tmp.setIdentifier(hash);
-        if (debugLevel > 0) {
-            tmp.setFea(fea);
-        }
         return tmp.build();
     }
 
@@ -78,7 +72,6 @@ public class BytesUtils {
         final int nameLength = groups[id].getNameBytes().length + 1;
         final int length = nameLength + value.length;
         System.arraycopy(value, 0, buffer, nameLength, value.length);
-
         return baseFea(buffer, length);
     }
 
@@ -94,7 +87,6 @@ public class BytesUtils {
         System.arraycopy(p1, 0, buffer, nameLength, p1.length);
         System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length, 1);
         System.arraycopy(p2, 0, buffer, nameLength + p1.length + 1, p2.length);
-
         return baseFea(buffer, length);
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/feature/GBDTFeatureExtractorBase.java

@@ -0,0 +1,52 @@
+package com.tzld.piaoquan.recommend.server.service.score.feature;
+
+
+
+import com.tzld.piaoquan.recommend.server.common.base.RequestContext;
+import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class GBDTFeatureExtractorBase {
+    private static Logger LOGGER = LoggerFactory.getLogger(GBDTFeatureExtractorBase.class);
+
+    // TODO
+    // need to add which features to extract
+    public static Map<String, Double> extractVideoFeatures(final RecallResult recallResult) {
+        Map<String, Double> features = new HashMap<String, Double>();
+        // need to add which features to extract
+
+        return features;
+    }
+
+
+    public static Map<String, Double> extractRequestContextFeatures(final RequestContext requestContext) {
+        Map<String, Double> features = new HashMap<String, Double>();
+        // need to add which features to extract
+
+        return features;
+    }
+
+
+
+    // TODO
+    public static Map<String, Double> extractUserFeatures(final UserFeature userFeature) {
+        Map<String, Double> features = new HashMap<String, Double>();
+        // need to add which features to extract
+
+        return features;
+
+    }
+
+
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/feature/LRFeatureExtractorBase.java → recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/feature/LRBytesFeatureExtractorBase.java

@@ -3,25 +3,17 @@ package com.tzld.piaoquan.recommend.server.service.score.feature;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ListMultimap;
 
-import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.VideoRankFeature;
+import com.tzld.piaoquan.recommend.server.common.base.*;
 import com.tzld.piaoquan.recommend.server.common.enums.VlogFeatureGroup;
-import com.tzld.piaoquan.recommend.server.gen.recommend.GroupedFeature;
 import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
 import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.gen.recommend.FeatureGroup;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import lombok.Synchronized;
 
-import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 
-
-public abstract class LRFeatureExtractorBase {
+public abstract class LRBytesFeatureExtractorBase {
     private static final double DEFAULT_USER_CTR_GROUP = 10.0;
     private static final double DEFAULT_ARTICLE_CTR_GROUP = 100.0;
 
@@ -29,16 +21,17 @@ public abstract class LRFeatureExtractorBase {
     double videoCtrGroup = 100.0;
 
     private BytesUtils utils;
+    //Feature Group & Features
     ListMultimap<FeatureGroup, BaseFeature> features = ArrayListMultimap.create();
     int groupCount;
-    LRFeatureExtractorBase() {
+    LRBytesFeatureExtractorBase() {
     };
 
-    LRFeatureExtractorBase(FeatureUsage usage) {
+    LRBytesFeatureExtractorBase(FeatureUsage usage) {
         this(usage, DEFAULT_USER_CTR_GROUP, DEFAULT_ARTICLE_CTR_GROUP);
     }
 
-    LRFeatureExtractorBase(FeatureUsage usage, double userCtrGroup, double videoCtrGroup) {
+    LRBytesFeatureExtractorBase(FeatureUsage usage, double userCtrGroup, double videoCtrGroup) {
         this.userCtrGroup = userCtrGroup;
         this.videoCtrGroup = videoCtrGroup;
         groupCount = VlogFeatureGroup.values().length;
@@ -47,7 +40,7 @@ public abstract class LRFeatureExtractorBase {
             groups[g.ordinal()] = new BytesGroup(g.ordinal(),
                     g.getGroupName(), g.getGroupNameBytes());
         }
-        utils = new BytesUtils(groups, usage, 0);
+        utils = new BytesUtils(groups, usage);
     }
 
     private FeatureGroup makeGroup(VlogFeatureGroup group){
@@ -71,7 +64,9 @@ public abstract class LRFeatureExtractorBase {
         features.putAll(g, featureList);
     }
 
-    public abstract LRSamples single(RecommendRequest request, UserFeature user, VideoRankFeature itemFeature);
+    public abstract LRSamples single(UserBytesFeature userBytesFeature,
+                                     VideoBytesFeature videoBytesFeature,
+                                     RequestContextBytesFeature requestContextBytesFeature);
 
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/feature/VlogShareFeatureExtractor.java

@@ -1,77 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.score.feature;
-
-import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.VideoRankFeature;
-import com.tzld.piaoquan.recommend.server.common.enums.VlogFeatureGroup;
-import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
-import com.tzld.piaoquan.recommend.server.gen.recommend.GroupedFeature;
-import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class VlogShareFeatureExtractor extends LRFeatureExtractorBase {
-
-    public VlogShareFeatureExtractor() {
-        super();
-    }
-    public VlogShareFeatureExtractor(FeatureUsage usage) {
-        super(usage);
-    }
-    public VlogShareFeatureExtractor(FeatureUsage usage, double userCtrGroup, double videoCtrGroup, int debugLevel) {
-        super(usage, userCtrGroup, videoCtrGroup);
-    }
-
-    private void getUserFeatures(UserFeature user) {
-        makeFea(VlogFeatureGroup.SEX, user.sex);
-    }
-
-    private void getContextFeature(RecommendRequest request) {
-        makeFea();
-    }
-
-
-    private void getItemFeatures(VideoRankFeature item) {
-        makeFea(VlogFeatureGroup.VIDEOID, item.videoid);
-    }
-
-
-    public LRSamples single(RecommendRequest request, UserFeature user, VideoRankFeature itemfeature) {
-        features.clear();
-        //
-        getUserFeatures(user);
-        getContextFeature(request);
-        getItemFeatures(itemfeature);
-
-        LRSamples.Builder lr =  com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples.newBuilder();
-        lr.setGroupNum(groupCount);
-        List<com.tzld.piaoquan.recommend.server.gen.recommend.FeatureGroup> keys = new ArrayList<>(features.keySet());
-        int count = 0;
-        for(com.tzld.piaoquan.recommend.server.gen.recommend.FeatureGroup group : keys) {
-            List<BaseFeature> fea = features.get(group);
-            GroupedFeature.Builder gf = GroupedFeature.newBuilder();
-            gf.setGroup(group);
-            gf.setCount(fea.size());
-            gf.addAllFeatures(fea);
-            count += fea.size();
-            lr.addFeatures(gf);
-        }
-        lr.setCount(count);
-        return lr.build();
-    }
-
-
-
-
-
-
-
-
-
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/feature/VlogShareLRFeatureExtractor.java

@@ -0,0 +1,80 @@
+package com.tzld.piaoquan.recommend.server.service.score.feature;
+
+import com.tzld.piaoquan.recommend.server.common.base.*;
+import com.tzld.piaoquan.recommend.server.common.enums.VlogFeatureGroup;
+import com.tzld.piaoquan.recommend.server.gen.recommend.*;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class VlogShareLRFeatureExtractor extends LRBytesFeatureExtractorBase {
+
+    public VlogShareLRFeatureExtractor() {
+        super();
+    }
+    public VlogShareLRFeatureExtractor(FeatureUsage usage) {
+        super(usage);
+    }
+    public VlogShareLRFeatureExtractor(FeatureUsage usage, double userCtrGroup, double videoCtrGroup) {
+        super(usage, userCtrGroup, videoCtrGroup);
+    }
+
+
+    // TODO
+    // 补充待抽取的context feature
+    private void getContextFeatures(RequestContextBytesFeature requestContextBytes) {
+        makeFea(VlogFeatureGroup.DAY_OF_WEEK, "1".getBytes());
+
+    }
+
+    //TODO
+    private void getUserFeatures(UserBytesFeature user) {
+        makeFea(VlogFeatureGroup.SEX, user.sex);
+
+    }
+
+    private void getItemFeature(VideoBytesFeature videoBytesFeature) {
+        makeFea(VlogFeatureGroup.VIDEOID, videoBytesFeature.videoId);
+    }
+
+    @Override
+    public synchronized LRSamples single(UserBytesFeature userBytesFeature,
+                            VideoBytesFeature videoBytesFeature,
+                            RequestContextBytesFeature requestContextBytesFeature) {
+        features.clear();
+        //
+        getUserFeatures(userBytesFeature);
+        getContextFeatures(requestContextBytesFeature);
+        getItemFeature(videoBytesFeature);
+
+        LRSamples.Builder lr =  com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples.newBuilder();
+        lr.setGroupNum(groupCount);
+        List<FeatureGroup> keys = new ArrayList<>(features.keySet());
+        int count = 0;
+        for(FeatureGroup group : keys) {
+            List<BaseFeature> fea = features.get(group);
+            GroupedFeature.Builder gf = GroupedFeature.newBuilder();
+            gf.setGroup(group);
+            gf.setCount(fea.size());
+            gf.addAllFeatures(fea);
+            count += fea.size();
+            lr.addFeatures(gf);
+        }
+        lr.setCount(count);
+        return lr.build();
+    }
+
+
+
+
+
+
+
+
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/GBDTModel.java

@@ -161,7 +161,7 @@ public class GBDTModel extends Model {
         }
     }
 
-    public Float score(final Map<String, Double> features, Map<String, Double> featuresScore, final int debugLevel) {
+    public Float score(final Map<String, Double> features, Map<String, Double> featuresScore) {
         Float result = 0f;
 
         boolean[] featureIsExists = new boolean[featureCount];
@@ -180,23 +180,6 @@ public class GBDTModel extends Model {
 
         Float transformedResult = transform(result);
 
-        if (debugLevel >= 1) {
-            if (null == featuresScore) {
-                featuresScore = new HashMap<String, Double>();
-            }
-
-            for (Map.Entry<String, Double> feature : features.entrySet()) {
-                featuresScore.put(feature.getKey(), feature.getValue());
-            }
-            int existFeatureCount = 0;
-            for (boolean isExisit : featureIsExists) {
-                existFeatureCount += isExisit ? 1 : 0;
-            }
-            featuresScore.put("COUNT", (double) existFeatureCount);
-            featuresScore.put("PREDICTED", (double) result);
-            featuresScore.put("TOTAL", (double) transformedResult);
-        }
-
         LOGGER.debug("[calc_dwelltime]features: " + Arrays.toString(features.entrySet().toArray()) + ", prediction:" + result + " , transformed:" + transformedResult);
 
         return transformedResult;