offline write samples and to redis

recommend-server-service/pom.xml

@@ -43,6 +43,12 @@
             <artifactId>aliyun-sdk-oss</artifactId>
             <version>3.15.1</version>
         </dependency>
+        <dependency>
+            <groupId>com.aliyun.odps</groupId>
+            <artifactId>odps-sdk-core</artifactId>
+            <version>0.45.6-public</version>
+        </dependency>
+
 <!--        <dependency>-->
 <!--            <groupId>org.apache.hadoop</groupId>-->
 <!--            <artifactId>hadoop-hdfs</artifactId>-->

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/Application.java

@@ -1,6 +1,8 @@
 package com.tzld.piaoquan.recommend.server;
 
+// import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
 import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
+import com.tzld.piaoquan.recommend.server.dataloader.OfflineSamplesLoader;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
@@ -8,6 +10,7 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.EnableAspectJAutoProxy;
 
+
 /**
  * https://github.com/grpc-swagger/grpc-swagger/blob/master/README_CN.md
  */
@@ -24,7 +27,7 @@ import org.springframework.context.annotation.EnableAspectJAutoProxy;
 public class Application {
     public static void main(String[] args) {
         SpringApplication.run(Application.class, args);
-
+        OfflineSamplesLoader.mutiplyParser("user_video_features_data_final", "20231203");
     }
 
 
@@ -32,5 +35,7 @@ public class Application {
     public FeatureClient featureClient() {
         return new FeatureClient();
     }
+
+
 }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/ItemFeature.java

@@ -26,12 +26,12 @@ public class ItemFeature {
 
 
     // 当天统计量信息
-    private UserActionFeature item_day1_cnt_features;
+    private UserActionFeature day1_cnt_features;
     // 3天内统计量
-    private UserActionFeature item_day3_cnt_features;
+    private UserActionFeature day3_cnt_features;
     // 7天内统计量
-    private UserActionFeature item_day7_cnt_features;
+    private UserActionFeature day7_cnt_features;
     // 3个月统计量
-    private UserActionFeature item_month3_cnt_features;
+    private UserActionFeature month3_cnt_features;
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -8,17 +8,15 @@ import java.util.Map;
 
 @Data
 public class RankItem implements Comparable<RankItem> {
-    public long videoId;
+    public long videoid;
     private double score; // 记录最终的score
-    private Video video;
 
     // 记录Item侧用到的特征
     private ItemFeature itemFeature;
 
     public RankItem(Video video) {
-        this.videoId = video.getVideoId();
-        this.score = 0.0;
-        this.video = video;
+        this.videoid = video.getVideoId() ;
+        this.score = 0.0 ;
     }
 
     private Map<String, Double> rankerScore;
@@ -40,4 +38,6 @@ public class RankItem implements Comparable<RankItem> {
     }
 
 
+
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/UserActionFeature.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.recommend.server.common.base;
 import lombok.Data;
 
 import java.util.Map;
+import java.lang.Math;
 
 @Data
 public class UserActionFeature {
@@ -14,4 +15,63 @@ public class UserActionFeature {
     private double ctr;
     private double str;
     private double rov;
+    private double ros;
+
+    private double ceilLog(Double key) {
+        return Math.ceil(Math.log(key));
+    }
+
+    private double bucketRatioFeature(Double key) {
+        long bucket = Math.round(Math.log(key * 100));
+        if( bucket > 100)
+            bucket = 100;
+        return (double) bucket;
+    }
+
+
+    public void setExp_cnt(Object key){
+        String formateKey = key.toString().replace("\\N", "-1");
+        this.exp_cnt = ceilLog(Double.valueOf(formateKey));
+    }
+
+    public void setClick_cnt(Object key){
+        String formateKey = key.toString().replace("\\N", "-1");
+        this.click_cnt = ceilLog(Double.valueOf(formateKey));
+    }
+    public void setShare_cnt(Object key){
+        String formateKey = key.toString().replace("\\N", "-1");
+        this.share_cnt = ceilLog(Double.valueOf(formateKey));
+    }
+    public void setReturn_cnt(Object key){
+        String formateKey = key.toString().replace("\\N", "-1");
+        this.return_cnt = ceilLog(Double.valueOf(formateKey));
+    }
+
+    public void setCtr(Object key){
+        String formateKey = key.toString().replace("\\N", "-1");
+        this.ctr = bucketRatioFeature(Double.valueOf(formateKey));
+    }
+    public void setStr(Object key){
+        String formateKey = key.toString().replace("\\N", "-1");
+        this.str = bucketRatioFeature(Double.valueOf(formateKey));
+    }
+    public void setRov(Object key){
+        String formateKey = key.toString().replace("\\N", "-1");
+        this.rov = bucketRatioFeature(Double.valueOf(formateKey));
+    }
+
+    public void setRos(Object key){
+        String formateKey = key.toString().replace("\\N", "-1");
+        this.ros = bucketRatioFeature(Double.valueOf(formateKey));
+    }
+
+
+
+
+
+
+
+
+
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/UserBytesFeature.java

@@ -2,12 +2,12 @@ package com.tzld.piaoquan.recommend.server.common.base;
 import io.lettuce.core.StrAlgoArgs;
 import lombok.Data;
 
+import java.util.HashMap;
 import java.util.Map;
 
 @Data
 public class UserBytesFeature {
 
-    private final byte[]  mid;
     private final byte[]  uid;
 
     // 当天统计量信息
@@ -24,50 +24,61 @@ public class UserBytesFeature {
     private final byte[]  user_share_bucket_30days;
 
 
-    public UserBytesFeature(UserFeature userFeature) {
-        mid = userFeature.getMid().getBytes();
-        uid = userFeature.getUid().getBytes();
-        user_cycle_bucket_7days = userFeature.getUser_cycle_bucket_7days().getBytes();
-        user_cycle_bucket_30days = userFeature.getUser_cycle_bucket_30days().getBytes();
-        user_share_bucket_30days = userFeature.getUser_share_bucket_30days().getBytes();
-
+    public UserBytesFeature(UserFeature feature) {
+        this.uid = feature.getUid().getBytes();
+        this.user_cycle_bucket_7days = feature.getUser_cycle_bucket_7days().getBytes();
+        this.user_cycle_bucket_30days = feature.getUser_cycle_bucket_30days().getBytes();
+        this.user_share_bucket_30days = feature.getUser_share_bucket_30days().getBytes();
 
+        this.day1_cnt_features = new HashMap<String, byte[]>();
         // 1 day statistic
-        day1_cnt_features.put("exp", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("click", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("share", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("return", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("ctr", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("str", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("rov", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
+        this.day1_cnt_features.put("exp", String.valueOf(feature.getDay1_cnt_features().getExp_cnt()).getBytes());
+        this.day1_cnt_features.put("click", String.valueOf(feature.getDay1_cnt_features().getClick_cnt()).getBytes());
+        this.day1_cnt_features.put("share", String.valueOf(feature.getDay1_cnt_features().getShare_cnt()).getBytes());
+        this.day1_cnt_features.put("return", String.valueOf(feature.getDay1_cnt_features().getReturn_cnt()).getBytes());
+        this.day1_cnt_features.put("ctr", String.valueOf(feature.getDay1_cnt_features().getCtr()).getBytes());
+        this.day1_cnt_features.put("str", String.valueOf(feature.getDay1_cnt_features().getStr()).getBytes());
+        this.day1_cnt_features.put("rov", String.valueOf(feature.getDay1_cnt_features().getRov()).getBytes());
+        this.day1_cnt_features.put("ros", String.valueOf(feature.getDay1_cnt_features().getRos()).getBytes());
+
+
 
         // 3 day statistic
-        day1_cnt_features.put("exp", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("click", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("share", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("return", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("ctr", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("str", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("rov", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
+        this.day3_cnt_features = new HashMap<String, byte[]>();
+        day3_cnt_features.put("exp", String.valueOf(feature.getDay3_cnt_features().getExp_cnt()).getBytes());
+        day3_cnt_features.put("click", String.valueOf(feature.getDay3_cnt_features().getClick_cnt()).getBytes());
+        day3_cnt_features.put("share", String.valueOf(feature.getDay3_cnt_features().getShare_cnt()).getBytes());
+        day3_cnt_features.put("return", String.valueOf(feature.getDay3_cnt_features().getReturn_cnt()).getBytes());
+        day3_cnt_features.put("ctr", String.valueOf(feature.getDay3_cnt_features().getCtr()).getBytes());
+        day3_cnt_features.put("str", String.valueOf(feature.getDay3_cnt_features().getStr()).getBytes());
+        day3_cnt_features.put("rov", String.valueOf(feature.getDay3_cnt_features().getRov()).getBytes());
+        day3_cnt_features.put("ros", String.valueOf(feature.getDay3_cnt_features().getRos()).getBytes());
+
 
         // 7 day statistic
-        day1_cnt_features.put("exp", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("click", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("share", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("return", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("ctr", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("str", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("rov", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
+        this.day7_cnt_features = new HashMap<String, byte[]>();
+        day7_cnt_features.put("exp", String.valueOf(feature.getDay7_cnt_features().getExp_cnt()).getBytes());
+        day7_cnt_features.put("click", String.valueOf(feature.getDay7_cnt_features().getClick_cnt()).getBytes());
+        day7_cnt_features.put("share", String.valueOf(feature.getDay7_cnt_features().getShare_cnt()).getBytes());
+        day7_cnt_features.put("return", String.valueOf(feature.getDay7_cnt_features().getReturn_cnt()).getBytes());
+        day7_cnt_features.put("ctr", String.valueOf(feature.getDay7_cnt_features().getCtr()).getBytes());
+        day7_cnt_features.put("str", String.valueOf(feature.getDay7_cnt_features().getStr()).getBytes());
+        day7_cnt_features.put("rov", String.valueOf(feature.getDay7_cnt_features().getRov()).getBytes());
+        day7_cnt_features.put("ros", String.valueOf(feature.getDay7_cnt_features().getRos()).getBytes());
+
 
 
         // 3 month statisic
-        day1_cnt_features.put("exp", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("click", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("share", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("return", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("ctr", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("str", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
-        day1_cnt_features.put("rov", String.valueOf(userFeature.getDay1_cnt_features().getExp_cnt()).getBytes());
+        this.month3_cnt_features = new HashMap<String, byte[]>();
+        month3_cnt_features.put("exp", String.valueOf(feature.getMonth3_cnt_features().getExp_cnt()).getBytes());
+        month3_cnt_features.put("click", String.valueOf(feature.getMonth3_cnt_features().getClick_cnt()).getBytes());
+        month3_cnt_features.put("share", String.valueOf(feature.getMonth3_cnt_features().getShare_cnt()).getBytes());
+        month3_cnt_features.put("return", String.valueOf(feature.getMonth3_cnt_features().getReturn_cnt()).getBytes());
+        month3_cnt_features.put("ctr", String.valueOf(feature.getMonth3_cnt_features().getCtr()).getBytes());
+        month3_cnt_features.put("str", String.valueOf(feature.getMonth3_cnt_features().getStr()).getBytes());
+        month3_cnt_features.put("rov", String.valueOf(feature.getMonth3_cnt_features().getRov()).getBytes());
+        month3_cnt_features.put("ros", String.valueOf(feature.getMonth3_cnt_features().getRos()).getBytes());
+
 
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/UserFeature.java

@@ -6,9 +6,7 @@ import java.util.Map;
 
 @Data
 public class UserFeature {
-    private String mid;
     private String uid;
-
     // 当天统计量信息
     private UserActionFeature day1_cnt_features;
     // 3天内统计量
@@ -22,4 +20,10 @@ public class UserFeature {
     private String user_cycle_bucket_30days;
     private String user_share_bucket_30days;
 
+
+    public String parsetoRedisString() {
+
+        return "";
+    }
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/VideoBytesFeature.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.recommend.server.common.base;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import lombok.Data;
 
+import java.util.HashMap;
 import java.util.Map;
 @Data
 public class VideoBytesFeature {
@@ -10,10 +11,6 @@ public class VideoBytesFeature {
 
     private final byte[] upId;
 
-    private final byte[] tags;
-
-    private final byte[] title;
-
     private final byte[] titleLength;
 
     private final byte[] playLength;
@@ -32,61 +29,65 @@ public class VideoBytesFeature {
     // 3个月统计量
     private Map<String, byte[]> item_month3_cnt_features;
 
-    public VideoBytesFeature(ItemFeature itemFeature) {
-        videoId  = itemFeature.getVideoId().getBytes();
-        upId  = itemFeature.getVideoId().getBytes();
+    public VideoBytesFeature(ItemFeature feature) {
+        videoId  = feature.getVideoId().getBytes();
+        upId  = feature.getUpId().getBytes();
 
-        tags  = itemFeature.getTags().getBytes();
-        title  = itemFeature.getTitle().getBytes();
-        titleLength  = itemFeature.getTitleLength().getBytes();
-        playLength  = itemFeature.getPlayLength().getBytes();
-        totolTime  = itemFeature.getTotolTime().getBytes();
-        daysSinceUpload  = itemFeature.getDaysSinceUpload().getBytes();
+        titleLength  = feature.getTitleLength().getBytes();
+        playLength  = feature.getPlayLength().getBytes();
+        totolTime  = feature.getTotalTime().getBytes();
+        daysSinceUpload  = feature.getDaysSinceUpload().getBytes();
         // 1day
-        item_day1_cnt_features.put("exp", String.valueOf(itemFeature.getItem_day1_cnt_features().getExp_cnt()).getBytes());
-        item_day1_cnt_features.put("click", String.valueOf(itemFeature.getItem_day1_cnt_features().getExp_cnt()).getBytes());
-        item_day1_cnt_features.put("share", String.valueOf(itemFeature.getItem_day1_cnt_features().getExp_cnt()).getBytes());
-        item_day1_cnt_features.put("return", String.valueOf(itemFeature.getItem_day1_cnt_features().getExp_cnt()).getBytes());
-        item_day1_cnt_features.put("ctr", String.valueOf(itemFeature.getItem_day1_cnt_features().getExp_cnt()).getBytes());
-        item_day1_cnt_features.put("str", String.valueOf(itemFeature.getItem_day1_cnt_features().getExp_cnt()).getBytes());
-        item_day1_cnt_features.put("rov", String.valueOf(itemFeature.getItem_day1_cnt_features().getExp_cnt()).getBytes());
-
-        // 3day
-        item_day3_cnt_features.put("exp", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("click", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("share", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("return", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("ctr", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("str", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("rov", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-
-
-        // 7day
-        item_day3_cnt_features.put("exp", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("click", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("share", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("return", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("ctr", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("str", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_day3_cnt_features.put("rov", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-
-
-
-        // 3month
-        item_month3_cnt_features.put("exp", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_month3_cnt_features.put("click", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_month3_cnt_features.put("share", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_month3_cnt_features.put("return", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_month3_cnt_features.put("ctr", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_month3_cnt_features.put("str", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-        item_month3_cnt_features.put("rov", String.valueOf(itemFeature.getItem_day3_cnt_features().getExp_cnt()).getBytes());
-
-
-
-
-
 
 
+        // 1 day statistic
+        item_day1_cnt_features = new HashMap<String, byte[]>();
+        item_day1_cnt_features.put("exp", String.valueOf(feature.getDay1_cnt_features().getExp_cnt()).getBytes());
+        item_day1_cnt_features.put("click", String.valueOf(feature.getDay1_cnt_features().getClick_cnt()).getBytes());
+        item_day1_cnt_features.put("share", String.valueOf(feature.getDay1_cnt_features().getShare_cnt()).getBytes());
+        item_day1_cnt_features.put("return", String.valueOf(feature.getDay1_cnt_features().getReturn_cnt()).getBytes());
+        item_day1_cnt_features.put("ctr", String.valueOf(feature.getDay1_cnt_features().getCtr()).getBytes());
+        item_day1_cnt_features.put("str", String.valueOf(feature.getDay1_cnt_features().getStr()).getBytes());
+        item_day1_cnt_features.put("rov", String.valueOf(feature.getDay1_cnt_features().getRov()).getBytes());
+        item_day1_cnt_features.put("ros", String.valueOf(feature.getDay1_cnt_features().getRos()).getBytes());
+
+
+
+        // 3 day statistic
+        item_day3_cnt_features = new HashMap<String, byte[]>();
+        item_day3_cnt_features.put("exp", String.valueOf(feature.getDay3_cnt_features().getExp_cnt()).getBytes());
+        item_day3_cnt_features.put("click", String.valueOf(feature.getDay3_cnt_features().getClick_cnt()).getBytes());
+        item_day3_cnt_features.put("share", String.valueOf(feature.getDay3_cnt_features().getShare_cnt()).getBytes());
+        item_day3_cnt_features.put("return", String.valueOf(feature.getDay3_cnt_features().getReturn_cnt()).getBytes());
+        item_day3_cnt_features.put("ctr", String.valueOf(feature.getDay3_cnt_features().getCtr()).getBytes());
+        item_day3_cnt_features.put("str", String.valueOf(feature.getDay3_cnt_features().getStr()).getBytes());
+        item_day3_cnt_features.put("rov", String.valueOf(feature.getDay3_cnt_features().getRov()).getBytes());
+        item_day3_cnt_features.put("ros", String.valueOf(feature.getDay3_cnt_features().getRos()).getBytes());
+
+
+        // 7 day statistic
+        item_day7_cnt_features = new HashMap<String, byte[]>();
+        item_day7_cnt_features.put("exp", String.valueOf(feature.getDay7_cnt_features().getExp_cnt()).getBytes());
+        item_day7_cnt_features.put("click", String.valueOf(feature.getDay7_cnt_features().getClick_cnt()).getBytes());
+        item_day7_cnt_features.put("share", String.valueOf(feature.getDay7_cnt_features().getShare_cnt()).getBytes());
+        item_day7_cnt_features.put("return", String.valueOf(feature.getDay7_cnt_features().getReturn_cnt()).getBytes());
+        item_day7_cnt_features.put("ctr", String.valueOf(feature.getDay7_cnt_features().getCtr()).getBytes());
+        item_day7_cnt_features.put("str", String.valueOf(feature.getDay7_cnt_features().getStr()).getBytes());
+        item_day7_cnt_features.put("rov", String.valueOf(feature.getDay7_cnt_features().getRov()).getBytes());
+        item_day7_cnt_features.put("ros", String.valueOf(feature.getDay7_cnt_features().getRos()).getBytes());
+
+
+
+        // 3 month statisic
+        item_month3_cnt_features = new HashMap<String, byte[]>();
+        item_month3_cnt_features.put("exp", String.valueOf(feature.getMonth3_cnt_features().getExp_cnt()).getBytes());
+        item_month3_cnt_features.put("click", String.valueOf(feature.getMonth3_cnt_features().getClick_cnt()).getBytes());
+        item_month3_cnt_features.put("share", String.valueOf(feature.getMonth3_cnt_features().getShare_cnt()).getBytes());
+        item_month3_cnt_features.put("return", String.valueOf(feature.getMonth3_cnt_features().getReturn_cnt()).getBytes());
+        item_month3_cnt_features.put("ctr", String.valueOf(feature.getMonth3_cnt_features().getCtr()).getBytes());
+        item_month3_cnt_features.put("str", String.valueOf(feature.getMonth3_cnt_features().getStr()).getBytes());
+        item_month3_cnt_features.put("rov", String.valueOf(feature.getMonth3_cnt_features().getRov()).getBytes());
+        item_month3_cnt_features.put("ros", String.valueOf(feature.getMonth3_cnt_features().getRos()).getBytes());
 
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/enums/VlogFeatureGroup.java

@@ -13,7 +13,6 @@ public enum VlogFeatureGroup {
     MACHINEINFO_SDKVERSION,
     MACHINEINFO_SYSTEM,
     MACHINEINFO_WECHATVERSION,
-
     UP_ID,
     TITLE_LEN,
     PLAY_LEN,
@@ -22,6 +21,82 @@ public enum VlogFeatureGroup {
     DAY,
     WEEK,
     HOUR,
+    REGION,
+    CITY,
+
+    USER_1DAY_EXP,
+    USER_1DAY_CLICK,
+    USER_1DAY_SHARE,
+    USER_1DAY_RETURN,
+    USER_1DAY_CTR,
+    USER_1DAY_STR,
+    USER_1DAY_ROV,
+    USER_1DAY_ROS,
+
+    USER_3DAY_EXP,
+    USER_3DAY_CLICK,
+    USER_3DAY_SHARE,
+    USER_3DAY_RETURN,
+    USER_3DAY_CTR,
+    USER_3DAY_STR,
+    USER_3DAY_ROV,
+    USER_3DAY_ROS,
+
+    USER_7DAY_EXP,
+    USER_7DAY_CLICK,
+    USER_7DAY_SHARE,
+    USER_7DAY_RETURN,
+    USER_7DAY_CTR,
+    USER_7DAY_STR,
+    USER_7DAY_ROV,
+    USER_7DAY_ROS,
+
+    USER_3MONTH_EXP,
+    USER_3MONTH_CLICK,
+    USER_3MONTH_SHARE,
+    USER_3MONTH_RETURN,
+    USER_3MONTH_CTR,
+    USER_3MONTH_STR,
+    USER_3MONTH_ROV,
+    USER_3MONTH_ROS,
+
+
+    ITEM_1DAY_EXP,
+    ITEM_1DAY_CLICK,
+    ITEM_1DAY_SHARE,
+    ITEM_1DAY_RETURN,
+    ITEM_1DAY_CTR,
+    ITEM_1DAY_STR,
+    ITEM_1DAY_ROV,
+    ITEM_1DAY_ROS,
+
+    ITEM_3DAY_EXP,
+    ITEM_3DAY_CLICK,
+    ITEM_3DAY_SHARE,
+    ITEM_3DAY_RETURN,
+    ITEM_3DAY_CTR,
+    ITEM_3DAY_STR,
+    ITEM_3DAY_ROV,
+    ITEM_3DAY_ROS,
+
+    ITEM_7DAY_EXP,
+    ITEM_7DAY_CLICK,
+    ITEM_7DAY_SHARE,
+    ITEM_7DAY_RETURN,
+    ITEM_7DAY_CTR,
+    ITEM_7DAY_STR,
+    ITEM_7DAY_ROV,
+    ITEM_7DAY_ROS,
+
+    ITEM_3MONTH_EXP,
+    ITEM_3MONTH_CLICK,
+    ITEM_3MONTH_SHARE,
+    ITEM_3MONTH_RETURN,
+    ITEM_3MONTH_CTR,
+    ITEM_3MONTH_STR,
+    ITEM_3MONTH_ROV,
+    ITEM_3MONTH_ROS,
+
 
     USER_CYCLE_BUCKET_7DAY,
     USER_CYCLE_BUCKET_30DAY,

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/dataloader/FeatureConstructor.java

@@ -0,0 +1,188 @@
+package com.tzld.piaoquan.recommend.server.dataloader;
+
+
+import com.fasterxml.jackson.databind.ser.Serializers;
+import com.google.common.collect.ListMultimap;
+import com.tzld.piaoquan.recommend.server.common.base.*;
+import com.tzld.piaoquan.recommend.server.common.enums.VlogFeatureGroup;
+import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
+import com.tzld.piaoquan.recommend.server.gen.recommend.FeatureGroup;
+import com.tzld.piaoquan.recommend.server.gen.recommend.GroupedFeature;
+import com.tzld.piaoquan.recommend.server.service.score.feature.VlogShareLRFeatureExtractor;
+import com.aliyun.odps.Instance;
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.OdpsException;
+import com.aliyun.odps.account.Account;
+import com.aliyun.odps.account.AliyunAccount;
+import com.aliyun.odps.data.Record;
+import com.aliyun.odps.task.SQLTask;
+import java.util.*;
+
+public class FeatureConstructor {
+
+    private static final String BUCKET_NAME = "ali-recommend";
+    private static  final Map<String, String> ODPS_CONFIG =  new HashMap<String, String>();
+    static {
+        ODPS_CONFIG.put("ENDPOINT", "http://service.cn.maxcompute.aliyun.com/api");
+        ODPS_CONFIG.put("ACCESSID", "LTAIWYUujJAm7CbH");
+        ODPS_CONFIG.put("ACCESSKEY", "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P");
+    };
+
+    public static List<Record> loadStreamDataFromOSS(String table, String dt) {
+        Account account = new AliyunAccount(ODPS_CONFIG.get("ACCESSID"), ODPS_CONFIG.get("ACCESSKEY"));
+        Odps odps = new Odps(account);
+        odps.setEndpoint(ODPS_CONFIG.get("ENDPOINT"));
+        odps.setDefaultProject("loghubods");
+        String sql = String.format("select * from %s where dt ='%s' limit 100;", table, dt);
+        Instance instance;
+        List<Record> records = new ArrayList<Record>();
+        try {
+            instance = SQLTask.run(odps, sql);
+            instance.waitForSuccess();
+            records = SQLTask.getResult(instance);
+        } catch (OdpsException e) {
+            e.printStackTrace();
+        }
+        return records;
+    }
+
+
+
+    public static RequestContext constructRequestContext(Record record) {
+        RequestContext requestContext = new RequestContext();
+        requestContext.setApptype(record.get("apptype").toString());
+        requestContext.setMachineinfo_brand(record.get("machineinfo_brand").toString());
+        requestContext.setMachineinfo_model(record.get("machineinfo_model").toString());
+        requestContext.setMachineinfo_platform(record.get("machineinfo_platform").toString());
+        requestContext.setMachineinfo_sdkversion(record.get("machineinfo_sdkversion").toString());
+        requestContext.setMachineinfo_system(record.get("machineinfo_system").toString());
+        requestContext.setMachineinfo_wechatversion(record.get("machineinfo_wechatversion").toString());
+        requestContext.setDay(record.get("ctx_day").toString());
+        requestContext.setWeek(record.get("ctx_week").toString());
+        requestContext.setHour(record.get("ctx_hour").toString());
+        requestContext.setRegion(record.get("ctx_region").toString());
+        requestContext.setCity(record.get("ctx_city").toString());
+        return requestContext;
+    }
+
+
+
+    public static UserFeature constructUserFeature(Record record) {
+        UserFeature userFeature = new UserFeature();
+        userFeature.setUid(record.getString("uid"));
+        userFeature.setUser_cycle_bucket_7days(record.getString("u_cycle_bucket_7days"));
+        userFeature.setUser_cycle_bucket_30days(record.getString("u_cycle_bucket_30days"));
+        userFeature.setUser_share_bucket_30days(record.getString("u_share_bucket_30days"));
+
+
+        // 1day features
+        UserActionFeature user1dayActionFeature = new UserActionFeature();
+        user1dayActionFeature.setExp_cnt(record.getString("u_1day_exp_cnt"));
+        user1dayActionFeature.setClick_cnt(record.getString("u_1day_click_cnt"));
+        user1dayActionFeature.setShare_cnt(record.getString("u_1day_share_cnt"));
+        user1dayActionFeature.setReturn_cnt(record.getString("u_1day_return_cnt"));
+        user1dayActionFeature.setCtr(record.getString("u_ctr_1day"));
+        user1dayActionFeature.setStr(record.getString("u_str_1day"));
+        user1dayActionFeature.setRov(record.getString("u_rov_1day"));
+        user1dayActionFeature.setRos(record.getString("u_ros_1day"));
+        userFeature.setDay1_cnt_features(user1dayActionFeature);
+
+        // 3day features
+        UserActionFeature user3dayActionFeature = new UserActionFeature();
+        user3dayActionFeature.setExp_cnt(record.getString("u_3day_exp_cnt"));
+        user3dayActionFeature.setClick_cnt(record.getString("u_3day_click_cnt"));
+        user3dayActionFeature.setShare_cnt(record.getString("u_3day_share_cnt"));
+        user3dayActionFeature.setReturn_cnt(record.getString("u_3day_return_cnt"));
+        user3dayActionFeature.setCtr(record.getString("u_ctr_3day"));
+        user3dayActionFeature.setStr(record.getString("u_str_3day"));
+        user3dayActionFeature.setRov(record.getString("u_rov_3day"));
+        user3dayActionFeature.setRos(record.getString("u_ros_3day"));
+        userFeature.setDay3_cnt_features(user3dayActionFeature);
+
+        // 7day features
+        UserActionFeature user7dayActionFeature = new UserActionFeature();
+        user7dayActionFeature.setExp_cnt(record.getString("u_7day_exp_cnt"));
+        user7dayActionFeature.setClick_cnt(record.getString("u_7day_click_cnt"));
+        user7dayActionFeature.setShare_cnt(record.getString("u_7day_share_cnt"));
+        user7dayActionFeature.setReturn_cnt(record.getString("u_7day_return_cnt"));
+        user7dayActionFeature.setCtr(record.getString("u_ctr_7day"));
+        user7dayActionFeature.setStr(record.getString("u_str_7day"));
+        user7dayActionFeature.setRov(record.getString("u_rov_7day"));
+        user7dayActionFeature.setRos(record.getString("u_ros_7day"));
+        userFeature.setDay7_cnt_features(user7dayActionFeature);
+
+        // 3month features
+        UserActionFeature user3monthActionFeature = new UserActionFeature();
+        user3monthActionFeature.setExp_cnt(record.getString("u_3month_exp_cnt"));
+        user3monthActionFeature.setClick_cnt(record.getString("u_3month_click_cnt"));
+        user3monthActionFeature.setShare_cnt(record.getString("u_3month_share_cnt"));
+        user3monthActionFeature.setReturn_cnt(record.getString("u_3month_return_cnt"));
+        user3monthActionFeature.setCtr(record.getString("u_ctr_3month"));
+        user3monthActionFeature.setStr(record.getString("u_str_3month"));
+        user3monthActionFeature.setRov(record.getString("u_rov_3month"));
+        user3monthActionFeature.setRos(record.getString("u_ros_3month"));
+        userFeature.setMonth3_cnt_features(user3monthActionFeature);
+
+        return userFeature;
+    }
+
+
+    public static ItemFeature constructItemFeature(Record record){
+        ItemFeature itemFeature = new ItemFeature();
+        itemFeature.setVideoId(record.get("videoid").toString());
+        itemFeature.setUpId(record.get("i_up_id").toString());
+        itemFeature.setTitleLength(record.get("i_title_len").toString());
+        itemFeature.setPlayLength(record.get("i_play_len").toString());
+        itemFeature.setTotalTime(record.get("total_time").toString());
+        itemFeature.setDaysSinceUpload(record.get("i_days_since_upload").toString());
+
+        UserActionFeature user1dayActionFeature = new UserActionFeature();
+        user1dayActionFeature.setExp_cnt(record.getString("i_1day_exp_cnt"));
+        user1dayActionFeature.setClick_cnt(record.getString("i_1day_click_cnt"));
+        user1dayActionFeature.setShare_cnt(record.getString("i_1day_share_cnt"));
+        user1dayActionFeature.setReturn_cnt(record.getString("i_1day_return_cnt"));
+        user1dayActionFeature.setCtr(record.getString("i_ctr_1day"));
+        user1dayActionFeature.setStr(record.getString("i_str_1day"));
+        user1dayActionFeature.setRov(record.getString("i_rov_1day"));
+        user1dayActionFeature.setRos(record.getString("i_ros_1day"));
+        itemFeature.setDay1_cnt_features(user1dayActionFeature);
+
+        UserActionFeature user3dayActionFeature = new UserActionFeature();
+        user3dayActionFeature.setExp_cnt(record.getString("i_3day_exp_cnt"));
+        user3dayActionFeature.setClick_cnt(record.getString("i_3day_click_cnt"));
+        user3dayActionFeature.setShare_cnt(record.getString("i_3day_share_cnt"));
+        user3dayActionFeature.setReturn_cnt(record.getString("i_3day_return_cnt"));
+        user3dayActionFeature.setCtr(record.getString("i_ctr_3day"));
+        user3dayActionFeature.setStr(record.getString("i_str_3day"));
+        user3dayActionFeature.setRov(record.getString("i_rov_3day"));
+        user3dayActionFeature.setRos(record.getString("i_ros_3day"));
+        itemFeature.setDay3_cnt_features(user1dayActionFeature);
+
+        UserActionFeature user7dayActionFeature = new UserActionFeature();
+        user7dayActionFeature.setExp_cnt(record.getString("i_7day_exp_cnt"));
+        user7dayActionFeature.setClick_cnt(record.getString("i_7day_click_cnt"));
+        user7dayActionFeature.setShare_cnt(record.getString("i_7day_share_cnt"));
+        user7dayActionFeature.setReturn_cnt(record.getString("i_7day_return_cnt"));
+        user7dayActionFeature.setCtr(record.getString("i_ctr_7day"));
+        user7dayActionFeature.setStr(record.getString("i_str_7day"));
+        user7dayActionFeature.setRov(record.getString("i_rov_7day"));
+        user7dayActionFeature.setRos(record.getString("i_ros_7day"));
+        itemFeature.setDay7_cnt_features(user1dayActionFeature);
+
+        UserActionFeature user3monthActionFeature = new UserActionFeature();
+        user3monthActionFeature.setExp_cnt(record.getString("i_3month_exp_cnt"));
+        user3monthActionFeature.setClick_cnt(record.getString("i_3month_click_cnt"));
+        user3monthActionFeature.setShare_cnt(record.getString("i_3month_share_cnt"));
+        user3monthActionFeature.setReturn_cnt(record.getString("i_3month_return_cnt"));
+        user3monthActionFeature.setCtr(record.getString("i_ctr_3month"));
+        user3monthActionFeature.setStr(record.getString("i_str_3month"));
+        user3monthActionFeature.setRov(record.getString("i_rov_3month"));
+        user3monthActionFeature.setRos(record.getString("i_ros_3month"));
+        itemFeature.setMonth3_cnt_features(user3monthActionFeature);
+        return itemFeature;
+    }
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/dataloader/FeatureToRedisLoader.java

@@ -0,0 +1,55 @@
+package com.tzld.piaoquan.recommend.server.dataloader;
+
+import com.aliyun.odps.Instance;
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.OdpsException;
+import com.aliyun.odps.account.Account;
+import com.aliyun.odps.account.AliyunAccount;
+import com.aliyun.odps.data.Record;
+import com.aliyun.odps.task.SQLTask;
+import com.google.common.collect.ListMultimap;
+import com.tzld.piaoquan.recommend.server.common.base.*;
+import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
+import com.tzld.piaoquan.recommend.server.gen.recommend.FeatureGroup;
+import com.tzld.piaoquan.recommend.server.service.score.feature.VlogShareLRFeatureExtractor;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class FeatureToRedisLoader {
+
+
+    public static void loadFeatureToRedis(String userTable, String itemTable, String dt) {
+        // 1. 数据读取
+        List<Record> userOriginData = FeatureConstructor.loadStreamDataFromOSS(userTable, dt);
+        List<Record> itemOriginData = FeatureConstructor.loadStreamDataFromOSS(itemTable, dt);
+
+        // 2. user item初始化
+        for(int i=0; i < userOriginData.size(); i++) {
+            Record userRecord = userOriginData.get(i);
+            UserFeature userFeature = FeatureConstructor.constructUserFeature(userRecord);
+        }
+
+        for(int i=0; i < itemOriginData.size(); i++) {
+            Record itemRecord = itemOriginData.get(i);
+            ItemFeature itemFeature = FeatureConstructor.constructItemFeature(itemRecord);
+        }
+    }
+
+    public static void writetoRedis(UserFeature userFeature) {
+
+
+
+
+    }
+
+
+    public static void main(String[] args) {
+        String Path = "";
+
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/dataloader/OfflineSamplesLoader.java

@@ -0,0 +1,87 @@
+package com.tzld.piaoquan.recommend.server.dataloader;
+
+import com.fasterxml.jackson.databind.ser.Serializers;
+import com.google.common.collect.ListMultimap;
+import com.tzld.piaoquan.recommend.server.common.base.*;
+import com.tzld.piaoquan.recommend.server.common.enums.VlogFeatureGroup;
+import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
+import com.tzld.piaoquan.recommend.server.gen.recommend.FeatureGroup;
+import com.tzld.piaoquan.recommend.server.gen.recommend.GroupedFeature;
+import com.tzld.piaoquan.recommend.server.service.score.feature.VlogShareLRFeatureExtractor;
+import com.aliyun.odps.Instance;
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.OdpsException;
+import com.aliyun.odps.account.Account;
+import com.aliyun.odps.account.AliyunAccount;
+import com.aliyun.odps.data.Record;
+import com.aliyun.odps.task.SQLTask;
+import java.util.*;
+
+
+public class OfflineSamplesLoader {
+
+    private static final String BUCKET_NAME = "ali-recommend";
+    private static  final Map<String, String> ODPS_CONFIG =  new HashMap<String, String>();
+    static {
+        ODPS_CONFIG.put("ENDPOINT", "http://service.cn.maxcompute.aliyun.com/api");
+        ODPS_CONFIG.put("ACCESSID", "LTAIWYUujJAm7CbH");
+        ODPS_CONFIG.put("ACCESSKEY", "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P");
+    };
+
+
+    // 单条日志处理逻辑
+    public static String singleParse(Record record) {
+        // 数据解析
+        String label = record.get("ui_is_out").toString();
+
+        // 从sql的 record中 初始化对象内容
+        RequestContext requestContext = FeatureConstructor.constructRequestContext(record);
+        UserFeature userFeature = FeatureConstructor.constructUserFeature(record);
+        ItemFeature itemFeature =  FeatureConstructor.constructItemFeature(record);
+
+        // 转化成bytes
+        RequestContextBytesFeature requestContextBytesFeature = new RequestContextBytesFeature(requestContext);
+        UserBytesFeature userBytesFeature = new UserBytesFeature(userFeature);
+        VideoBytesFeature videoBytesFeature = new VideoBytesFeature(itemFeature);
+
+        // 特征抽取
+        VlogShareLRFeatureExtractor bytesFeatureExtractor;
+        bytesFeatureExtractor = new VlogShareLRFeatureExtractor();
+
+        bytesFeatureExtractor.getUserFeatures(userBytesFeature);
+        bytesFeatureExtractor.getItemFeature(videoBytesFeature);
+        bytesFeatureExtractor.getContextFeatures(requestContextBytesFeature);
+
+        ListMultimap<FeatureGroup, BaseFeature> featureMap = bytesFeatureExtractor.getFeatures();
+        return parseSamplesToString(label, featureMap);
+    }
+
+    // 构建样本的字符串
+    public static String parseSamplesToString(String label, ListMultimap<FeatureGroup, BaseFeature>  featureMap) {
+        ArrayList<String> featureList = new ArrayList<String>();
+        for (Map.Entry<FeatureGroup, BaseFeature> entry : featureMap.entries()) {
+            FeatureGroup groupedFeature = entry.getKey();
+            BaseFeature baseFeature = entry.getValue();
+            Long featureIdentifier = baseFeature.getIdentifier();
+            featureList.add(String.valueOf(featureIdentifier));
+        }
+        return label + "\t" + String.join("\t", featureList);
+    }
+
+
+    //  主处理逻辑
+    public static void mutiplyParser(String table, String dt) {
+        List<Record> dataFrame = FeatureConstructor.loadStreamDataFromOSS(table, dt);
+        for(int i=0; i< dataFrame.size(); i++){
+            String samples = singleParse(dataFrame.get(i));
+            System.out.println(samples);
+        }
+    }
+
+
+    public static void main(String[] args) {
+        String Path = "";
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/remote/FeatureRemoteService.java

@@ -7,17 +7,10 @@ import com.tzld.piaoquan.recommend.feature.model.feature.VideoFeatureProto;
 import com.tzld.piaoquan.recommend.server.common.base.ItemFeature;
 import com.tzld.piaoquan.recommend.server.common.base.UserActionFeature;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.math.NumberUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
 
 /**
  * @author dyp
@@ -29,7 +22,6 @@ public class FeatureRemoteService {
     @Autowired
     private FeatureClient client;
 
-    // 有性能问题再增加localCache
     public UserFeature getUserFeature(String uid) {
         if (StringUtils.isBlank(uid)) {
             return null;
@@ -43,7 +35,6 @@ public class FeatureRemoteService {
             return null;
         }
         UserFeature feature = new UserFeature();
-        feature.setMid(proto.getMid());
         feature.setUid(proto.getUid());
         if (proto.hasDay1CntFeature()) {
             feature.setDay1_cnt_features(convert(proto.getDay1CntFeature()));
@@ -84,15 +75,6 @@ public class FeatureRemoteService {
         return convert(proto);
     }
 
-    public Map<Long, ItemFeature> getVideoFeatureMap(List<Long> videoIds) {
-        if (CollectionUtils.isEmpty(videoIds)) {
-            return Collections.emptyMap();
-        }
-        List<VideoFeatureProto> protoList = client.getAllVideoFeature(videoIds);
-
-        return CommonCollectionUtils.toMap(protoList, p -> NumberUtils.toLong(p.getVideoId(), 0L), this::convert);
-    }
-
     private ItemFeature convert(VideoFeatureProto proto) {
         if (proto == null) {
             return null;
@@ -106,20 +88,19 @@ public class FeatureRemoteService {
         feature.setVideoId(proto.getVideoId());
         feature.setDaysSinceUpload(proto.getDaysSinceUpload());
 
+
         if (proto.hasVideoDay1CntFeature()) {
-            feature.setItem_day1_cnt_features(convert(proto.getVideoDay1CntFeature()));
+            feature.setDay1_cnt_features(convert(proto.getVideoDay1CntFeature()));
         }
         if (proto.hasVideoDay3CntFeature()) {
-            feature.setItem_day3_cnt_features(convert(proto.getVideoDay3CntFeature()));
+            feature.setDay3_cnt_features(convert(proto.getVideoDay3CntFeature()));
         }
         if (proto.hasVideoDay7CntFeature()) {
-            feature.setItem_day7_cnt_features(convert(proto.getVideoDay7CntFeature()));
+            feature.setDay7_cnt_features(convert(proto.getVideoDay7CntFeature()));
         }
         if (proto.hasVideoMonth3CntFeature()) {
-            feature.setItem_month3_cnt_features(convert(proto.getVideoMonth3CntFeature()));
+            feature.setMonth3_cnt_features(convert(proto.getVideoMonth3CntFeature()));
         }
-
-
         return feature;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -341,8 +341,6 @@ public class RecommendService {
         rankParam.setTopK(param.getTopK());
         rankParam.setRankKeyPrefix(param.getRankKeyPrefix());
         rankParam.setAppType(param.getAppType());
-        rankParam.setUid(param.getUid());
-        rankParam.setCityCode(param.getCityCode());
         return rankParam;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -1,15 +1,17 @@
 package com.tzld.piaoquan.recommend.server.service.rank;
 
 
-import com.tzld.piaoquan.recommend.server.common.base.ItemFeature;
+import com.typesafe.config.Config;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.remote.FeatureRemoteService;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerPipeline;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
@@ -35,7 +37,66 @@ public class RankService {
     @Autowired
     private FeatureRemoteService featureRemoteService;
 
+
+    // LR scores
+    public RankResult rank(RecommendRequest recommendRequest,
+                           RankParam param,
+                           RecallResult recallResult,
+                           UserFeature userFeature,
+                           List<RankItem> videoRankFeatures,
+                           Config config) {
+        if (param == null
+                || param.getRecallResult() == null
+                || CollectionUtils.isEmpty(param.getRecallResult().getData())) {
+            return null;
+        }
+
+        if (param.isSpecialRecommend()) {
+            Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                    .filter(d -> d.getPushFrom().equals(SpecialRecallStrategy.PUSH_FROM))
+                    .findFirst();
+            if (data.isPresent()
+                    && data.get() != null) {
+                return new RankResult(data.get().getVideos());
+            }
+            return null;
+        }
+
+        ScorerPipeline scorerPipeline = ScorerUtils.getScorerPipeline(config);
+        // TODO  merge 后返回待排序的video list
+        List<Video> recallVideos = recallResult.mergeRecallVideos();
+        // TODO 转化成rankitem
+        // convert List<Video> to List<RankItem>
+        List<RankItem> rankItem = new ArrayList<RankItem>();
+
+        List<RankItem> rovRecallRank  =
+                scorerPipeline.scoring(recommendRequest, param, userFeature, rankItem);
+
+        log.info("mergeAndRankRovRecall rovRecallRank={}", JSONUtils.toJson(rovRecallRank));
+        List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
+        log.info("mergeAndRankFlowPoolRecall flowPoolRank={}", JSONUtils.toJson(flowPoolRank));
+
+        //TODO
+        removeDuplicate(param, recallVideos, flowPoolRank);
+        // convert List<RankItem> to List<Video>
+
+
+
+        log.info("removeDuplicate rovRecallRank={}, flowPoolRank={}",
+                JSONUtils.toJson(rovRecallRank),
+                JSONUtils.toJson(flowPoolRank));
+
+        // 融合排序
+        //TODO need to repair
+        return mergeAndSort(param, recallVideos, flowPoolRank);
+
+
+    }
+
+
     public RankResult rank(RankParam param) {
+
+
         if (param == null
                 || param.getRecallResult() == null
                 || CollectionUtils.isEmpty(param.getRecallResult().getData())) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -107,16 +107,14 @@ public class RecallService implements ApplicationContextAware {
                 || param.getAbCode().equals("60088")
                 || param.getAbCode().equals("60089")
                 || param.getAbCode().equals("60090")
-                || param.getAbCode().equals("60091")) {
+                || param.getAbCode().equals("60091")
+                || param.getAbCode().equals("60092")
+                || param.getAbCode().equals("60093")
+                || param.getAbCode().equals("60094")) {
             strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
         } else if (param.getAbCode().equals("60068")
                 || param.getAbCode().equals("60081")
-                || param.getAbCode().equals("60084")
-                || param.getAbCode().equals("60092")
-                || param.getAbCode().equals("60093")
-                || param.getAbCode().equals("60094")
-                || param.getAbCode().equals("60096")
-                || param.getAbCode().equals("60095")) {
+                || param.getAbCode().equals("60084")) {
             strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
             strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/AbstractScorer.java

@@ -63,8 +63,9 @@ public abstract class AbstractScorer {
         return scorerConfigInfo;
     }
 
-    public abstract List<RankItem> scoring(final RankParam param,
-                                           final UserFeature userFeature,
-                                           final List<RankItem> rankItems);
+    public abstract List<RankItem> scoring(final RecommendRequest request,
+                                        final RankParam param,
+                                        final UserFeature userFeature,
+                                        final List<RankItem> rankItems);
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerPipeline.java

@@ -2,7 +2,11 @@ package com.tzld.piaoquan.recommend.server.service.score;
 
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.common.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
@@ -11,7 +15,11 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 
 
 public class ScorerPipeline {
@@ -20,7 +28,7 @@ public class ScorerPipeline {
     public static final Logger LOGGER = LoggerFactory.getLogger(ScorerPipeline.class);
     public static final ExecutorService executorService = Executors.newFixedThreadPool(corePoolSize);
 
-    private List<AbstractScorer> scorers;
+    public List<AbstractScorer> scorers;
 
     public ScorerPipeline(List<AbstractScorer> scorers) {
         this.scorers = scorers;
@@ -28,12 +36,12 @@ public class ScorerPipeline {
 
     /**
      * scoring
-     *
      * @return
      */
-    public List<RankItem> scoring(final RankParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
+    public List<RankItem> scoring(final RecommendRequest recommendRequest,
+                               final RankParam param,
+                               final UserFeature userFeature,
+                               final List<RankItem> rankItems) {
         // check recall is empty?
         if (CollectionUtils.isEmpty(param.getRecallResult().getData())) {
             // log.error
@@ -61,7 +69,7 @@ public class ScorerPipeline {
             Callable<List<RankItem>> callable = new Callable<List<RankItem>>() {
                 @Override
                 public List<RankItem> call() throws Exception {
-                    return scorer.scoring(param, userFeature, scoreRankerItems);
+                    return scorer.scoring(recommendRequest, param, userFeature, scoreRankerItems);
                 }
             };
 
@@ -100,7 +108,7 @@ public class ScorerPipeline {
                 item.getRankerScore().put(scorerName, item.getScore());
             }
 
-            //
+           //
             long spentTime = System.currentTimeMillis() - startTime;
             LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
                     new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareGBDTScorer.java

@@ -29,16 +29,17 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
     }
 
 
-    private RequestContext getRequestContext(RankParam param) {
+    private RequestContext getRequestContext(RecommendRequest request) {
 
         RequestContext requestContext = new RequestContext();
         return requestContext;
     }
 
     @Override
-    public List<RankItem> scoring(final RankParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
+    public List<RankItem> scoring(final RecommendRequest request,
+                               final RankParam param,
+                               final UserFeature userFeature,
+                               final List<RankItem> rankItems) {
         RecallResult recallResult = param.getRecallResult();
         if (recallResult.getData().size() == 0) {
             return rankItems;
@@ -51,10 +52,10 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
             LOGGER.error("not found model");
             return rankItems;
         }
-        RequestContext requestContext = getRequestContext(param);
+        RequestContext requestContext = getRequestContext(request);
 
         // 多Rank的rank打分
-        multipleGBDTScore(rankItems, model, userFeature, requestContext);
+        multipleGBDTScore(rankItems, model, userFeature, request, requestContext);
         Collections.sort(rankItems);
         LOGGER.debug("dwelltime ranker excute time: [{}]", System.currentTimeMillis() - startTime);
         return rankItems;
@@ -70,8 +71,8 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
      * @return
      */
     private double predictMultipleGBDTScore(final double oldScore, final double dwelltimeOrCompletion,
-                                            final double duration, final String itemId,
-                                            final RecommendRequest requestData, final UserFeature user) {
+                                         final double duration, final String itemId,
+                                         final RecommendRequest requestData, final UserFeature user) {
         double ctrScalePower = 0;
         double dwelltimeScalePower = 0;
 
@@ -81,7 +82,7 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
         }
         double newScore = Math.pow(ctrScore, ctrScalePower) * Math.pow(dwelltimeOrCompletion, dwelltimeScalePower);
         //值越大压制越弱
-        double alpha = 1.0;
+        double alpha =  1.0;
         double suppressRatio = (1 + alpha) / (Math.max(duration, 300.0) / 300 + alpha);
         newScore = newScore * suppressRatio;
         LOGGER.debug("Expected dwell time score [{}]: {} ,{}, {}--> {}", new Object[]{itemId, oldScore, dwelltimeOrCompletion, duration, newScore});
@@ -89,11 +90,11 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
     }
 
     private void gbdtBaseScore(final RankItem item,
-                               final GBDTModel model,
-                               final RequestContext requestContext,
-                               final Map<String, Double> userFeatures,
-                               final Map<String, Double> contextFeatures,
-                               final UserFeature user) {
+                                final GBDTModel model,
+                                final RequestContext requestContext,
+                                final Map<String, Double> userFeatures,
+                                final Map<String, Double> contextFeatures,
+                                final RecommendRequest requestData, final UserFeature user) {
         try {
 
             Map<String, Double> articleFeatures;
@@ -120,9 +121,10 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
 
 
     public void multipleGBDTScore(final List<RankItem> items,
-                                  final GBDTModel model,
-                                  final UserFeature user,
-                                  final RequestContext requestContext) {
+                                       final GBDTModel model,
+                                       final UserFeature user,
+                                       final RecommendRequest requestData,
+                                       final RequestContext requestContext) {
         final int size = items.size();
         if (size == 0) {
             return;
@@ -144,7 +146,7 @@ public class VlogShareGBDTScorer extends BaseGBDTModelScorer {
                               @Override
                               public Object call() throws Exception {
                                   try {
-                                      gbdtBaseScore(items.get(fIndex), model, requestContext, userFeatures, contextFeatures, user);
+                                      gbdtBaseScore(items.get(fIndex), model, requestContext, userFeatures, contextFeatures, requestData, user);
                                   } catch (Exception e) {
                                       LOGGER.error("dwelltime exception: [{}] [{}]", items.get(fIndex), ExceptionUtils.getFullStackTrace(e));
                                   }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareLRScorer.java

@@ -1,10 +1,17 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
 
+
 import com.tzld.piaoquan.recommend.server.common.base.*;
+import com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples;
 import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.score.feature.FeatureUsage;
+import com.tzld.piaoquan.recommend.server.service.score.feature.LRBytesFeatureExtractorBase;
 import com.tzld.piaoquan.recommend.server.service.score.feature.VlogShareLRFeatureExtractor;
 import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
 import org.apache.commons.lang.exception.ExceptionUtils;
@@ -17,14 +24,13 @@ import java.util.List;
 import java.util.concurrent.*;
 
 
-public class VlogShareLRScorer extends BaseLRModelScorer {
+public class VlogShareLRScorer extends BaseLRModelScorer{
 
     private final static int CORE_POOL_SIZE = 64;
 
-    private static final int LOCAL_TIME_OUT = 150;
+    private static final int LOCAL_TIME_OUT= 150;
     private final static Logger LOGGER = LoggerFactory.getLogger(VlogShareLRScorer.class);
     private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
-    private static final FeatureUsage featureUsage = new FeatureUsage();
     private static final double defaultUserCtrGroupNumber = 10.0;
     private static final int enterFeedsScoreRatio = 10;
     private static final int enterFeedsScoreNum = 20;
@@ -36,9 +42,10 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
 
 
     @Override
-    public List<RankItem> scoring(final RankParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
+    public List<RankItem> scoring(final RecommendRequest request,
+                               final RankParam param,
+                               final UserFeature userFeature,
+                               final List<RankItem> rankItems) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
@@ -47,7 +54,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
         }
 
         List<RankItem> result = rankItems;
-        result = rankByJava(rankItems, param, userFeature);
+        result = rankByJava(rankItems, request, userFeature);
 
         LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
                 System.currentTimeMillis() - startTime);
@@ -56,17 +63,16 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
     }
 
     private List<RankItem> rankByJava(final List<RankItem> items,
-                                      final RankParam param,
-                                      final UserFeature user) {
+                                        final RecommendRequest request,
+                                        final UserFeature user) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
 
         // init request context
-        // TODO  应该用哪个？ city or province or region
-        String cityCode = param.getCityCode();
+        String cityCode = request.getCityCode();
         RequestContext requestContext = new RequestContext();
-        requestContext.setCity(param.getCityCode());
+        requestContext.setCity(request.getCityCode());
 
         // userBytes
         UserBytesFeature userInfoBytes = null;
@@ -101,16 +107,15 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
 
         LRSamples lrSamples = null;
         VlogShareLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new VlogShareLRFeatureExtractor(featureUsage, 100.0, 1);
+        bytesFeatureExtractor = new VlogShareLRFeatureExtractor();
 
         try {
             VideoBytesFeature newsInfoBytes = new VideoBytesFeature(item.getItemFeature());
             lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
                     new RequestContextBytesFeature(requestContext));
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]",
-                    new Object[]{new String(userInfoBytes.getUid()), item.getVideoId(),
-                            ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getUid()), item.getVideoid(),
+                    ExceptionUtils.getFullStackTrace(e)});
         }
 
 
@@ -120,24 +125,19 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
                 LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                        item.getVideoid(), ExceptionUtils.getFullStackTrace(e)});
             }
-
-
-//            CtrSamples.Builder samples =  CtrSamples.newBuilder();
-//            samples.setLr_samples(lrSamples);
-//            item.setSamples(samples);
+            // 增加实时特征后打开在线存储日志逻辑
+            //
+            // CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
+            // samples.setLr_samples(lrSamples);
+            // item.setSamples(samples);
+            //
         }
-
         item.setScore(pro);
-        // item.setRecScore(pro);
         return pro;
     }
 
-    /**
-     * 并行打分
-     *
-     */
 
     /**
      * 并行打分
@@ -162,8 +162,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                     try {
                         calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
                     } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId,
-                                ExceptionUtils.getFullStackTrace(e));
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoid, ExceptionUtils.getFullStackTrace(e));
                     }
                     return new Object();
                 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/feature/BytesUtils.java

@@ -35,18 +35,16 @@ public class BytesUtils {
         }
     }
 
-    public BytesUtils(BytesGroup[] groups, FeatureUsage usage) {
+    public BytesUtils(BytesGroup[] groups) {
         this.groups = groups;
         for (BytesGroup g : groups) {
-            byte[] buffer = prepareBuffer(g.getName(), g.getNameBytes(), usage);
+            byte[] buffer = prepareBuffer(g.getName(), g.getNameBytes());
             groups[g.getId()].setBuffer(buffer);
         }
     }
 
-    public byte[] prepareBuffer(String name, byte[] nameBytes, FeatureUsage usage) {
-        if (usage.disable(name)) {
-            return null;
-        }
+    public byte[] prepareBuffer(String name, byte[] nameBytes) {
+
         byte[] buffer = new byte[MAX_FEATURE_BYTES_LENGTH];
         System.arraycopy(nameBytes, 0, buffer, 0, nameBytes.length);
         System.arraycopy(FEATURE_SEPARATOR, 0, buffer, nameBytes.length, 1);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/feature/GBDTFeatureExtractorBase.java

@@ -21,11 +21,10 @@ public class GBDTFeatureExtractorBase {
     public static Map<String, Double> extractVideoFeatures(final RecallResult recallResult) {
         Map<String, Double> features = new HashMap<String, Double>();
         // need to add which features to extract
-
         return features;
     }
 
-
+    // TODO
     public static Map<String, Double> extractRequestContextFeatures(final RequestContext requestContext) {
         Map<String, Double> features = new HashMap<String, Double>();
         // need to add which features to extract

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/feature/LRBytesFeatureExtractorBase.java

@@ -17,30 +17,21 @@ public abstract class LRBytesFeatureExtractorBase {
     private static final double DEFAULT_USER_CTR_GROUP = 10.0;
     private static final double DEFAULT_ARTICLE_CTR_GROUP = 100.0;
 
-    double userCtrGroup = 10.0;
-    double videoCtrGroup = 100.0;
 
     private BytesUtils utils;
     //Feature Group & Features
     ListMultimap<FeatureGroup, BaseFeature> features = ArrayListMultimap.create();
     int groupCount;
-    LRBytesFeatureExtractorBase() {
-    };
 
-    LRBytesFeatureExtractorBase(FeatureUsage usage) {
-        this(usage, DEFAULT_USER_CTR_GROUP, DEFAULT_ARTICLE_CTR_GROUP);
-    }
 
-    LRBytesFeatureExtractorBase(FeatureUsage usage, double userCtrGroup, double videoCtrGroup) {
-        this.userCtrGroup = userCtrGroup;
-        this.videoCtrGroup = videoCtrGroup;
+    LRBytesFeatureExtractorBase() {
         groupCount = VlogFeatureGroup.values().length;
         BytesGroup[] groups = new BytesGroup[groupCount];
         for (VlogFeatureGroup g: VlogFeatureGroup.values()) {
             groups[g.ordinal()] = new BytesGroup(g.ordinal(),
                     g.getGroupName(), g.getGroupNameBytes());
         }
-        utils = new BytesUtils(groups, usage);
+        utils = new BytesUtils(groups);
     }
 
     private FeatureGroup makeGroup(VlogFeatureGroup group){
@@ -53,9 +44,9 @@ public abstract class LRBytesFeatureExtractorBase {
 
 
     void makeFea(VlogFeatureGroup group, byte[] value) {
-        FeatureGroup g = makeGroup(group);
+        FeatureGroup featureGroup = makeGroup(group);
         BaseFeature feature = utils.makeFea(group.ordinal(), value);
-        features.put(g, feature);
+        features.put(featureGroup, feature);
     }
 
     void makeFea(VlogFeatureGroup group, byte[][] list) {
@@ -64,6 +55,10 @@ public abstract class LRBytesFeatureExtractorBase {
         features.putAll(g, featureList);
     }
 
+    public ListMultimap<FeatureGroup, BaseFeature> getFeatures() {
+        return features;
+    }
+
     public abstract LRSamples single(UserBytesFeature userBytesFeature,
                                      VideoBytesFeature videoBytesFeature,
                                      RequestContextBytesFeature requestContextBytesFeature);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/feature/VlogShareLRFeatureExtractor.java

@@ -14,17 +14,10 @@ public class VlogShareLRFeatureExtractor extends LRBytesFeatureExtractorBase {
     public VlogShareLRFeatureExtractor() {
         super();
     }
-    public VlogShareLRFeatureExtractor(FeatureUsage usage) {
-        super(usage);
-    }
-    public VlogShareLRFeatureExtractor(FeatureUsage usage, double userCtrGroup, double videoCtrGroup) {
-        super(usage, userCtrGroup, videoCtrGroup);
-    }
-
 
     // TODO
     // 补充待抽取的context feature
-    private void getContextFeatures(RequestContextBytesFeature requestContextBytes) {
+    public void getContextFeatures(RequestContextBytesFeature requestContextBytes) {
         makeFea(VlogFeatureGroup.MACHINEINFO_BRAND, requestContextBytes.getMachineinfo_brand());
         makeFea(VlogFeatureGroup.MACHINEINFO_MODEL, requestContextBytes.getMachineinfo_model());
         makeFea(VlogFeatureGroup.MACHINEINFO_PLATFORM, requestContextBytes.getMachineinfo_platform());
@@ -39,16 +32,96 @@ public class VlogShareLRFeatureExtractor extends LRBytesFeatureExtractorBase {
     }
 
     //TODO
-    private void getUserFeatures(UserBytesFeature user) {
-        makeFea(VlogFeatureGroup.USER_CYCLE_BUCKET_7DAY,user.getUser_cycle_bucket_7days());
-        makeFea(VlogFeatureGroup.USER_SHARE_BUCKET_30DAY,user.getUser_share_bucket_30days());
-        makeFea(VlogFeatureGroup.USER_SHARE_BUCKET_30DAY,user.getUser_cycle_bucket_30days());
+    public void getUserFeatures(UserBytesFeature user) {
+        makeFea(VlogFeatureGroup.USER_CYCLE_BUCKET_7DAY, user.getUser_cycle_bucket_7days());
+        makeFea(VlogFeatureGroup.USER_SHARE_BUCKET_30DAY, user.getUser_share_bucket_30days());
+        makeFea(VlogFeatureGroup.USER_SHARE_BUCKET_30DAY, user.getUser_cycle_bucket_30days());
+
+        // 1day features
+        makeFea(VlogFeatureGroup.USER_1DAY_EXP, user.getDay1_cnt_features().get("exp"));
+        makeFea(VlogFeatureGroup.USER_1DAY_CLICK, user.getDay1_cnt_features().get("click"));
+        makeFea(VlogFeatureGroup.USER_1DAY_SHARE, user.getDay1_cnt_features().get("share"));
+        makeFea(VlogFeatureGroup.USER_1DAY_RETURN, user.getDay1_cnt_features().get("return"));
+        makeFea(VlogFeatureGroup.USER_1DAY_CTR, user.getDay1_cnt_features().get("ctr"));
+        makeFea(VlogFeatureGroup.USER_1DAY_STR, user.getDay1_cnt_features().get("str"));
+        makeFea(VlogFeatureGroup.USER_1DAY_ROV, user.getDay1_cnt_features().get("rov"));
+        makeFea(VlogFeatureGroup.USER_1DAY_ROS, user.getDay1_cnt_features().get("ros"));
+
+        // 3day features
+        makeFea(VlogFeatureGroup.USER_3DAY_EXP, user.getDay3_cnt_features().get("exp"));
+        makeFea(VlogFeatureGroup.USER_3DAY_CLICK, user.getDay3_cnt_features().get("click"));
+        makeFea(VlogFeatureGroup.USER_3DAY_SHARE, user.getDay3_cnt_features().get("share"));
+        makeFea(VlogFeatureGroup.USER_3DAY_RETURN, user.getDay3_cnt_features().get("return"));
+        makeFea(VlogFeatureGroup.USER_3DAY_CTR, user.getDay3_cnt_features().get("ctr"));
+        makeFea(VlogFeatureGroup.USER_3DAY_STR, user.getDay3_cnt_features().get("str"));
+        makeFea(VlogFeatureGroup.USER_3DAY_ROV, user.getDay3_cnt_features().get("rov"));
+        makeFea(VlogFeatureGroup.USER_3DAY_ROS, user.getDay3_cnt_features().get("ros"));
+
+        // 7day features
+        makeFea(VlogFeatureGroup.USER_7DAY_EXP, user.getDay7_cnt_features().get("exp"));
+        makeFea(VlogFeatureGroup.USER_7DAY_CLICK, user.getDay7_cnt_features().get("click"));
+        makeFea(VlogFeatureGroup.USER_7DAY_SHARE, user.getDay7_cnt_features().get("share"));
+        makeFea(VlogFeatureGroup.USER_7DAY_RETURN, user.getDay7_cnt_features().get("return"));
+        makeFea(VlogFeatureGroup.USER_7DAY_CTR, user.getDay7_cnt_features().get("ctr"));
+        makeFea(VlogFeatureGroup.USER_7DAY_STR, user.getDay7_cnt_features().get("str"));
+        makeFea(VlogFeatureGroup.USER_7DAY_ROV, user.getDay7_cnt_features().get("rov"));
+        makeFea(VlogFeatureGroup.USER_7DAY_ROS, user.getDay7_cnt_features().get("ros"));
+
+        // 3month features
+        makeFea(VlogFeatureGroup.USER_3MONTH_EXP, user.getMonth3_cnt_features().get("exp"));
+        makeFea(VlogFeatureGroup.USER_3MONTH_CLICK, user.getMonth3_cnt_features().get("click"));
+        makeFea(VlogFeatureGroup.USER_3MONTH_SHARE, user.getMonth3_cnt_features().get("share"));
+        makeFea(VlogFeatureGroup.USER_3MONTH_RETURN, user.getMonth3_cnt_features().get("return"));
+        makeFea(VlogFeatureGroup.USER_3MONTH_CTR, user.getMonth3_cnt_features().get("ctr"));
+        makeFea(VlogFeatureGroup.USER_3MONTH_STR, user.getMonth3_cnt_features().get("str"));
+        makeFea(VlogFeatureGroup.USER_3MONTH_ROV, user.getMonth3_cnt_features().get("rov"));
+        makeFea(VlogFeatureGroup.USER_3MONTH_ROS, user.getMonth3_cnt_features().get("ros"));
 
     }
 
-    private void getItemFeature(VideoBytesFeature videoBytesFeature) {
-        makeFea(VlogFeatureGroup.VIDEOID, videoBytesFeature.getVideoId());
-        makeFea(VlogFeatureGroup.UP_ID, videoBytesFeature.getUpId());
+    public void getItemFeature(VideoBytesFeature item) {
+        makeFea(VlogFeatureGroup.VIDEOID, item.getVideoId());
+        makeFea(VlogFeatureGroup.UP_ID, item.getUpId());
+        // 1day features
+        makeFea(VlogFeatureGroup.ITEM_1DAY_EXP, item.getItem_day1_cnt_features().get("exp"));
+        makeFea(VlogFeatureGroup.ITEM_1DAY_CLICK, item.getItem_day1_cnt_features().get("click"));
+        makeFea(VlogFeatureGroup.ITEM_1DAY_SHARE, item.getItem_day1_cnt_features().get("share"));
+        makeFea(VlogFeatureGroup.ITEM_1DAY_RETURN, item.getItem_day1_cnt_features().get("return"));
+        makeFea(VlogFeatureGroup.ITEM_1DAY_CTR, item.getItem_day1_cnt_features().get("ctr"));
+        makeFea(VlogFeatureGroup.ITEM_1DAY_STR, item.getItem_day1_cnt_features().get("str"));
+        makeFea(VlogFeatureGroup.ITEM_1DAY_ROV, item.getItem_day1_cnt_features().get("rov"));
+        makeFea(VlogFeatureGroup.ITEM_1DAY_ROS, item.getItem_day1_cnt_features().get("ros"));
+
+        // 3day features
+        makeFea(VlogFeatureGroup.ITEM_3DAY_EXP, item.getItem_day1_cnt_features().get("exp"));
+        makeFea(VlogFeatureGroup.ITEM_3DAY_CLICK, item.getItem_day1_cnt_features().get("click"));
+        makeFea(VlogFeatureGroup.ITEM_3DAY_SHARE, item.getItem_day1_cnt_features().get("share"));
+        makeFea(VlogFeatureGroup.ITEM_3DAY_RETURN, item.getItem_day1_cnt_features().get("return"));
+        makeFea(VlogFeatureGroup.ITEM_3DAY_CTR, item.getItem_day1_cnt_features().get("ctr"));
+        makeFea(VlogFeatureGroup.ITEM_3DAY_STR, item.getItem_day1_cnt_features().get("str"));
+        makeFea(VlogFeatureGroup.ITEM_3DAY_ROV, item.getItem_day1_cnt_features().get("rov"));
+        makeFea(VlogFeatureGroup.ITEM_3DAY_ROS, item.getItem_day1_cnt_features().get("ros"));
+
+        // 7day features
+        makeFea(VlogFeatureGroup.ITEM_7DAY_EXP, item.getItem_day7_cnt_features().get("exp"));
+        makeFea(VlogFeatureGroup.ITEM_7DAY_CLICK, item.getItem_day7_cnt_features().get("click"));
+        makeFea(VlogFeatureGroup.ITEM_7DAY_SHARE, item.getItem_day7_cnt_features().get("share"));
+        makeFea(VlogFeatureGroup.ITEM_7DAY_RETURN, item.getItem_day7_cnt_features().get("return"));
+        makeFea(VlogFeatureGroup.ITEM_7DAY_CTR, item.getItem_day7_cnt_features().get("ctr"));
+        makeFea(VlogFeatureGroup.ITEM_7DAY_STR, item.getItem_day7_cnt_features().get("str"));
+        makeFea(VlogFeatureGroup.ITEM_7DAY_ROV, item.getItem_day7_cnt_features().get("rov"));
+        makeFea(VlogFeatureGroup.ITEM_7DAY_ROS, item.getItem_day7_cnt_features().get("ros"));
+
+        // 3month features
+        makeFea(VlogFeatureGroup.ITEM_3MONTH_EXP, item.getItem_month3_cnt_features().get("exp"));
+        makeFea(VlogFeatureGroup.ITEM_3MONTH_CLICK, item.getItem_month3_cnt_features().get("click"));
+        makeFea(VlogFeatureGroup.ITEM_3MONTH_SHARE, item.getItem_month3_cnt_features().get("share"));
+        makeFea(VlogFeatureGroup.ITEM_3MONTH_RETURN, item.getItem_month3_cnt_features().get("return"));
+        makeFea(VlogFeatureGroup.ITEM_3MONTH_CTR, item.getItem_month3_cnt_features().get("ctr"));
+        makeFea(VlogFeatureGroup.ITEM_3MONTH_STR, item.getItem_month3_cnt_features().get("str"));
+        makeFea(VlogFeatureGroup.ITEM_3MONTH_ROV, item.getItem_month3_cnt_features().get("rov"));
+        makeFea(VlogFeatureGroup.ITEM_3MONTH_ROS, item.getItem_month3_cnt_features().get("ros"));
+
     }
 
     @Override
@@ -61,8 +134,7 @@ public class VlogShareLRFeatureExtractor extends LRBytesFeatureExtractorBase {
         getContextFeatures(requestContextBytesFeature);
         getItemFeature(videoBytesFeature);
 
-
-        LRSamples.Builder lr =  com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples.newBuilder();
+        LRSamples.Builder lr = com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples.newBuilder();
         lr.setGroupNum(groupCount);
         List<FeatureGroup> keys = new ArrayList<>(features.keySet());
         int count = 0;
@@ -80,14 +152,4 @@ public class VlogShareLRFeatureExtractor extends LRBytesFeatureExtractorBase {
     }
 
 
-
-
-
-
-
-
-
-
-
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/LRModel.java

@@ -73,16 +73,15 @@ public class LRModel extends Model {
     public Float score(LRSamples lrSamples) {
         float sum = 0.0f;
         for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
-            GroupedFeature gf = lrSamples.getFeatures(i);
-            if (gf != null && gf.getFeatures(i) != null) {
-                for (int j = 0; j < gf.getFeaturesCount(); j++) {
-                    BaseFeature fea = gf.getFeatures(j);
-                    if (fea != null) {
-                        float tmp = getWeight(this.lrModel, fea.getIdentifier());
-                        fea.toBuilder().setWeight(tmp);
-                        sum += tmp;
+            GroupedFeature groupedFeature = lrSamples.getFeatures(i);
+            if (groupedFeature != null && groupedFeature.getFeaturesCount() != 0) {
+                for (int j = 0; j < groupedFeature.getFeaturesCount(); j++) {
+                    BaseFeature baseFeature = groupedFeature.getFeatures(j);
+                    if (baseFeature != null) {
+                        float weight = getWeight(this.lrModel, baseFeature.getIdentifier());
+                        baseFeature.toBuilder().setWeight(weight);
+                        sum += weight;
                     }
-
                 }
             }
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/CommonCollectionUtils.java

@@ -19,13 +19,6 @@ public class CommonCollectionUtils {
         return list.stream().map(map).collect(Collectors.toList());
     }
 
-    public static <T, R> List<R> toListDistinct(List<T> list, Function<T, R> map) {
-        if (CollectionUtils.isEmpty(list)) {
-            return Collections.emptyList();
-        }
-        return list.stream().map(map).distinct().collect(Collectors.toList());
-    }
-
     public static <T, K, V> Map<K, V> toMap(List<T> list, Function<T, K> keyFunc, Function<T, V> valueFunc) {
         if (CollectionUtils.isEmpty(list)) {
             return Collections.emptyMap();