ad rank feature & model by sunmingze 1213

ad-engine-commons/pom.xml

@@ -17,7 +17,48 @@
         <maven.compiler.target>1.8</maven.compiler.target>
     </properties>
     <dependencies>
-<!--        <dependency>-->
+        <dependency>
+            <groupId>com.typesafe</groupId>
+            <artifactId>config</artifactId>
+            <version>1.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.tzld.piaoquan</groupId>
+            <artifactId>recommend-feature-client</artifactId>
+            <version>1.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.tzld.piaoquan</groupId>
+            <artifactId>recommend-server-client</artifactId>
+            <version>1.0.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>3.12.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.aliyun.oss</groupId>
+            <artifactId>aliyun-sdk-oss</artifactId>
+            <version>3.15.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.aliyun.odps</groupId>
+            <artifactId>odps-sdk-core</artifactId>
+            <version>0.45.6-public</version>
+        </dependency>
+
+        <dependency>
+            <groupId>it.unimi.dsi</groupId>
+            <artifactId>fastutil</artifactId>
+            <version>7.0.12</version>
+        </dependency>
+
+
+
+        <!--        <dependency>-->
 <!--            <groupId>net.devh</groupId>-->
 <!--            <artifactId>grpc-client-spring-boot-starter</artifactId>-->
 <!--            <version>2.9.0.RELEASE</version>-->

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/AdActionFeature.java

@@ -0,0 +1,72 @@
+package com.tzld.piaoquan.ad.engine.commons.base;
+
+import lombok.Data;
+
+@Data
+public class AdActionFeature {
+    private double adView;
+    private double adClick;
+    private double adConversion;
+
+    private double ctr;
+    private double cvr;
+
+    private double ceilLog(Double key) {
+        return Math.ceil(Math.log(key));
+    }
+
+    private double bucketRatioFeature(Double key) {
+        long bucket = Math.round(Math.log(key * 100));
+        if (bucket > 100)
+            bucket = 100;
+        return (double) bucket;
+    }
+
+
+    public void setAdView(Object key) {
+        if (key == null) {
+            this.adView = 0.0;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.adView = ceilLog(Double.valueOf(formateKey));
+        }
+    }
+
+    public void setAdClick(Object key) {
+        if (key == null) {
+            this.adClick = 0.0;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.adClick = ceilLog(Double.valueOf(formateKey));
+        }
+    }
+
+    public void setAdConversion(Object key) {
+        if (key == null) {
+            this.adConversion = 0.0;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.adConversion = ceilLog(Double.valueOf(formateKey));
+        }
+    }
+
+    public void setCtr(Object key) {
+        if (key == null) {
+            this.ctr = 0.0;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.ctr = bucketRatioFeature(Double.valueOf(formateKey));
+        }
+    }
+
+    public void setCvr(Object key) {
+        if (key == null) {
+            this.cvr = 0.0;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.cvr = bucketRatioFeature(Double.valueOf(formateKey));
+        }
+    }
+
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/AdItemBytesFeature.java

@@ -0,0 +1,87 @@
+package com.tzld.piaoquan.ad.engine.commons.base;
+
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Data
+public class AdItemBytesFeature {
+
+    private final byte[] adId;
+
+    private final byte[] adCode;
+
+    private final byte[] advertiserId;
+
+    private final byte[] advertiserCode;
+
+    private final byte[] campaignId;
+
+    private final byte[] campaignCode;
+
+    private final byte[] creative;
+
+    private final byte[] creativeCode;
+
+
+    private Map<String, byte[]> day1_cnt_features;
+    // 3天内统计量
+    private Map<String, byte[]> day3_cnt_features;
+    // 7天内统计量
+    private Map<String, byte[]> day7_cnt_features;
+    // 3个月统计量
+    private Map<String, byte[]> month3_cnt_features;
+
+
+    public AdItemBytesFeature(AdItemFeature feature) {
+
+        adId = feature.getAdId().getBytes();
+        adCode = feature.getAdCode().getBytes();
+        advertiserId = feature.getAdvertiserId().getBytes();
+        advertiserCode = feature.getAdvertiserCode().getBytes();
+        campaignId = feature.getCampaignId().getBytes();
+        campaignCode = feature.getCampaignCode().getBytes();
+        creative = feature.getCreative().getBytes();
+        creativeCode = feature.getCreativeCode().getBytes();
+
+
+
+        // 1 day statistic
+        day1_cnt_features = new HashMap<String, byte[]>();
+        day1_cnt_features.put("view", String.valueOf(feature.getDay1_cnt_features().getAdView()).getBytes());
+        day1_cnt_features.put("click", String.valueOf(feature.getDay1_cnt_features().getAdClick()).getBytes());
+        day1_cnt_features.put("conversion", String.valueOf(feature.getDay1_cnt_features().getAdConversion()).getBytes());
+        day1_cnt_features.put("ctr", String.valueOf(feature.getDay1_cnt_features().getCtr()).getBytes());
+        day1_cnt_features.put("cvr", String.valueOf(feature.getDay1_cnt_features().getCvr()).getBytes());
+
+
+        // 3 day statistic
+        day3_cnt_features = new HashMap<String, byte[]>();
+        day3_cnt_features.put("view", String.valueOf(feature.getDay3_cnt_features().getAdView()).getBytes());
+        day3_cnt_features.put("click", String.valueOf(feature.getDay3_cnt_features().getAdClick()).getBytes());
+        day3_cnt_features.put("conversion", String.valueOf(feature.getDay3_cnt_features().getAdConversion()).getBytes());
+        day3_cnt_features.put("ctr", String.valueOf(feature.getDay3_cnt_features().getCtr()).getBytes());
+        day3_cnt_features.put("cvr", String.valueOf(feature.getDay3_cnt_features().getCvr()).getBytes());
+
+
+        // 7 day statistic
+        day7_cnt_features = new HashMap<String, byte[]>();
+        day7_cnt_features.put("view", String.valueOf(feature.getDay7_cnt_features().getAdView()).getBytes());
+        day7_cnt_features.put("click", String.valueOf(feature.getDay7_cnt_features().getAdClick()).getBytes());
+        day7_cnt_features.put("conversion", String.valueOf(feature.getDay7_cnt_features().getAdConversion()).getBytes());
+        day7_cnt_features.put("ctr", String.valueOf(feature.getDay7_cnt_features().getCtr()).getBytes());
+        day7_cnt_features.put("cvr", String.valueOf(feature.getDay7_cnt_features().getCvr()).getBytes());
+
+
+        // 3 month statisic
+        month3_cnt_features = new HashMap<String, byte[]>();
+        month3_cnt_features.put("view", String.valueOf(feature.getMonth3_cnt_features().getAdView()).getBytes());
+        month3_cnt_features.put("click", String.valueOf(feature.getMonth3_cnt_features().getAdClick()).getBytes());
+        month3_cnt_features.put("conversion", String.valueOf(feature.getMonth3_cnt_features().getAdConversion()).getBytes());
+        month3_cnt_features.put("ctr", String.valueOf(feature.getMonth3_cnt_features().getCtr()).getBytes());
+        month3_cnt_features.put("cvr", String.valueOf(feature.getMonth3_cnt_features().getCvr()).getBytes());
+
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/AdItemFeature.java

@@ -0,0 +1,79 @@
+package com.tzld.piaoquan.ad.engine.commons.base;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+import javax.validation.constraints.NotNull;
+
+@Getter
+@NoArgsConstructor
+public class AdItemFeature {
+    @NotNull
+    private String adId;
+
+    private String adCode;
+
+    private String advertiserId;
+
+    private String advertiserCode;
+
+    private String campaignId;
+
+    private String campaignCode;
+
+    private String creative;
+
+    private String creativeCode;
+
+
+    // 当天统计量信息
+    private AdActionFeature day1_cnt_features;
+    // 3天内统计量
+    private AdActionFeature day3_cnt_features;
+    // 7天内统计量
+    private AdActionFeature day7_cnt_features;
+    // 3个月统计量
+    private AdActionFeature month3_cnt_features;
+
+
+    public void setAdvertiserId(String key){
+        if(key == null){
+            this.advertiserId = "0";
+        } else {
+            this.advertiserId = key;
+        }
+    }
+
+    public void setDay1_cnt_features(AdActionFeature feature){
+        this.day1_cnt_features = feature;
+    }
+
+
+    public void setDay3_cnt_features(AdActionFeature feature){
+        this.day3_cnt_features = feature;
+
+    }
+
+    public void setDay7_cnt_features(AdActionFeature feature){
+        this.day7_cnt_features = feature;
+
+    }
+
+    public void setMonth3_cnt_features(AdActionFeature feature){
+        this.month3_cnt_features= feature;
+
+    }
+
+    public String getKey() {
+        return this.adId;
+    }
+
+    public String getValue(){
+        Gson gson = new GsonBuilder().serializeSpecialFloatingPointValues().create();
+        return gson.toJson(this);
+    }
+
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/AdRankItem.java

@@ -0,0 +1,40 @@
+package com.tzld.piaoquan.ad.engine.commons.base;
+
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+@Data
+public class AdRankItem implements Comparable<AdRankItem> {
+    public long adId;
+    private double score; // 记录最终的score
+
+    // 记录Item侧用到的特征
+    private AdItemFeature itemFeature;
+
+    public AdRankItem() {
+        //TODO
+    }
+
+    private Map<String, Double> rankerScore = new HashMap<>();
+    private Map<String, Integer> rankerIndex = new HashMap<>();
+
+
+    @Override
+    public int compareTo(AdRankItem o) {
+        if (o == null) {
+            return -1;
+        }
+        if (score > o.score) {
+            return -1;
+        } else if (score < o.score) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/Constant.java

@@ -1,14 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-/**
- * 常量
- *
- * @author supeng
- * @date 2020/08/19
- */
-public class Constant {
-    /**
-     * traceID
-     */
-    public static final String LOG_TRACE_ID = "logTraceId";
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/RequestContext.java

@@ -0,0 +1,115 @@
+package com.tzld.piaoquan.ad.engine.commons.base;
+
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+@Getter
+@NoArgsConstructor
+public class RequestContext {
+
+    // 机型等信息
+    private String apptype;
+    private String machineinfoBrand;
+    private String machineinfoModel;
+    private String machineinfoPlatform;
+    private String machineinfoSdkversion;
+    private String machineinfoSystem;
+    private String machineinfoWechatversion;
+
+    // 时间等信息
+    private String day;
+    private String week;
+    private String hour;
+    private String region;
+    private String city;
+
+    // video信息
+    private String headVideoId;
+    private String PageSource;
+
+    // position 信息
+    private String ownAdPositionId;
+
+
+
+    public void setApptype(String apptype) {
+        this.apptype = apptype;
+        if (apptype == null)
+            this.apptype = "-1";
+    }
+
+    public void setMachineinfoBrand(String machineinfoBrand) {
+        this.machineinfoBrand = machineinfoBrand;
+        if (machineinfoBrand == null)
+            this.machineinfoBrand = "-1";
+    }
+
+    public void setMachineinfoModel(String machineinfoModel) {
+        this.machineinfoModel = machineinfoModel;
+        if (machineinfoModel == null)
+            this.machineinfoModel = "-1";
+    }
+
+
+    public void setMachineinfoWchatversion(String machineinfo_wechatversion) {
+        this.machineinfoWechatversion = machineinfo_wechatversion;
+        if (machineinfo_wechatversion == null)
+            this.machineinfoWechatversion = "-1";
+    }
+
+
+    public void setMachineinfoSdkversion(String machineinfo_sdkversion) {
+        this.machineinfoSdkversion = machineinfo_sdkversion;
+        if (machineinfoSdkversion == null)
+            this.machineinfoSdkversion = "-1";
+    }
+
+    public void setMachineinfoPlatform(String machineinfo_platform) {
+        this.machineinfoPlatform = machineinfo_platform;
+        if (machineinfoPlatform == null)
+            this.machineinfoPlatform = "-1";
+    }
+
+    public void setMachineinfo_system(String machineinfo_system) {
+        this.machineinfoSystem = machineinfo_system;
+        if (machineinfo_system == null)
+            this.machineinfoSystem = "-1";
+    }
+
+
+    public void setHour(String hour) {
+        this.hour = hour;
+        if (hour == null)
+            this.hour = "-1";
+    }
+
+
+    public void setDay(String day) {
+        this.day = day;
+        if (day == null)
+            this.day = "-1";
+    }
+
+    public void setWeek(String week) {
+        this.week = week;
+        if (week == null)
+            this.week = "-1";
+    }
+
+
+    public void setRegion(String region) {
+        this.region = region;
+        if (region == null)
+            this.region = "-1";
+    }
+
+
+    public void setCity(String city) {
+        this.city = city;
+        if (city == null)
+            this.city = "-1";
+    }
+
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/RequestContextBytesFeature.java

@@ -0,0 +1,55 @@
+package com.tzld.piaoquan.ad.engine.commons.base;
+
+
+import lombok.Data;
+
+@Data
+public class RequestContextBytesFeature {
+    private final byte[] apptype;
+    private final byte[] machineinfo_brand;
+    private final byte[] machineinfo_model;
+    private final byte[] machineinfo_platform;
+    private final byte[] machineinfo_sdkversion;
+    private final byte[] machineinfo_system;
+    private final byte[] machineinfo_wechatversion;
+
+    // 时间等信息
+    private final byte[] day;
+    private final byte[] week;
+    private final byte[] hour;
+    private final byte[] region;
+    private final byte[] city;
+
+    // position 等信息
+    private final byte[] headVideoId;
+    private final byte[] pageSource;
+
+    // position 信息
+    private final byte[] ownAdPositionId;
+
+
+    public RequestContextBytesFeature(RequestContext requestContext) {
+        apptype = requestContext.getApptype().getBytes();
+        machineinfo_brand = requestContext.getMachineinfoBrand().getBytes();
+        machineinfo_model = requestContext.getMachineinfoModel().getBytes();
+        machineinfo_platform = requestContext.getMachineinfoPlatform().getBytes();
+        machineinfo_sdkversion = requestContext.getMachineinfoSdkversion().getBytes();
+        machineinfo_system = requestContext.getMachineinfoSystem().getBytes();
+        machineinfo_wechatversion = requestContext.getMachineinfoWechatversion().getBytes();
+
+        day = requestContext.getDay().getBytes();
+        week = requestContext.getWeek().getBytes();
+        hour = requestContext.getHour().getBytes();
+        region = requestContext.getRegion().getBytes();
+        city = requestContext.getCity().getBytes();
+
+
+        headVideoId = requestContext.getHeadVideoId().getBytes();
+        pageSource = requestContext.getPageSource().getBytes();
+        ownAdPositionId = requestContext.getOwnAdPositionId().getBytes();
+
+
+    }
+
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/UserAdBytesFeature.java

@@ -0,0 +1,155 @@
+package com.tzld.piaoquan.ad.engine.commons.base;
+
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Data
+public class UserAdBytesFeature {
+
+    private final byte[] mid;
+
+    private final byte[] gender;
+    private final byte[] user_type;
+    private final byte[] gmt_create;
+    private final byte[] tags;
+    private final byte[] category_name;
+    private final byte[] isvip;
+    private final byte[] isreward;
+    private final byte[] isad;
+    private final byte[] isgood;
+
+    private final byte[] first_up_datetime;
+    private final byte[] last_up_datetime;
+    private final byte[] next_to_last_up_datetime;
+    private final byte[] videos; // 上传视频数量
+    private final byte[] idols; // 关注人数
+    private final byte[] fans;  // 粉丝数
+    private final byte[] play_count;  // 播放人数
+    private final byte[] play_count_total;  // 累计播放次数
+
+    private final byte[] total_reward;
+    private final byte[] currentday_reward;
+    private final byte[] reward_person;
+    private final byte[] total_reward_times;
+
+    private final byte[] reward_videos;
+    private final byte[] total_price;
+    private final byte[] currentday_price;
+    private final byte[] total_price_times;
+    private final byte[] total_price_person;
+    private final byte[] total_price_videos;
+    private final byte[] cgrain_user_type;
+    private final byte[] identity_tagname;
+
+
+    private final byte[] operation_tags;
+    private final byte[] identity_tag_id;
+    private final byte[] identity_create_time;
+    private final byte[] country;
+    private final byte[] province;
+    private final byte[] city;
+
+
+    // 当天统计量信息
+    private Map<String, byte[]> day1_cnt_features;
+    // 3天内统计量
+    private Map<String, byte[]> day3_cnt_features;
+    // 7天内统计量
+    private Map<String, byte[]> day7_cnt_features;
+    // 3个月统计量
+    private Map<String, byte[]> month3_cnt_features;
+
+
+    public UserAdBytesFeature(UserAdFeature feature) {
+        this.mid = feature.getMid().getBytes();
+
+
+        this.gender = feature.getGender().getBytes();
+        this.user_type = feature.getUser_type().getBytes();
+        this.gmt_create = feature.getGmt_create().getBytes();
+        this.tags = feature.getTags().getBytes() ;
+        this.category_name = feature.getCategory_name().getBytes();
+        this.isvip = feature.getIsvip().getBytes();
+        this.isreward = feature.getIsreward().getBytes();
+        this.isad = feature.getIsad().getBytes();
+        this.isgood = feature.getIsgood().getBytes();
+
+        this.first_up_datetime = feature.getFirst_up_datetime().getBytes();
+        this.last_up_datetime = feature.getLast_up_datetime().getBytes();
+        this.next_to_last_up_datetime = feature.getNext_to_last_up_datetime().getBytes();
+        this.videos = feature.getVideos().getBytes(); // 上传视频数量
+        this.idols = feature.getIdols().getBytes(); // 关注人数
+        this.fans = feature.getFans().getBytes();  // 粉丝数
+        this.play_count = feature.getPlay_count().getBytes();  // 播放人数
+        this.play_count_total = feature.getPlay_count_total().getBytes();  // 累计播放次数
+
+        this.total_reward = feature.getTotal_reward().getBytes();
+        this.currentday_reward = feature.getCurrentday_reward().getBytes();
+        this.reward_person = feature.getReward_person().getBytes();
+        this.total_reward_times = feature.getTotal_reward_times().getBytes();
+
+        this.reward_videos = feature.getReward_videos().getBytes();
+        this.total_price = feature.getTotal_price().getBytes();
+        this.currentday_price = feature.getCurrentday_price().getBytes();
+        this.total_price_times = feature.getTotal_price_times().getBytes();
+        this.total_price_person = feature.getTotal_price_person().getBytes();
+        this.total_price_videos = feature.getTotal_price_videos().getBytes();
+        this.cgrain_user_type = feature.getCgrain_user_type().getBytes();
+        this.identity_tagname = feature.getIdentity_tagname().getBytes();
+
+
+        this.operation_tags = feature.getOperation_tags().getBytes();
+        this.identity_tag_id = feature.getIdentity_tag_id().getBytes();
+        this.identity_create_time = feature.getIdentity_create_time().getBytes();
+        this.country = feature.getCountry().getBytes();
+        this.province = feature.getProvince().getBytes();
+        this.city = feature.getCity().getBytes();
+
+
+        this.day1_cnt_features = new HashMap<String, byte[]>();
+
+        // 1 day statistic
+        day1_cnt_features = new HashMap<String, byte[]>();
+        day1_cnt_features.put("view", String.valueOf(feature.getDay1_cnt_features().getAdView()).getBytes());
+        day1_cnt_features.put("click", String.valueOf(feature.getDay1_cnt_features().getAdClick()).getBytes());
+        day1_cnt_features.put("conversion", String.valueOf(feature.getDay1_cnt_features().getAdConversion()).getBytes());
+        day1_cnt_features.put("ctr", String.valueOf(feature.getDay1_cnt_features().getCtr()).getBytes());
+        day1_cnt_features.put("cvr", String.valueOf(feature.getDay1_cnt_features().getCvr()).getBytes());
+
+
+
+
+        // 3 day statistic
+        day3_cnt_features = new HashMap<String, byte[]>();
+        day3_cnt_features.put("view", String.valueOf(feature.getDay3_cnt_features().getAdView()).getBytes());
+        day3_cnt_features.put("click", String.valueOf(feature.getDay3_cnt_features().getAdClick()).getBytes());
+        day3_cnt_features.put("conversion", String.valueOf(feature.getDay3_cnt_features().getAdConversion()).getBytes());
+        day3_cnt_features.put("ctr", String.valueOf(feature.getDay3_cnt_features().getCtr()).getBytes());
+        day3_cnt_features.put("cvr", String.valueOf(feature.getDay3_cnt_features().getCvr()).getBytes());
+
+
+        // 7 day statistic
+        day7_cnt_features = new HashMap<String, byte[]>();
+        day7_cnt_features.put("view", String.valueOf(feature.getDay7_cnt_features().getAdView()).getBytes());
+        day7_cnt_features.put("click", String.valueOf(feature.getDay7_cnt_features().getAdClick()).getBytes());
+        day7_cnt_features.put("conversion", String.valueOf(feature.getDay7_cnt_features().getAdConversion()).getBytes());
+        day7_cnt_features.put("ctr", String.valueOf(feature.getDay7_cnt_features().getCtr()).getBytes());
+        day7_cnt_features.put("cvr", String.valueOf(feature.getDay7_cnt_features().getCvr()).getBytes());
+
+
+        // 3 month statisic
+        month3_cnt_features = new HashMap<String, byte[]>();
+        month3_cnt_features.put("view", String.valueOf(feature.getMonth3_cnt_features().getAdView()).getBytes());
+        month3_cnt_features.put("click", String.valueOf(feature.getMonth3_cnt_features().getAdClick()).getBytes());
+        month3_cnt_features.put("conversion", String.valueOf(feature.getMonth3_cnt_features().getAdConversion()).getBytes());
+        month3_cnt_features.put("ctr", String.valueOf(feature.getMonth3_cnt_features().getCtr()).getBytes());
+        month3_cnt_features.put("cvr", String.valueOf(feature.getMonth3_cnt_features().getCvr()).getBytes());
+
+
+
+    }
+
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/UserAdFeature.java

@@ -0,0 +1,112 @@
+package com.tzld.piaoquan.ad.engine.commons.base;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+@Getter
+@NoArgsConstructor
+public class UserAdFeature {
+    private String mid;
+
+    private String gender;
+    private String user_type;
+    private String gmt_create;
+    private String tags;
+    private String category_name;
+    private String isvip;
+    private String isreward;
+    private String isad;
+    private String isgood;
+
+    private String first_up_datetime;
+    private String last_up_datetime;
+    private String next_to_last_up_datetime;
+    private String videos; // 上传视频数量
+    private String idols; // 关注人数
+    private String fans;  // 粉丝数
+    private String play_count;  // 播放人数
+    private String play_count_total;  // 累计播放次数
+
+    private String total_reward;
+    private String currentday_reward;
+    private String reward_person;
+    private String total_reward_times;
+
+    private String reward_videos;
+    private String total_price;
+    private String currentday_price;
+    private String total_price_times;
+    private String total_price_person;
+    private String total_price_videos;
+    private String cgrain_user_type;
+    private String identity_tagname;
+
+
+    private String operation_tags;
+    private String identity_tag_id;
+    private String identity_create_time;
+    private String country;
+    private String province;
+    private String city;
+
+
+
+
+    // 当天统计量信息
+    private AdActionFeature day1_cnt_features;
+    // 3天内统计量
+    private AdActionFeature day3_cnt_features;
+    // 7天内统计量
+    private AdActionFeature day7_cnt_features;
+    // 3个月统计量
+    private AdActionFeature month3_cnt_features;
+
+
+
+
+    public void setMid(String key){
+        this.mid = key;
+        if(key == null)
+            this.mid = "0";
+    }
+
+
+    public void setDay1_cnt_features(AdActionFeature key){
+        this.day1_cnt_features = key;
+        if(key == null)
+            this.day1_cnt_features = new AdActionFeature();
+    }
+
+    public void setDay3_cnt_features(AdActionFeature key){
+        this.day3_cnt_features = key;
+        if(key == null)
+            this.day3_cnt_features = new AdActionFeature();
+    }
+
+    public void setDay7_cnt_features(AdActionFeature key){
+        this.day7_cnt_features = key;
+        if(key == null)
+            this.day7_cnt_features = new AdActionFeature();
+    }
+
+    public void setMonth3_cnt_features(AdActionFeature key) {
+        this.month3_cnt_features = key;
+        if(key == null)
+            this.month3_cnt_features = new AdActionFeature();
+    }
+
+
+
+
+    public String getKey() {
+        return this.mid;
+    }
+
+    public String getValue(){
+        Gson gson = new GsonBuilder().serializeSpecialFloatingPointValues().create();
+        return gson.toJson(this);
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/UserVideoActionFeature.java

@@ -0,0 +1,108 @@
+package com.tzld.piaoquan.ad.engine.commons.base;
+
+import lombok.Data;
+
+@Data
+public class UserVideoActionFeature {
+    private double exp_cnt;
+    private double click_cnt;
+    private double share_cnt;
+    private double return_cnt;
+
+    private double ctr;
+    private double str;
+    private double rov;
+    private double ros;
+
+    private double ceilLog(Double key) {
+        return Math.ceil(Math.log(key));
+    }
+
+    private double bucketRatioFeature(Double key) {
+        long bucket = Math.round(Math.log(key * 100));
+        if( bucket > 100)
+            bucket = 100;
+        return (double) bucket;
+    }
+
+
+    public void setExp_cnt(Object key){
+        if(key == null ) {
+            this.exp_cnt = 0.0;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.exp_cnt = ceilLog(Double.valueOf(formateKey));
+        }
+    }
+
+    public void setClick_cnt(Object key){
+        if(key == null ){
+            this.click_cnt = 0.0 ;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.click_cnt = ceilLog(Double.valueOf(formateKey));
+        }
+    }
+    public void setShare_cnt(Object key){
+        if(key == null ){
+            this.share_cnt = 0.0 ;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.share_cnt = ceilLog(Double.valueOf(formateKey));
+        }
+    }
+    public void setReturn_cnt(Object key){
+        if(key == null ){
+            this.return_cnt = 0.0 ;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.return_cnt = ceilLog(Double.valueOf(formateKey));
+        }
+    }
+
+    public void setCtr(Object key){
+        if(key == null ){
+            this.ctr = 0.0 ;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.ctr = bucketRatioFeature(Double.valueOf(formateKey));
+        }
+    }
+
+    public void setStr(Object key){
+        if(key == null ){
+            this.str = 0.0 ;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.str = bucketRatioFeature(Double.valueOf(formateKey));
+        }
+    }
+
+    public void setRov(Object key){
+        if(key == null ){
+            this.rov = 0.0 ;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.rov = bucketRatioFeature(Double.valueOf(formateKey));
+        }
+    }
+
+    public void setRos(Object key){
+        if(key == null ){
+            this.ros = 0.0 ;
+        } else {
+            String formateKey = key.toString().replace("\\N", "-1");
+            this.ros = bucketRatioFeature(Double.valueOf(formateKey));
+        }
+    }
+
+
+
+
+
+
+
+
+
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/UserVideoFeature.java

@@ -0,0 +1,86 @@
+package com.tzld.piaoquan.ad.engine.commons.base;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+@Getter
+@NoArgsConstructor
+public class UserVideoFeature {
+
+    private String uid;
+    // 当天统计量信息
+    private AdActionFeature day1_cnt_features;
+    // 3天内统计量
+    private AdActionFeature day3_cnt_features;
+    // 7天内统计量
+    private AdActionFeature day7_cnt_features;
+    // 3个月统计量
+    private AdActionFeature month3_cnt_features;
+    // 用户行为周期
+    private String user_cycle_bucket_7days;
+    private String user_cycle_bucket_30days;
+    private String user_share_bucket_30days;
+
+    public void setUid(String key){
+        this.uid = key;
+        if(key == null)
+            this.uid = "0";
+    }
+
+
+    public void setDay1_cnt_features(AdActionFeature key){
+        this.day1_cnt_features = key;
+        if(key == null)
+            this.day1_cnt_features = new AdActionFeature();
+    }
+
+    public void setDay3_cnt_features(AdActionFeature key){
+        this.day3_cnt_features = key;
+        if(key == null)
+            this.day3_cnt_features = new AdActionFeature();
+    }
+
+    public void setDay7_cnt_features(AdActionFeature key){
+        this.day7_cnt_features = key;
+        if(key == null)
+            this.day7_cnt_features = new AdActionFeature();
+    }
+
+    public void setMonth3_cnt_features(AdActionFeature key) {
+        this.month3_cnt_features = key;
+        if(key == null)
+            this.month3_cnt_features = new AdActionFeature();
+    }
+
+
+    public void setUser_cycle_bucket_7days(String key){
+        this.user_cycle_bucket_7days = key;
+        if(key == null)
+            this.user_cycle_bucket_7days = "0";
+    }
+
+    public void setUser_cycle_bucket_30days(String key){
+        this.user_cycle_bucket_30days = key;
+        if(key == null)
+            this.user_cycle_bucket_30days = "0";
+    }
+
+    public void setUser_share_bucket_30days(String key){
+        this.user_share_bucket_30days = key;
+        if(key == null)
+            this.user_share_bucket_30days = "0";
+    }
+
+
+    public String getKey() {
+        return this.uid;
+    }
+
+    public String getValue(){
+        Gson gson = new GsonBuilder().serializeSpecialFloatingPointValues().create();
+        return gson.toJson(this);
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/enums/VlogAdFeatureGroup.java

@@ -0,0 +1,129 @@
+package com.tzld.piaoquan.ad.engine.commons.enums;
+
+public enum VlogAdFeatureGroup {
+    MID,
+    MACHINEINFO_BRAND,
+    MACHINEINFO_MODEL,
+    MACHINEINFO_PLATFORM,
+    MACHINEINFO_SDKVERSION,
+    MACHINEINFO_SYSTEM,
+    MACHINEINFO_WECHATVERSION,
+
+
+    DAY,
+    WEEK,
+    HOUR,
+    REGION,
+    CITY,
+
+    // AD info
+    ADID,
+    CREATIVE,
+    ADVERID,
+    CAMPAIGIN,
+
+
+
+
+    USER_1DAY_EXP,
+    USER_1DAY_CLICK,
+    USER_1DAY_SHARE,
+    USER_1DAY_RETURN,
+    USER_1DAY_CTR,
+    USER_1DAY_STR,
+    USER_1DAY_ROV,
+    USER_1DAY_ROS,
+
+    USER_3DAY_EXP,
+    USER_3DAY_CLICK,
+    USER_3DAY_SHARE,
+    USER_3DAY_RETURN,
+    USER_3DAY_CTR,
+    USER_3DAY_STR,
+    USER_3DAY_ROV,
+    USER_3DAY_ROS,
+
+    USER_7DAY_EXP,
+    USER_7DAY_CLICK,
+    USER_7DAY_SHARE,
+    USER_7DAY_RETURN,
+    USER_7DAY_CTR,
+    USER_7DAY_STR,
+    USER_7DAY_ROV,
+    USER_7DAY_ROS,
+
+    USER_3MONTH_EXP,
+    USER_3MONTH_CLICK,
+    USER_3MONTH_SHARE,
+    USER_3MONTH_RETURN,
+    USER_3MONTH_CTR,
+    USER_3MONTH_STR,
+    USER_3MONTH_ROV,
+    USER_3MONTH_ROS,
+
+
+    ITEM_1DAY_EXP,
+    ITEM_1DAY_CLICK,
+    ITEM_1DAY_SHARE,
+    ITEM_1DAY_RETURN,
+    ITEM_1DAY_CTR,
+    ITEM_1DAY_STR,
+    ITEM_1DAY_ROV,
+    ITEM_1DAY_ROS,
+
+    ITEM_3DAY_EXP,
+    ITEM_3DAY_CLICK,
+    ITEM_3DAY_SHARE,
+    ITEM_3DAY_RETURN,
+    ITEM_3DAY_CTR,
+    ITEM_3DAY_STR,
+    ITEM_3DAY_ROV,
+    ITEM_3DAY_ROS,
+
+    ITEM_7DAY_EXP,
+    ITEM_7DAY_CLICK,
+    ITEM_7DAY_SHARE,
+    ITEM_7DAY_RETURN,
+    ITEM_7DAY_CTR,
+    ITEM_7DAY_STR,
+    ITEM_7DAY_ROV,
+    ITEM_7DAY_ROS,
+
+    ITEM_3MONTH_EXP,
+    ITEM_3MONTH_CLICK,
+    ITEM_3MONTH_SHARE,
+    ITEM_3MONTH_RETURN,
+    ITEM_3MONTH_CTR,
+    ITEM_3MONTH_STR,
+    ITEM_3MONTH_ROV,
+    ITEM_3MONTH_ROS,
+
+
+    ;
+
+
+    private final byte[] idBytes;
+    private final byte[] nameBytes;
+
+    VlogAdFeatureGroup() {
+        this.nameBytes = name().toLowerCase().getBytes();
+        this.idBytes = String.valueOf(ordinal()).getBytes();
+    }
+
+    public final int getId() {
+        return ordinal();
+    }
+
+    public final String getGroupName() {
+        return name().toLowerCase();
+    }
+
+    public final byte[] getGroupNameBytes() {
+        return getGroupName().getBytes();
+    }
+
+    public final byte[] getIdBytes() {
+        return idBytes;
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/AbstractScorer.java

@@ -0,0 +1,67 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+
+import com.tzld.piaoquan.ad.engine.commons.base.UserVideoFeature;
+import com.tzld.piaoquan.ad.engine.commons.score.model.Model;
+import com.tzld.piaoquan.ad.engine.commons.score.model.ModelManager;
+import com.tzld.piaoquan.ad.engine.commons.base.AdRankItem;
+import com.tzld.piaoquan.ad.engine.commons.base.UserAdFeature;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+
+public abstract class AbstractScorer {
+    public static Logger LOGGER = LoggerFactory.getLogger(AbstractScorer.class);
+    protected ScorerConfigInfo scorerConfigInfo;
+    protected ModelManager modelManager = ModelManager.getInstance();
+
+    public AbstractScorer(ScorerConfigInfo scorerConfigInfo) {
+        this.scorerConfigInfo = scorerConfigInfo;
+    }
+
+    public void loadModel() {
+    }
+
+    public boolean isEnable() {
+        return !getScorerConfigInfo().getDisableSwitch();
+    }
+
+
+    public void doLoadModel(Class<? extends Model> modelClass) {
+
+        String modelPath = scorerConfigInfo.getModelPath();
+        if (StringUtils.isNotBlank(modelPath)) {
+            try {
+                // 使用 modelPath 作为 modelName 注册
+                modelManager.registerModel(modelPath, modelPath, modelClass);
+                LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
+            } catch (ModelManager.ModelRegisterException e) {
+                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+            } catch (IOException e) {
+                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+            }
+        } else {
+            LOGGER.error("modelpath is null, for model class [{}]", modelClass);
+        }
+    }
+
+    public Model getModel() {
+        if (StringUtils.isBlank(scorerConfigInfo.getModelPath())) {
+            return null;
+        }
+        return modelManager.getModel(scorerConfigInfo.getModelPath());
+    }
+
+    public ScorerConfigInfo getScorerConfigInfo() {
+        return scorerConfigInfo;
+    }
+
+    public abstract List<AdRankItem> scoring(final ScoreParam param,
+                                           final UserAdFeature userAdFeature,
+                                           final List<AdRankItem> rankItems);
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/BaseLRModelScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+import com.tzld.piaoquan.ad.engine.commons.score.model.LRModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseLRModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRModelScorer.class);
+
+    public BaseLRModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(LRModel.class);
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScoreParam.java

@@ -0,0 +1,12 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+import com.tzld.piaoquan.ad.engine.commons.base.RequestContext;
+import lombok.Data;
+
+/**
+ * @author dyp
+ */
+@Data
+public class ScoreParam {
+    private RequestContext requestContext;
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerConfig.java

@@ -0,0 +1,140 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+public class ScorerConfig {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
+    private List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
+
+    public ScorerConfig(Config config) {
+        this.load(config);
+    }
+
+    public ScorerConfig(String configFile) {
+        this.load(configFile);
+    }
+
+    public ScorerConfig() {
+    }
+
+    public static Config mergeConfig(Config baselineConfig, List<Config> expConfigs) {
+        if (expConfigs == null || expConfigs.size() == 0) {
+            return baselineConfig;
+        }
+        Config resultConfig = expConfigs.get(0);
+        for (Config config : expConfigs) {
+            resultConfig = resultConfig.withFallback(config);
+        }
+        resultConfig = resultConfig.withFallback(baselineConfig);
+        return resultConfig;
+    }
+
+    public boolean load(String configFile) {
+        Config config = ConfigFactory.parseResources(configFile);
+        return load(config);
+    }
+
+    public boolean load(Config baselineConfig, List<Config> expConfigs) {
+        Config config = mergeConfig(baselineConfig, expConfigs);
+        return load(config);
+    }
+
+    public boolean load(Config config) {
+        Config scorerConfig = config.getConfig("scorer-config");
+
+        try {
+            loadScorers(scorerConfig);
+            int pos = 0;
+            for (ScorerConfigInfo scorerConfigInfo : configInfoList) {
+                LOGGER.debug("scorer at position [{}], priority [{}], scorer name [{}]",
+                        new Object[]{pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName()});
+            }
+            LOGGER.debug("Load scorer config success");
+        } catch (Exception e) {
+            LOGGER.error("Load scorer config failed, [{}]", ExceptionUtils.getFullStackTrace(e));
+            return false;
+        }
+
+        return true;
+    }
+
+    public List<ScorerConfigInfo> getConfigInfoList() {
+        return configInfoList;
+    }
+
+
+    public String loadOptionStringConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getString(path) : null;
+    }
+
+    private Config loadOptionConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
+    }
+
+    private void loadScorers(Config config) throws Exception {
+
+        ConfigObject confObj = config.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            Config conf = ((ConfigObject) it.getValue()).toConfig();
+            // parse config
+            String configName = it.getKey();
+            String scorerName = conf.getString("scorer-name");
+            int scorerPriority = 0;
+            if (conf.hasPath("scorer-priority"))
+                scorerPriority = conf.getInt("scorer-priority");
+            Boolean disableSwitch = false;
+            if (conf.hasPath("disable-switch")) {
+                disableSwitch = conf.getBoolean("disable-switch");
+            }
+            Config paramConfig = loadOptionConfig(conf, "param-config");
+            // model path
+            String modelPath = loadOptionStringConfig(conf, "model-path");
+            if (modelPath == null) {
+                modelPath = loadOptionStringConfig(conf, "default-model-path");
+                LOGGER.debug("model-path is not exists in config file, use default-model-path instead, modelPath={}", modelPath);
+            }
+            // enable queues
+            Set<String> enableQueues = new HashSet<String>();
+            if (conf.hasPath("enable-queues")) {
+                enableQueues.addAll(conf.getStringList("enable-queues"));
+            }
+            ScorerConfigInfo configInfo = new ScorerConfigInfo(configName,
+                    scorerName,
+                    scorerPriority,
+                    disableSwitch,
+                    enableQueues,
+                    modelPath,
+                    paramConfig
+            );
+            LOGGER.debug("parse scorer config info [{}]", configInfo);
+            // add to ConfigInfoList
+            addConfigByPriority(configInfoList, configInfo);
+        }
+    }
+
+    private void addConfigByPriority(List<ScorerConfigInfo> configInfoList, ScorerConfigInfo addConfigInfo) {
+
+        int pos = 0;
+        for (; pos < configInfoList.size(); pos++) {
+            if (configInfoList.get(pos).getScorerPriority() <= addConfigInfo.getScorerPriority()) {
+                break;
+            }
+        }
+
+        configInfoList.add(pos, addConfigInfo);
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerConfigInfo.java

@@ -0,0 +1,78 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+import com.google.gson.Gson;
+import com.typesafe.config.Config;
+
+import java.util.Set;
+
+
+public class ScorerConfigInfo {
+
+    private String configName;
+    private Integer scorerPriority;
+    private Boolean disableSwitch;
+    private String scorerName;
+    private Set<String> enableQueues;
+    private String modelPath;
+    private Config paramConfig; // param config
+
+    public ScorerConfigInfo(String configName,
+                            String scorerName,
+                            Integer scorerPriority,
+                            Boolean disableSwitch,
+                            Set<String> enableQueues,
+                            String modelPath,
+                            Config paramConfig) {
+
+        this.configName = configName;
+        this.scorerName = scorerName;
+        this.scorerPriority = scorerPriority;
+        this.disableSwitch = disableSwitch;
+        this.enableQueues = enableQueues;
+        this.modelPath = modelPath;
+        this.paramConfig = paramConfig;
+    }
+
+    public Config getParamConfig() {
+        return paramConfig;
+    }
+
+    public Set<String> getEnableQueues() {
+        return enableQueues;
+    }
+
+    public void setEnableQueues(Set<String> enableQueues) {
+        this.enableQueues = enableQueues;
+    }
+
+    public Integer getScorerPriority() {
+        return scorerPriority;
+    }
+
+    public String getScorerName() {
+        return scorerName;
+    }
+
+    public String getConfigName() {
+        return configName;
+    }
+
+    public boolean isQueueEnable(String queueName) {
+        return this.enableQueues == null ||
+                this.enableQueues.isEmpty() ||
+                this.enableQueues.contains(queueName);
+    }
+
+    public String getModelPath() {
+        return modelPath;
+    }
+
+    public Boolean getDisableSwitch() {
+        return disableSwitch;
+    }
+
+    @Override
+    public String toString() {
+        return new Gson().toJson(this);
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerPipeline.java

@@ -0,0 +1,111 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+import com.tzld.piaoquan.ad.engine.commons.base.AdRankItem;
+import com.tzld.piaoquan.ad.engine.commons.base.UserAdFeature;
+import com.tzld.piaoquan.ad.engine.commons.base.UserVideoFeature;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.*;
+
+
+@Slf4j
+public class ScorerPipeline {
+    public static final int corePoolSize = 128;
+    public static final int SCORE_TIME_OUT = 400;
+    public static final Logger LOGGER = LoggerFactory.getLogger(ScorerPipeline.class);
+    public static final ExecutorService executorService = Executors.newFixedThreadPool(corePoolSize);
+
+    public List<AbstractScorer> scorers;
+
+    public ScorerPipeline(List<AbstractScorer> scorers) {
+        this.scorers = scorers;
+    }
+
+    /**
+     * scoring
+     *
+     * @return
+     */
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserVideoFeature userVideoFeature,
+                                    final UserAdFeature userAdFeature,
+                                    final List<AdRankItem> rankItems) {
+
+        if (CollectionUtils.isEmpty(scorers)) {
+            log.error("scorers is empty");
+            return rankItems;
+        }
+        List<AdRankItem> items = rankItems;
+
+        for (final AbstractScorer scorer : scorers) {
+            if (!scorer.isEnable()) {
+                continue;
+            }
+
+            final int beforeSize = rankItems.size();
+            final long startTime = System.currentTimeMillis();
+
+            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
+            String[] scorerNames = fullScorerName.split("\\.");
+            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
+
+            final List<AdRankItem> scoreRankerItems = items;
+            Callable<List<AdRankItem>> callable = () -> scorer.scoring(param, userAdFeature, scoreRankerItems);
+
+            // execute score use thread to protected score worst time
+            List<AdRankItem> scoredItems = new ArrayList<AdRankItem>();
+            try {
+                List<Future<List<AdRankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
+                for (Future<List<AdRankItem>> future : futures) {
+                    try {
+                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                            scoredItems.addAll(future.get());
+                        } else {
+                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
+                                    new Object[]{scorerName, scoreRankerItems.size()});
+                        }
+                    } catch (Exception e) {
+                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
+                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+                    }
+                }
+            } catch (Exception e) {
+                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
+                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+            }
+
+            //  变更item
+            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
+                items = scoreRankerItems;
+            } else {
+                items = new ArrayList<>(items);
+            }
+
+            int position = 0;
+            for (AdRankItem item : items) {
+                item.getRankerIndex().put(scorerName, position++);
+                item.getRankerScore().put(scorerName, item.getScore());
+            }
+
+            //
+            long spentTime = System.currentTimeMillis() - startTime;
+            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
+                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
+        }
+
+        int position = 0;
+        for (AdRankItem item : items) {
+            item.getRankerIndex().put("finalScore", position++);
+            item.getRankerScore().put("finalScore", item.getScore());
+        }
+
+        return items;
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerUtils.java

@@ -0,0 +1,146 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+
+import com.typesafe.config.Config;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+@Slf4j
+public final class ScorerUtils {
+    private static Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
+
+    private static Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
+
+    public static String BASE_CONF = "feeds_score_config_baseline.conf";
+
+    public static void warmUp() {
+        log.info("scorer warm up ");
+        ScorerUtils.init(BASE_CONF);
+    }
+
+    private ScorerUtils() {
+        // init(BASE_CONF);
+    }
+
+    public static void init(String configFile) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(configFile);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+
+        initLoadModel(scorers);
+        scorerPipelineCache.put(configFile, new ScorerPipeline(scorers));
+    }
+
+    /**
+     * init load model
+     *
+     * @param scorers
+     */
+    public static void initLoadModel(List<AbstractScorer> scorers) {
+        for (AbstractScorer scorer : scorers) {
+            if (scorer.isEnable()) {
+                scorer.loadModel();
+            }
+        }
+    }
+
+    public static void initLoadModel(String configFile) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(configFile);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        initLoadModel(scorers);
+    }
+
+    public static void initLoadModel(Config config) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(config);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        initLoadModel(scorers);
+    }
+
+    public static ScorerPipeline getScorerPipeline(String configFile) {
+        // 不需要保证严格意义的单例
+        if (scorerPipelineCache.containsKey(configFile)) {
+            return scorerPipelineCache.get(configFile);
+        }
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(configFile);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        ScorerPipeline pipeline = new ScorerPipeline(scorers);
+        scorerPipelineCache.put(configFile, pipeline);
+        return pipeline;
+    }
+
+    public static ScorerPipeline getScorerPipeline(Config mergeConfig) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(mergeConfig);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        return new ScorerPipeline(scorers);
+    }
+
+    public static ScorerPipeline getScorerPipeline(Config baselineConfig, List<Config> configList) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(baselineConfig, configList);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        return new ScorerPipeline(scorers);
+    }
+
+    /**
+     * construct scorers
+     *
+     * @param scorerConfig
+     * @return
+     */
+    public static List<AbstractScorer> constructScorers(ScorerConfig scorerConfig) {
+
+        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
+        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
+            if (!configInfo.getDisableSwitch()) {
+                try {
+                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
+                            .getConstructor(ScorerConfigInfo.class)
+                            .newInstance(configInfo);
+                    scorers.add(scorer);
+                    LOGGER.debug("construct score [{}]", configInfo.getScorerName());
+                } catch (Exception e) {
+                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
+                }
+
+            }
+        }
+        return scorers;
+    }
+
+    /**
+     * construct scorers
+     *
+     * @param scorerConfig
+     * @return
+     */
+    public static List<AbstractScorer> constructQueueScorers(ScorerConfig scorerConfig, String queueName) {
+
+        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
+        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
+            if (!configInfo.getDisableSwitch() && configInfo.isQueueEnable(queueName)) {
+                try {
+                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
+                            .getConstructor(ScorerConfigInfo.class)
+                            .newInstance(configInfo);
+                    scorers.add(scorer);
+                    LOGGER.debug("construct queue scorer [{}] [{}]", queueName, configInfo.getScorerName());
+                } catch (Exception e) {
+                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        return scorers;
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/BytesGroup.java

@@ -0,0 +1,35 @@
+package com.tzld.piaoquan.ad.engine.commons.score.feature;
+
+
+public class BytesGroup {
+    private int id;
+    private String name;
+    private byte[] nameBytes;
+    private byte[] buffer;
+
+    public BytesGroup(int id, String name, byte[] nameBytes) {
+        this.id = id;
+        this.name = name;
+        this.nameBytes = nameBytes;
+    }
+
+    public int getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public byte[] getNameBytes() {
+        return nameBytes;
+    }
+
+    public byte[] getBuffer() {
+        return buffer;
+    }
+
+    public void setBuffer(byte[] buffer) {
+        this.buffer = buffer;
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/BytesUtils.java

@@ -0,0 +1,192 @@
+package com.tzld.piaoquan.ad.engine.commons.score.feature;
+
+
+import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Extract features from user, item & context info. Returns 64-bit murmurhash of feature string as results.
+ */
+public class BytesUtils {
+    private static final byte[] SEPARATOR = "_".getBytes();
+    private static final byte[] FEATURE_SEPARATOR = "#".getBytes();
+    private static final int MAX_FEATURE_BYTES_LENGTH = 512;
+    private static final long SEED = 11L;
+    private BytesGroup[] groups;
+
+    /**
+     * 一个种特殊的List,在尝试写入null的时候回默默地扔掉.
+     * @param <E> List的元素类型.
+     */
+    public static class NullRejectingArrayList<E> extends ArrayList<E> {
+        public NullRejectingArrayList(int capacity) {
+            super(capacity);
+        }
+
+        public NullRejectingArrayList() {
+            super();
+        }
+
+        @Override
+        public boolean add(E e) {
+            return e != null && super.add(e);
+        }
+    }
+
+    public BytesUtils(BytesGroup[] groups) {
+        this.groups = groups;
+        for (BytesGroup g : groups) {
+            byte[] buffer = prepareBuffer(g.getName(), g.getNameBytes());
+            groups[g.getId()].setBuffer(buffer);
+        }
+    }
+
+    public byte[] prepareBuffer(String name, byte[] nameBytes) {
+
+        byte[] buffer = new byte[MAX_FEATURE_BYTES_LENGTH];
+        System.arraycopy(nameBytes, 0, buffer, 0, nameBytes.length);
+        System.arraycopy(FEATURE_SEPARATOR, 0, buffer, nameBytes.length, 1);
+        return buffer;
+    }
+
+    public BaseFeature baseFea(byte[] buffer, int length) {
+        long hash = FeatureHash.MurmurHash64(buffer, 0, length, SEED);
+
+        // debug中查看 String fea = new String(buffer, 0, length);
+        // 初始化protobuf并赋值
+        BaseFeature.Builder tmp = BaseFeature.newBuilder();
+        tmp.setIdentifier(hash);
+        return tmp.build();
+    }
+
+    public BaseFeature makeFea(int id, byte[] value) {
+        byte[] buffer = groups[id].getBuffer();
+        if (buffer == null || value == null) {
+            return null;
+        }
+
+        final int nameLength = groups[id].getNameBytes().length + 1;
+        final int length = nameLength + value.length;
+        System.arraycopy(value, 0, buffer, nameLength, value.length);
+        return baseFea(buffer, length);
+    }
+
+    public BaseFeature makeFea(int id, final byte[] p1, final byte[] p2) {
+        byte[] buffer = groups[id].getBuffer();
+        if (buffer == null || p1 == null || p2 == null) {
+            return null;
+        }
+
+        final int nameLength = groups[id].getNameBytes().length + 1;
+        final int length = nameLength + p1.length + 1 + p2.length;
+
+        System.arraycopy(p1, 0, buffer, nameLength, p1.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length, 1);
+        System.arraycopy(p2, 0, buffer, nameLength + p1.length + 1, p2.length);
+        return baseFea(buffer, length);
+    }
+
+    public BaseFeature makeFea(int id, final byte[] p1, final byte[] p2, final byte[] p3) {
+        byte[] buffer = groups[id].getBuffer();
+        if (buffer == null || p1 == null || p2 == null || p3 == null) {
+            return null;
+        }
+
+        final int nameLength = groups[id].getNameBytes().length + 1;
+        final int length = nameLength + p1.length + 1 + p2.length + 1 + p3.length;
+        System.arraycopy(p1, 0, buffer, nameLength, p1.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length, 1);
+        System.arraycopy(p2, 0, buffer, nameLength + p1.length + 1, p2.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length + 1 + p2.length, 1);
+        System.arraycopy(p3, 0, buffer, nameLength + p1.length + 1 + p2.length + 1, p3.length);
+
+        return baseFea(buffer, length);
+    }
+
+    public BaseFeature makeFea(int id, final byte[] p1, final byte[] p2, final byte[] p3, final byte[] p4) {
+        byte[] buffer = groups[id].getBuffer();
+        if (buffer == null || p1 == null || p2 == null || p3 == null || p4 == null) {
+            return null;
+        }
+
+        final int nameLength = groups[id].getNameBytes().length + 1;
+        final int length = nameLength + p1.length + 1 + p2.length + 1 + p3.length + 1 + p4.length;
+        System.arraycopy(p1, 0, buffer, nameLength, p1.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length, 1);
+        System.arraycopy(p2, 0, buffer, nameLength + p1.length + 1, p2.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length + 1 + p2.length, 1);
+        System.arraycopy(p3, 0, buffer, nameLength + p1.length + 1 + p2.length + 1, p3.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length + 1 + p2.length + 1 + p3.length, 1);
+        System.arraycopy(p4, 0, buffer, nameLength + p1.length + 1 + p2.length + 1 + p3.length + 1, p4.length);
+
+        return baseFea(buffer, length);
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[][] list) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(list.length);
+        for (byte[] t: list) {
+            result.add(makeFea(id, t));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[][] left, byte[] right) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length);
+        for (byte[] l: left) {
+            result.add(makeFea(id, l, right));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[][] left, byte[] right1, byte[] right2) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length);
+        for (byte[] l: left) {
+            result.add(makeFea(id, l, right1, right2));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[][] left, byte[] right1, byte[] right2, byte[] right3) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length);
+        for (byte[] l: left) {
+            result.add(makeFea(id, l, right1, right2, right3));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[] left, byte[][] right) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(right.length);
+        for (byte[] r : right) {
+            result.add(makeFea(id, left, r));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[] left1, byte[] left2, byte[][] right) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(right.length);
+        for (byte[] r : right) {
+            result.add(makeFea(id, left1, left2, r));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[] left1, byte[] left2, byte[] left3, byte[][] right) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(right.length);
+        for (byte[] r : right) {
+            result.add(makeFea(id, left1, left2, left3, r));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[][] left, byte[][] right) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length * right.length);
+        for (byte[] l: left) {
+            for (byte[] r: right) {
+                result.add(makeFea(id, l, r));
+            }
+        }
+        return result;
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/FeatureHash.java

@@ -0,0 +1,230 @@
+package com.tzld.piaoquan.ad.engine.commons.score.feature;
+
+
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+
+public class FeatureHash {
+    public static Charset CharSetUTF8 = Charset.forName("UTF-8");
+
+    public static long getUInt32(byte a, byte b, byte c, byte d) {
+        return (d << 24 | (c & 0xFF) << 16 | (b & 0xFF) << 8 | (a & 0xFF));
+    }
+
+    public static long hash64(byte[] data) {
+        return MurmurHash64A(ByteBuffer.wrap(data), 0, data.length, 11L);
+    }
+
+    public static long MurmurHash64A(ByteBuffer buffer, int from, int len, long seed) {
+        final long m = 0xc6a4a7935bd1e995L;
+        final int r = 47;
+
+        long h = (seed) ^ (len * m);
+        int longLength = len / 8;
+
+        for (int i = 0; i < longLength; ++i) {
+            final int bytePos = from + i * 8;
+            long k = buffer.getLong(bytePos);
+
+            k *= m;
+            k ^= k >> r;
+            k *= m;
+            h ^= k;
+            h *= m;
+        }
+
+        final int remainingPos = len & ~7;
+        switch (len % 8) {
+            case 7: h ^= (long)(buffer.get(remainingPos + 6) & 0xFF) << 48;
+            case 6: h ^= (long)(buffer.get(remainingPos + 5) & 0xFF) << 40;
+            case 5: h ^= (long)(buffer.get(remainingPos + 4) & 0xFF) << 32;
+            case 4: h ^= (long)(buffer.get(remainingPos + 3) & 0xFF) << 24;
+            case 3: h ^= (long)(buffer.get(remainingPos + 2) & 0xFF) << 16;
+            case 2: h ^= (long)(buffer.get(remainingPos + 1) & 0xFF) << 8;
+            case 1:
+                h ^= (long)(buffer.get(remainingPos) & 0xFF);
+                h *= m;
+        }
+
+        h ^= h >>> r;
+        h *= m;
+        h ^= h >>> r;
+        return h;
+    }
+
+    public static long MurmurHash32(byte data[], int len, long seed) {
+        long m = 0x5bd1e995L;
+        int r = 24;
+
+        long h = seed ^ len;
+
+        int offset = 0;
+        while (len >= 4) {
+            long k = getUInt32(data[offset], data[offset + 1], data[offset + 2], data[offset + 3]);
+
+            k *= m;
+            k &= 0xFFFFFFFFL;
+            k ^= k >> r;
+            k *= m;
+            k &= 0xFFFFFFFFL;
+
+            h *= m;
+            h &= 0xFFFFFFFFL;
+            h ^= k;
+
+            offset += 4;
+            len -= 4;
+        }
+
+        // Handle the last few bytes of the input array
+        switch (len) {
+            case 3: h ^= data[offset + 2] << 16;
+            case 2: h ^= data[offset + 1] << 8;
+            case 1: h ^= data[offset];
+                h *= m;
+                h &= 0xFFFFFFFFL;
+        } ;
+
+        // Do a few final mixes of the hash to ensure the last few
+        // bytes are well-incorporated.
+
+        h ^= h >> 13;
+        h *= m;
+        h &= 0xFFFFFFFFL;
+        h ^= h >> 15;
+
+        return h;
+    }
+
+    // 64-bit hash for 32-bit platforms
+    public static long MurmurHash64(byte[] buffer, int start, int len, long seed) {
+        final long m = 0x5bd1e995L;
+        final int r = 24;
+        final int original = len;
+
+        long h1 = (seed) ^ len;
+        long h2 = (seed >> 32);
+
+        int offset = start;
+        while (len >= 8) {
+            long k1 = getUInt32(buffer[offset], buffer[offset + 1], buffer[offset + 2], buffer[offset + 3]);
+            // long k1 = buffer.getInt(offset);
+
+            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
+            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
+            offset += 4;
+
+            long k2 = getUInt32(buffer[offset], buffer[offset + 1], buffer[offset + 2], buffer[offset + 3]);
+            // long k2 = buffer.getInt(offset);
+            k2 *= m; k2 &= 0xFFFFFFFFL; k2 ^= k2 >> r; k2 *= m; k2 &= 0xFFFFFFFFL;
+            h2 *= m; h2 &= 0xFFFFFFFFL; h2 ^= k2;
+
+            offset += 4;
+            len -= 8;
+        }
+
+        if (len >= 4) {
+            long k1 = getUInt32(buffer[offset], buffer[offset + 1], buffer[offset + 2], buffer[offset + 3]);
+            // long k1 = buffer.getInt(offset);
+            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
+            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
+            offset += 4;
+            len -= 4;
+        }
+
+        switch (len) {
+            case 3: h2 ^= (buffer[offset + 2] & 0xFF) << 16;
+            case 2: h2 ^= (buffer[offset + 1] & 0xFF) << 8;
+            case 1: h2 ^= (buffer[offset] & 0xFF);
+                h2 *= m;
+                h2 &= 0xFFFFFFFFL;
+        } ;
+
+        h1 ^= h2 >> 18;
+        h1 *= m; h1 &= 0xFFFFFFFFL;
+        h2 ^= h1 >> 22;
+        h2 *= m; h2 &= 0xFFFFFFFFL;
+        h1 ^= h2 >> 17;
+        h1 *= m; h1 &= 0xFFFFFFFFL;
+        h2 ^= h1 >> 19;
+        h2 *= m; h2 &= 0xFFFFFFFFL;
+
+        /*BigInteger ans = BigInteger.valueOf(h1).shiftLeft(32).or(BigInteger.valueOf(h2));
+        return ans.longValue();*/
+        //System.err.println("feature: " + new String(buffer, 0, original) + " length: " + original + " hash: " + (h1 << 32 | h2) + " daze");
+        return h1 << 32 | h2;
+    }
+
+    // 64-bit hash for 32-bit platforms
+    public static BigInteger MurmurHash64(byte data[], int len, long seed) {
+        long m = 0x5bd1e995L;
+        int r = 24;
+
+        long h1 = (seed) ^ len;
+        long h2 = (seed >> 32);
+
+        int offset = 0;
+        while (len >= 8) {
+            long k1 = getUInt32(data[offset], data[offset + 1], data[offset + 2], data[offset + 3]);
+            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
+            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
+
+            long k2 = getUInt32(data[offset + 4], data[offset + 5], data[offset + 6], data[offset + 7]);
+            k2 *= m; k2 &= 0xFFFFFFFFL; k2 ^= k2 >> r; k2 *= m; k2 &= 0xFFFFFFFFL;
+            h2 *= m; h2 &= 0xFFFFFFFFL; h2 ^= k2;
+
+            offset += 8;
+            len -= 8;
+        }
+
+        if (len >= 4) {
+            long k1 = getUInt32(data[offset], data[offset + 1], data[offset + 2], data[offset + 3]);
+            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
+            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
+            offset += 4;
+            len -= 4;
+        }
+
+        switch (len) {
+            case 3: h2 ^= (data[offset + 2] & 0xFF) << 16;
+            case 2: h2 ^= (data[offset + 1] & 0xFF) << 8;
+            case 1: h2 ^= (data[offset] & 0xFF);
+                h2 *= m;
+                h2 &= 0xFFFFFFFFL;
+        } ;
+
+        h1 ^= h2 >> 18;
+        h1 *= m; h1 &= 0xFFFFFFFFL;
+        h2 ^= h1 >> 22;
+        h2 *= m; h2 &= 0xFFFFFFFFL;
+        h1 ^= h2 >> 17;
+        h1 *= m; h1 &= 0xFFFFFFFFL;
+        h2 ^= h1 >> 19;
+        h2 *= m; h2 &= 0xFFFFFFFFL;
+
+        BigInteger ans = BigInteger.valueOf(h1).shiftLeft(32).or(BigInteger.valueOf(h2));
+        return ans;
+    }
+
+    public static String hash(String input) {
+        byte[] tt = input.getBytes(CharSetUTF8);
+        return MurmurHash64(tt, tt.length, 11L).toString();
+    }
+
+    public static Long hashToLong(String input) {
+        byte[] tt = input.getBytes(CharSetUTF8);
+        return MurmurHash64(tt, tt.length, 11L).longValue();
+    }
+
+    /** the constant 2^64 */
+    private static final BigInteger TWO_64 = BigInteger.ONE.shiftLeft(64);
+
+    public static String asUnsignedLongString(long l) {
+        BigInteger b = BigInteger.valueOf(l);
+        if (b.signum() < 0) {
+            b = b.add(TWO_64);
+        }
+        return b.toString();
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/FeatureUsage.java

@@ -0,0 +1,43 @@
+package com.tzld.piaoquan.ad.engine.commons.score.feature;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.Set;
+
+public class FeatureUsage implements Serializable {
+    private final Logger LOGGER = LoggerFactory.getLogger(FeatureUsage.class);
+    private Set<String> hash;
+
+    public FeatureUsage() {
+        hash = new HashSet<String>();
+    }
+
+    public FeatureUsage(String confName) {
+        // note that these fields are NOT lazy, because if we're going to get any exceptions, we want to get them on startup.
+        Config config = ConfigFactory.load(confName);
+        config.checkValid(config, "features");
+        ConfigObject conf = config.getConfig("features").root();
+        LOGGER.info("create_feature_usage: " + confName);
+        hash = new HashSet<String>();
+        for (ConfigObject.Entry<String, ConfigValue> it : conf.entrySet()) {
+            String key = it.getKey();
+            Config cf = ((ConfigObject) it.getValue()).toConfig();
+            LOGGER.info("key: " + key + ", value: " + cf.toString());
+            boolean disable = cf.getBoolean("disable");
+            if (disable) {
+                hash.add(key);
+            }
+        }
+    }
+
+    public boolean disable(String fea) {
+        return hash.contains(fea);
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/LRBytesFeatureExtractorBase.java

@@ -0,0 +1,67 @@
+package com.tzld.piaoquan.ad.engine.commons.score.feature;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.tzld.piaoquan.ad.engine.commons.base.AdItemBytesFeature;
+import com.tzld.piaoquan.ad.engine.commons.base.RequestContextBytesFeature;
+import com.tzld.piaoquan.ad.engine.commons.base.UserAdBytesFeature;
+import com.tzld.piaoquan.ad.engine.commons.enums.VlogAdFeatureGroup;
+import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
+import com.tzld.piaoquan.recommend.server.gen.recommend.FeatureGroup;
+import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
+
+import java.util.List;
+
+
+public abstract class LRBytesFeatureExtractorBase {
+    private static final double DEFAULT_USER_CTR_GROUP = 10.0;
+    private static final double DEFAULT_ARTICLE_CTR_GROUP = 100.0;
+
+
+    private BytesUtils utils;
+    //Feature Group & Features
+    ListMultimap<FeatureGroup, BaseFeature> features = ArrayListMultimap.create();
+    int groupCount;
+
+
+    LRBytesFeatureExtractorBase() {
+        groupCount = VlogAdFeatureGroup.values().length;
+        BytesGroup[] groups = new BytesGroup[groupCount];
+        for (VlogAdFeatureGroup g: VlogAdFeatureGroup.values()) {
+            groups[g.ordinal()] = new BytesGroup(g.ordinal(),
+                    g.getGroupName(), g.getGroupNameBytes());
+        }
+        utils = new BytesUtils(groups);
+    }
+
+    private FeatureGroup makeGroup(VlogAdFeatureGroup group){
+        FeatureGroup.Builder g = FeatureGroup.newBuilder();
+        g.setType("1");
+        g.setName(group.getGroupName());
+        g.setId(group.ordinal());
+        return g.build();
+    };
+
+
+    void makeFea(VlogAdFeatureGroup group, byte[] value) {
+        FeatureGroup featureGroup = makeGroup(group);
+        BaseFeature feature = utils.makeFea(group.ordinal(), value);
+        features.put(featureGroup, feature);
+    }
+
+    void makeFea(VlogAdFeatureGroup group, byte[][] list) {
+        FeatureGroup g = makeGroup(group);
+        List<BaseFeature> featureList = utils.makeFea(group.ordinal(), list);
+        features.putAll(g, featureList);
+    }
+
+    public ListMultimap<FeatureGroup, BaseFeature> getFeatures() {
+        return features;
+    }
+
+    public abstract LRSamples single(UserAdBytesFeature userBytesFeature,
+                                     AdItemBytesFeature videoBytesFeature,
+                                     RequestContextBytesFeature requestContextBytesFeature);
+
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/VlogAdCtrLRFeatureExtractor.java

@@ -0,0 +1,135 @@
+package com.tzld.piaoquan.ad.engine.commons.score.feature;
+
+import com.tzld.piaoquan.ad.engine.commons.base.AdItemBytesFeature;
+import com.tzld.piaoquan.ad.engine.commons.base.RequestContextBytesFeature;
+import com.tzld.piaoquan.ad.engine.commons.base.UserAdBytesFeature;
+import com.tzld.piaoquan.ad.engine.commons.enums.VlogAdFeatureGroup;
+import com.tzld.piaoquan.recommend.server.gen.recommend.FeatureGroup;
+import com.tzld.piaoquan.recommend.server.gen.recommend.GroupedFeature;
+import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
+import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
+
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class VlogAdCtrLRFeatureExtractor extends LRBytesFeatureExtractorBase {
+
+    public VlogAdCtrLRFeatureExtractor() {
+        super();
+    }
+
+    // TODO
+    // 补充待抽取的context feature
+    public void getContextFeatures(RequestContextBytesFeature requestContextBytes) {
+        makeFea(VlogAdFeatureGroup.MACHINEINFO_BRAND, requestContextBytes.getMachineinfo_brand());
+        makeFea(VlogAdFeatureGroup.MACHINEINFO_MODEL, requestContextBytes.getMachineinfo_model());
+        makeFea(VlogAdFeatureGroup.MACHINEINFO_PLATFORM, requestContextBytes.getMachineinfo_platform());
+        makeFea(VlogAdFeatureGroup.MACHINEINFO_SDKVERSION, requestContextBytes.getMachineinfo_sdkversion());
+        makeFea(VlogAdFeatureGroup.MACHINEINFO_SYSTEM, requestContextBytes.getMachineinfo_system());
+        makeFea(VlogAdFeatureGroup.MACHINEINFO_WECHATVERSION, requestContextBytes.getMachineinfo_brand());
+
+        makeFea(VlogAdFeatureGroup.DAY, requestContextBytes.getWeek());
+        makeFea(VlogAdFeatureGroup.WEEK, requestContextBytes.getWeek());
+        makeFea(VlogAdFeatureGroup.HOUR, requestContextBytes.getHour());
+
+    }
+
+    //TODO
+    public void getUserFeatures(UserAdBytesFeature userAdBytesFeature) {
+
+        // 1day features
+        makeFea(VlogAdFeatureGroup.USER_1DAY_EXP, userAdBytesFeature.getDay1_cnt_features().get("view"));
+        makeFea(VlogAdFeatureGroup.USER_1DAY_CLICK, userAdBytesFeature.getDay1_cnt_features().get("click"));
+        makeFea(VlogAdFeatureGroup.USER_1DAY_SHARE, userAdBytesFeature.getDay1_cnt_features().get("ctr"));
+        makeFea(VlogAdFeatureGroup.USER_1DAY_RETURN, userAdBytesFeature.getDay1_cnt_features().get("cvr"));
+
+        // 3day features
+        makeFea(VlogAdFeatureGroup.USER_3DAY_EXP, userAdBytesFeature.getDay3_cnt_features().get("view"));
+        makeFea(VlogAdFeatureGroup.USER_3DAY_CLICK, userAdBytesFeature.getDay3_cnt_features().get("click"));
+        makeFea(VlogAdFeatureGroup.USER_3DAY_SHARE, userAdBytesFeature.getDay3_cnt_features().get("ctr"));
+        makeFea(VlogAdFeatureGroup.USER_3DAY_RETURN, userAdBytesFeature.getDay3_cnt_features().get("cvr"));
+
+        // 7day features
+        makeFea(VlogAdFeatureGroup.USER_7DAY_EXP, userAdBytesFeature.getDay7_cnt_features().get("view"));
+        makeFea(VlogAdFeatureGroup.USER_7DAY_CLICK, userAdBytesFeature.getDay7_cnt_features().get("click"));
+        makeFea(VlogAdFeatureGroup.USER_7DAY_SHARE, userAdBytesFeature.getDay7_cnt_features().get("ctr"));
+        makeFea(VlogAdFeatureGroup.USER_7DAY_RETURN, userAdBytesFeature.getDay7_cnt_features().get("cvr"));
+
+        // 3month features
+        makeFea(VlogAdFeatureGroup.USER_3MONTH_EXP, userAdBytesFeature.getMonth3_cnt_features().get("view"));
+        makeFea(VlogAdFeatureGroup.USER_3MONTH_CLICK, userAdBytesFeature.getMonth3_cnt_features().get("click"));
+        makeFea(VlogAdFeatureGroup.USER_3MONTH_SHARE, userAdBytesFeature.getMonth3_cnt_features().get("ctr"));
+        makeFea(VlogAdFeatureGroup.USER_3MONTH_RETURN, userAdBytesFeature.getMonth3_cnt_features().get("cvr"));
+
+
+    }
+
+    public void getItemFeature(AdItemBytesFeature item) {
+
+        // Vdieo
+        makeFea(VlogAdFeatureGroup.ADID, item.getAdId());
+        makeFea(VlogAdFeatureGroup.CREATIVE, item.getCreative());
+        makeFea(VlogAdFeatureGroup.CAMPAIGIN, item.getCampaignId());
+        makeFea(VlogAdFeatureGroup.ADVERID, item.getAdvertiserId());
+
+
+
+        // 1day features
+        makeFea(VlogAdFeatureGroup.ITEM_1DAY_EXP, item.getDay1_cnt_features().get("view"));
+        makeFea(VlogAdFeatureGroup.ITEM_1DAY_CLICK, item.getDay1_cnt_features().get("click"));
+        makeFea(VlogAdFeatureGroup.ITEM_1DAY_SHARE, item.getDay1_cnt_features().get("ctr"));
+        makeFea(VlogAdFeatureGroup.ITEM_1DAY_RETURN, item.getDay1_cnt_features().get("cvr"));
+
+        // 3day features
+        makeFea(VlogAdFeatureGroup.ITEM_3DAY_EXP, item.getDay1_cnt_features().get("view"));
+        makeFea(VlogAdFeatureGroup.ITEM_3DAY_CLICK, item.getDay1_cnt_features().get("click"));
+        makeFea(VlogAdFeatureGroup.ITEM_3DAY_SHARE, item.getDay1_cnt_features().get("ctr"));
+        makeFea(VlogAdFeatureGroup.ITEM_3DAY_RETURN, item.getDay1_cnt_features().get("cvr"));
+
+
+        // 7day features
+        makeFea(VlogAdFeatureGroup.ITEM_7DAY_EXP, item.getDay1_cnt_features().get("view"));
+        makeFea(VlogAdFeatureGroup.ITEM_7DAY_CLICK, item.getDay1_cnt_features().get("click"));
+        makeFea(VlogAdFeatureGroup.ITEM_7DAY_SHARE, item.getDay1_cnt_features().get("ctr"));
+        makeFea(VlogAdFeatureGroup.ITEM_7DAY_RETURN, item.getDay1_cnt_features().get("cvr"));
+
+
+        // 3month features
+        makeFea(VlogAdFeatureGroup.ITEM_3MONTH_EXP, item.getMonth3_cnt_features().get("view"));
+        makeFea(VlogAdFeatureGroup.ITEM_3MONTH_CLICK, item.getMonth3_cnt_features().get("click"));
+        makeFea(VlogAdFeatureGroup.ITEM_3MONTH_SHARE, item.getMonth3_cnt_features().get("ctr"));
+        makeFea(VlogAdFeatureGroup.ITEM_3MONTH_RETURN, item.getMonth3_cnt_features().get("cvr"));
+
+    }
+
+    @Override
+    public synchronized LRSamples single(UserAdBytesFeature userAdBytesFeature,
+                                         AdItemBytesFeature videoBytesFeature,
+                                         RequestContextBytesFeature requestContextBytesFeature) {
+        features.clear();
+        // extract features
+        getUserFeatures(userAdBytesFeature);
+        getContextFeatures(requestContextBytesFeature);
+        getItemFeature(videoBytesFeature);
+
+        LRSamples.Builder lr = com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples.newBuilder();
+        lr.setGroupNum(groupCount);
+        List<FeatureGroup> keys = new ArrayList<>(features.keySet());
+        int count = 0;
+        for(FeatureGroup group : keys) {
+            List<BaseFeature> fea = features.get(group);
+            GroupedFeature.Builder gf = GroupedFeature.newBuilder();
+            gf.setGroup(group);
+            gf.setCount(fea.size());
+            gf.addAllFeatures(fea);
+            count += fea.size();
+            lr.addFeatures(gf);
+        }
+        lr.setCount(count);
+        return lr.build();
+    }
+
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/GBDTModel.java

@@ -0,0 +1,196 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.*;
+
+
+
+public class GBDTModel extends Model {
+    private static final Logger LOGGER = LoggerFactory.getLogger(GBDTModel.class);
+    private static final float LINEAR_TRANSFORM_LOWER_BOUND = 0f;
+    private static final float LINEAR_TRANSFORM_SLOPE = 1f / 1f;
+    private int featureCount = 0;
+    private Map<String, Integer> featureIdMap = null;
+    private List<HashMap<Integer, Node>> boosterModel = null;
+
+    private float transform(final float score) {
+        return (score - LINEAR_TRANSFORM_LOWER_BOUND) * LINEAR_TRANSFORM_SLOPE;
+    }
+
+    private int getFeatureId(final String feature) {
+        return featureIdMap.containsKey(feature) ? featureIdMap.get(feature) : -1;
+    }
+
+    @Override
+    public int getModelSize() {
+        return boosterModel.size();
+    }
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws Exception {
+        List<HashMap<Integer, Node>> model = new ArrayList<HashMap<Integer, Node>>();
+        HashMap<Integer, Node> tree = null;
+
+        featureIdMap = new HashMap<String, Integer>();
+        featureCount = 0;
+
+        BufferedReader input = new BufferedReader(in);
+        String line;
+        while ((line = input.readLine()) != null) {
+            String[] tokens = line.trim().split(":");
+            if (tokens.length == 1) {
+                if (tokens[0].startsWith("booster")) {
+                    if (tree != null && tree.size() > 0) {
+                        model.add(tree);
+                    }
+
+                    tree = new HashMap<Integer, Node>();
+                }
+            } else if (tokens.length == 2) {
+                Node node = new Node();
+
+                Integer id = Integer.parseInt(tokens[0]);
+
+                String[] items = tokens[1].split(" ");
+                if (items.length == 1) {
+                    node.isLeaf = true;
+
+                    String[] dt = items[0].split("=");
+                    node.value = Float.parseFloat(dt[1]);
+                } else if (items.length == 2) {
+                    node.isLeaf = false;
+                    String[] fieldDescriptions = null;
+                    node.compareType = NodeCompareType.LT;
+                    if (items[0].substring(1, items[0].length() - 1).contains("<=")) {
+                        fieldDescriptions = items[0].substring(1, items[0].length() - 1).split("<=");
+                        node.compareType = NodeCompareType.LE;
+                    } else {
+                        fieldDescriptions = items[0].substring(1, items[0].length() - 1).split("<");
+                    }
+                    // feature to id
+                    int featureId = getFeatureId(fieldDescriptions[0]);
+                    if (featureId < 0) {
+                        featureIdMap.put(fieldDescriptions[0], featureCount);
+                        featureId = featureCount;
+                        featureCount += 1;
+                    }
+                    node.splitFeatureId = featureId;
+
+                    if (fieldDescriptions.length == 1) {
+                        node.fieldType = NodeFieldType.BINARY;
+                    } else {
+                        node.fieldType = NodeFieldType.QUANTITATIVE;
+                        node.splitCondition = Float.parseFloat(fieldDescriptions[1]);
+                    }
+
+                    String[] childrenDescriptions = items[1].split(",");
+                    for (String childDescription : childrenDescriptions) {
+                        String[] descs = childDescription.split("=");
+                        if (descs[0].equals("yes")) {
+                            node.yes = Integer.parseInt(descs[1]);
+                        } else if (descs[0].equals("no")) {
+                            node.no = Integer.parseInt(descs[1]);
+                        } else if (descs[0].equals("missing")) {
+                            node.missing = Integer.parseInt(descs[1]);
+                        }
+                    }
+                }
+                tree.put(id, node);
+            }
+        }
+        if (tree != null && !tree.isEmpty()) {
+            model.add(tree);
+        }
+
+        LOGGER.info("Boosted tree model load over and tree number is " + model.size());
+        input.close();
+        in.close();
+        if (model != null && model.size() > 0) {
+            boosterModel = model;
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private float score(final Map<Integer, Node> tree, final boolean[] featureIsExists, final Float[] featureValues) {
+        int id = 0;
+        while (true) {
+            Node currentNode = tree.get(id);
+            if (currentNode.isLeaf) {
+                return currentNode.value;
+            }
+
+            if (currentNode.fieldType == NodeFieldType.BINARY) {
+                if (!featureIsExists[currentNode.splitFeatureId]) {
+                    id = currentNode.no;
+                } else {
+                    id = currentNode.yes;
+                }
+            } else if (currentNode.fieldType == NodeFieldType.QUANTITATIVE) {
+                if (!featureIsExists[currentNode.splitFeatureId]) {
+                    id = currentNode.missing;
+                } else {
+                    Float value = featureValues[currentNode.splitFeatureId];
+
+                    if ((currentNode.compareType == NodeCompareType.LT && value < currentNode.splitCondition)
+                            || (currentNode.compareType == NodeCompareType.LE && value <= currentNode.splitCondition)) {
+                        id = currentNode.yes;
+                    } else {
+                        id = currentNode.no;
+                    }
+                }
+            } else {
+                LOGGER.error("Reach undefined condition: {}", id);
+            }
+        }
+    }
+
+    public Float score(final Map<String, Double> features, Map<String, Double> featuresScore) {
+        Float result = 0f;
+
+        boolean[] featureIsExists = new boolean[featureCount];
+        Float[] featureValues = new Float[featureCount];
+        for (Map.Entry<String, Double> entry : features.entrySet()) {
+            int featureId = getFeatureId(entry.getKey());
+            if (featureId >= 0) {
+                featureIsExists[featureId] = true;
+                featureValues[featureId] = entry.getValue().floatValue();
+            }
+        }
+
+        for (Map<Integer, Node> tree : boosterModel) {
+            result += score(tree, featureIsExists, featureValues);
+        }
+
+        Float transformedResult = transform(result);
+
+        LOGGER.debug("[calc_dwelltime]features: " + Arrays.toString(features.entrySet().toArray()) + ", prediction:" + result + " , transformed:" + transformedResult);
+
+        return transformedResult;
+    }
+
+    private enum NodeFieldType {
+        BINARY, QUANTITATIVE
+    }
+
+    private enum NodeCompareType {
+        LT, LE
+    }
+
+    private class Node {
+
+        boolean isLeaf;
+        float value;
+        NodeFieldType fieldType;
+        NodeCompareType compareType;
+        int splitFeatureId;
+        Float splitCondition;
+        Integer yes, no, missing;
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/LRModel.java

@@ -0,0 +1,168 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
+import com.tzld.piaoquan.recommend.server.gen.recommend.GroupedFeature;
+import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
+import it.unimi.dsi.fastutil.longs.Long2FloatMap;
+import it.unimi.dsi.fastutil.longs.Long2FloatOpenHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+
+public class LRModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(LRModel.class);
+    private final int bucketBits = 10;  // power(2, 10) => 1024 个槽位
+    private List<Long2FloatMap> lrModel;
+
+    public LRModel() {
+        //配置不同环境的hdfs conf
+        this.lrModel = constructModel();
+    }
+
+    public List<Long2FloatMap> getLrModel() {
+        return lrModel;
+    }
+
+    public List<Long2FloatMap> constructModel() {
+        List<Long2FloatMap> initModel = new ArrayList<Long2FloatMap>();
+        int buckets = (int) Math.pow(2, bucketBits);
+        for (int i = 0; i < buckets; i++) {
+            Long2FloatMap internalModel = new Long2FloatOpenHashMap();
+            internalModel.defaultReturnValue(0.0f);
+            initModel.add(internalModel);
+        }
+        return initModel;
+    }
+
+    public int getBucket(long featureHash) {
+        return (int) (((featureHash >> bucketBits) << bucketBits) ^ featureHash);
+    }
+
+    public void putFeature(List<Long2FloatMap> model, long featureHash, float weight) {
+        model.get(getBucket(featureHash)).put(featureHash, weight);
+    }
+
+    public float getWeight(List<Long2FloatMap> model, long featureHash) {
+        return model.get(getBucket(featureHash)).get(featureHash);
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.lrModel == null)
+            return 0;
+        int sum = 0;
+        for (Map<Long, Float> model : this.lrModel) {
+            sum += model.size();
+        }
+        return sum;
+    }
+
+    public void cleanModel() {
+        this.lrModel = null;
+    }
+
+    public Float score(LRSamples lrSamples) {
+        float sum = 0.0f;
+        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
+            GroupedFeature groupedFeature = lrSamples.getFeatures(i);
+            if (groupedFeature != null && groupedFeature.getFeaturesCount() != 0) {
+                for (int j = 0; j < groupedFeature.getFeaturesCount(); j++) {
+                    BaseFeature baseFeature = groupedFeature.getFeatures(j);
+                    if (baseFeature != null) {
+                        float weight = getWeight(this.lrModel, baseFeature.getIdentifier());
+                        baseFeature.toBuilder().setWeight(weight);
+                        sum += weight;
+                    }
+                }
+            }
+        }
+
+        float pro = (float) (1.0f / (1 + Math.exp(-sum)));
+        lrSamples.toBuilder().setPredictCtr(pro);
+        return pro;
+    }
+
+    public Float getWeights(LRSamples lrSamples) {
+        float sum = 0.0f;
+
+        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
+            GroupedFeature gf = lrSamples.getFeatures(i);
+            if (gf != null && gf.getFeatures(i) != null) {
+                for (int j = 0; j < gf.getFeaturesCount(); j++) {
+                    BaseFeature fea = gf.getFeatures(j);
+                    if (fea != null) {
+                        float tmp = getWeight(this.lrModel, fea.getIdentifier());
+                        fea.toBuilder().setWeight(tmp);
+                        sum += tmp;
+                    }
+                }
+            }
+        }
+        lrSamples.toBuilder().setWeight(sum);
+        return sum;
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        List<Long2FloatMap> model = constructModel();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+
+            putFeature(model, new BigInteger(items[0].trim()).longValue(), Float.valueOf(items[1].trim()).floatValue());
+            if (cnt++ < 10) {
+                LOGGER.debug("fea: " + items[0] + ", weight: " + items[1]);
+            }
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.lrModel = model;
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+            putFeature(model, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/Model.java

@@ -0,0 +1,11 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+
+import java.io.InputStreamReader;
+
+abstract public class Model {
+    public abstract int getModelSize();
+
+    public abstract boolean loadFromStream(InputStreamReader in) throws Exception;
+}
+

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/ModelManager.java

@@ -0,0 +1,243 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.OSSClientBuilder;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.auth.DefaultCredentialProvider;
+import com.aliyun.oss.model.OSSObject;
+import com.ctrip.framework.apollo.Config;
+import com.ctrip.framework.apollo.ConfigService;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+
+@Slf4j
+public class ModelManager {
+    private static final int SCHEDULE_PERIOD = 10;
+    private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+    private static ModelManager instance;
+    Map<String, ModelLoadTask> loadTasks = new HashMap<>();
+    Map<String, String> modelPathMap = new HashMap<>();
+    private OSS client;
+    private String bucketName;
+
+    private final String modelOssEndpoint = "model.oss.internal.endpoint";
+    private final String modelOssAccessKeyId = "model.oss.accessKeyId";
+    private final String modelOssAccessKeySecret = "model.oss.accessKetSecret";
+    private final String modelOssBucketName = "model.oss.bucketName";
+
+    private ModelManager() {
+        // config load
+        Config config = ConfigService.getAppConfig();
+        String endpoint = config.getProperty(modelOssEndpoint, "");
+        String accessKeyId = config.getProperty(modelOssAccessKeyId, "");
+        String accessKetSecret = config.getProperty(modelOssAccessKeySecret, "");
+        // oss client
+        CredentialsProvider credentialsProvider = new DefaultCredentialProvider(accessKeyId, accessKetSecret);
+        this.client = new OSSClientBuilder().build(endpoint, credentialsProvider);
+        this.bucketName = config.getProperty(modelOssBucketName, "");
+
+        config.addChangeListener(changeEvent -> {
+            if (changeEvent.isChanged(modelOssEndpoint)
+                    || changeEvent.isChanged(modelOssAccessKeyId)
+                    || changeEvent.isChanged(modelOssAccessKeySecret)) {
+                String endpointNew = config.getProperty(modelOssEndpoint, "");
+                String accessKeyIdNew = config.getProperty(modelOssAccessKeyId, "");
+                String accessKetSecretNew = config.getProperty(modelOssAccessKeySecret, "");
+                CredentialsProvider credentialsProviderNew = new DefaultCredentialProvider(accessKeyIdNew,
+                        accessKetSecretNew);
+                this.client = new OSSClientBuilder().build(endpointNew, credentialsProviderNew);
+            }
+            if (changeEvent.isChanged(modelOssBucketName)) {
+                this.bucketName = config.getProperty(modelOssBucketName, "");
+            }
+        });
+
+
+        start(SCHEDULE_PERIOD);
+    }
+
+    public static ModelManager getInstance() {
+        if (instance == null) {
+            synchronized (ModelManager.class) {
+                if (instance == null) {
+                    instance = new ModelManager();
+                }
+            }
+        }
+        return instance;
+    }
+
+    /**
+     * 添加一个加载任务到管理器
+     *
+     * @param modelName  Model的名字, 注册到ModelManager的不同model需要不同的名字
+     * @param path       Model在OSS上的全路径
+     * @param modelClass Model的子类型
+     */
+    public void registerModel(String modelName, String path, Class<? extends Model> modelClass) throws ModelRegisterException, IOException {
+        if (modelPathMap.containsKey(modelName)) {
+            // fail fast
+            throw new RuntimeException(modelName + " already exists");
+//            String oldPath = modelPathMap.get(modelName);
+//            if (path.equals(oldPath)) {
+//                //如果模型的path没有发生改变, 不做任何操作
+//                log.info("Model [{}] and Path [{}] has exist", modelName, path);
+//                return;
+//            } else {
+//                //如果模型的path发生改变, 需要注销掉原有的任务
+//                unRegisterModel(modelName);
+//            }
+        }
+
+        modelPathMap.put(modelName, path);
+        if (loadTasks.containsKey(path)) {
+            ModelLoadTask loadTask = loadTasks.get(path);
+            loadTask.refCount++;
+        } else {
+            ModelLoadTask task = new ModelLoadTask(path, modelClass);
+            task.refCount++;
+            loadTasks.put(path, task);
+            loadModel(task, false, true);
+        }
+    }
+
+    /**
+     * 删除一个加载任务
+     *
+     * @param modelName Model的名字, 需要和registerModel的名字一致
+     */
+    private void unRegisterModel(String modelName) {
+        if (modelPathMap.containsKey(modelName)) {
+            String path = modelPathMap.get(modelName);
+            if (loadTasks.containsKey(path)) {
+                ModelLoadTask task = loadTasks.get(path);
+                task.refCount--;
+                if (task.refCount == 0) {
+                    loadTasks.remove(path);
+                }
+            }
+            modelPathMap.remove(modelName);
+        }
+    }
+
+    /**
+     * @param modelName
+     * @return
+     */
+    public Model getModel(String modelName) {
+        if (modelPathMap.containsKey(modelName) && loadTasks.containsKey(modelPathMap.get(modelName))) {
+            return loadTasks.get(modelPathMap.get(modelName)).model;
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * 开始调度
+     *
+     * @param period
+     */
+    protected void start(long period) {
+        final Runnable task = new Runnable() {
+            public void run() {
+                // 模型更新开关
+                // boolean modelUpdateSwitch = Configuration.getBoolean("recommend-service-framework.model-update-switch", true);
+                boolean modelUpdateSwitch = true;
+                log.info("model update switch [{}]", modelUpdateSwitch);
+                if (modelUpdateSwitch) {
+                    updateModels(false);
+                }
+            }
+        };
+        scheduler.scheduleAtFixedRate(task, 10, period, TimeUnit.MINUTES); // 10分钟
+    }
+
+    /**
+     * 更新模型
+     */
+    public void updateModels(final boolean isForceLoads) {
+        log.info("begin to update: [{}]", loadTasks.keySet().size());
+        for (String modelPath : loadTasks.keySet()) {
+            log.debug("load task model path [{}]", modelPath);
+            ModelLoadTask task = loadTasks.get(modelPath);
+            loadModel(task, isForceLoads, false);
+        }
+    }
+
+    /**
+     * 检查并加载模型
+     * <p>
+     * 从oss加载：
+     * https://help.aliyun.com/zh/oss/developer-reference/streaming-download-7?spm=a2c4g.11186623.0.0.4b527c7dm8LejC
+     *
+     * @param loadTask
+     */
+    private void loadModel(final ModelLoadTask loadTask, final boolean isForceLoads, final boolean isRegister) {
+        if (loadTask.isLoading) {
+            return;
+        }
+        loadTask.isLoading = true;
+        OSSObject ossObj = null;
+        try {
+            ossObj = client.getObject(bucketName, loadTask.path);
+            long timeStamp = ossObj.getObjectMetadata().getLastModified().getTime();
+            if (loadTask.lastModifyTime <= timeStamp || isForceLoads) {
+                log.info("model file changed, ready to update, last modify: [{}]", loadTask.lastModifyTime);
+
+                Model model = loadTask.modelClass.newInstance();
+                if (model.loadFromStream(new InputStreamReader(ossObj.getObjectContent()))) {
+                    loadTask.model = model;
+                    loadTask.lastModifyTime = timeStamp;
+                }
+            }
+            ossObj.close();
+        } catch (Exception e) {
+            log.error("update model fail", e);
+        } finally {
+            loadTask.isLoading = false;
+            if (ossObj != null) {
+                try {
+                    ossObj.close();
+                } catch (IOException e) {
+                    log.error("close ossObj fail", e);
+                }
+            }
+        }
+    }
+
+    public class ModelRegisterException extends Exception {
+
+        public ModelRegisterException(String s) {
+            super(s);
+        }
+    }
+
+    /**
+     * 调度的任务单元
+     */
+    private class ModelLoadTask {
+
+        private int refCount;
+        private String path;
+        private long lastModifyTime;
+        private boolean isLoading;
+        private Class<? extends Model> modelClass;
+        private Model model;
+
+        ModelLoadTask(String path, Class<? extends Model> modelClass) {
+            this.refCount = 0;
+            this.path = path;
+            this.lastModifyTime = 0;
+            this.modelClass = modelClass;
+        }
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/OssConfig.java

@@ -0,0 +1,15 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+import lombok.Data;
+
+/**
+ * @author dyp
+ */
+@Data
+public class OssConfig {
+
+    private String accessKeyId;
+    private String accessKeySecret;
+    private String endpoint;
+    private String bucketName;
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/thread/CommonThreadPoolExecutor.java

@@ -0,0 +1,40 @@
+package com.tzld.piaoquan.ad.engine.commons.thread;
+
+import org.slf4j.MDC;
+
+import java.util.Map;
+import java.util.concurrent.*;
+
+/**
+ * @author dyp
+ */
+public class CommonThreadPoolExecutor extends ThreadPoolExecutor {
+    public CommonThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue);
+    }
+
+    public CommonThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory);
+    }
+
+    public CommonThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, RejectedExecutionHandler handler) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler);
+    }
+
+    public CommonThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory, RejectedExecutionHandler handler) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
+    }
+
+    @Override
+    public void execute(Runnable command) {
+        Map<String, String> mdcMap = MDC.getCopyOfContextMap();
+        super.execute(() -> {
+            if (mdcMap != null) {
+                MDC.setContextMap(mdcMap);
+            }
+            command.run();
+            MDC.clear();
+        });
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/thread/ThreadPoolFactory.java

@@ -0,0 +1,38 @@
+package com.tzld.piaoquan.ad.engine.commons.thread;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.tzld.piaoquan.ad.engine.commons.thread.CommonThreadPoolExecutor;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author dyp
+ */
+public final class ThreadPoolFactory {
+    private final static ExecutorService DEFAULT = new CommonThreadPoolExecutor(
+            32,
+            32,
+            0L, TimeUnit.SECONDS,
+            new LinkedBlockingQueue<>(1000),
+            new ThreadFactoryBuilder().setNameFormat("DEFAULT-%d").build(),
+            new ThreadPoolExecutor.AbortPolicy());
+    public final static ExecutorService RECALL = new CommonThreadPoolExecutor(
+            128,
+            128,
+            0L, TimeUnit.SECONDS,
+            new LinkedBlockingQueue<>(1000),
+            new ThreadFactoryBuilder().setNameFormat("RecallService-%d").build(),
+            new ThreadPoolExecutor.AbortPolicy());
+
+    public static ExecutorService defaultPool() {
+        return DEFAULT;
+    }
+
+    public static ExecutorService recallPool() {
+        return RECALL;
+    }
+
+}

ad-engine-server/pom.xml

@@ -17,6 +17,9 @@
             <artifactId>ad-engine-service</artifactId>
             <version>1.0.0</version>
         </dependency>
+
+
+
     </dependencies>
     <build>
         <!-- 固定包名 避免随着版本变动 -->

ad-engine-service/pom.xml

@@ -23,5 +23,12 @@
             <artifactId>ad-engine-commons</artifactId>
             <version>1.0.0</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>3.12.0</version>
+        </dependency>
+
+
     </dependencies>
 </project>

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCtrLRScorer.java

@@ -0,0 +1,188 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+
+import com.tzld.piaoquan.ad.engine.commons.base.*;
+import com.tzld.piaoquan.ad.engine.commons.score.BaseLRModelScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.feature.VlogAdCtrLRFeatureExtractor;
+import com.tzld.piaoquan.ad.engine.commons.score.model.LRModel;
+import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.*;
+
+
+public class VlogAdCtrLRScorer extends BaseLRModelScorer {
+
+    private final static int CORE_POOL_SIZE = 64;
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogAdCtrLRScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+    private static final double defaultUserCtrGroupNumber = 10.0;
+    private static final int enterFeedsScoreRatio = 10;
+    private static final int enterFeedsScoreNum = 20;
+
+
+    public VlogAdCtrLRScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+
+    @Override
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userFeature,
+                                    final List<AdRankItem> rankItems) {
+
+        if (userFeature == null || CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<AdRankItem> result = rankItems;
+        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final List<AdRankItem> items,
+                                      final RequestContext requestContext,
+                                      final UserAdFeature user) {
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // userAdBytes
+        UserAdBytesFeature userInfoBytes = null;
+        userInfoBytes = new UserAdBytesFeature(user);
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userInfoBytes, requestContext, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+
+    /**
+     * 计算 predict ctr
+     */
+    public double calcScore(final LRModel lrModel,
+                            final AdRankItem item,
+                            final UserAdBytesFeature userInfoBytes,
+                            final RequestContext requestContext) {
+
+        LRSamples lrSamples = null;
+        VlogAdCtrLRFeatureExtractor bytesFeatureExtractor;
+        bytesFeatureExtractor = new VlogAdCtrLRFeatureExtractor();
+
+        try {
+            AdItemBytesFeature newsInfoBytes = new AdItemBytesFeature(item.getItemFeature());
+            lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
+                    new RequestContextBytesFeature(requestContext));
+        } catch (Exception e) {
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getMid()), item.getAdId(),
+                    ExceptionUtils.getFullStackTrace(e)});
+        }
+
+
+        double pro = 0.0;
+        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
+            try {
+                pro = lrModel.score(lrSamples);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", new Object[]{
+                        item.getAdId(), ExceptionUtils.getFullStackTrace(e)});
+            }
+            // 增加实时特征后打开在线存储日志逻辑
+            //
+            // CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
+            // samples.setLr_samples(lrSamples);
+            // item.setSamples(samples);
+            //
+        }
+        item.setScore(pro);
+        return pro;
+    }
+
+
+    /**
+     * 并行打分
+     *
+     * @param items
+     * @param userInfoBytes
+     * @param requestContext
+     * @param model
+     */
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final UserAdBytesFeature userInfoBytes,
+                                  final RequestContext requestContext,
+                                  final LRModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).adId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", requestContext.getHeadVideoId(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{requestContext.getHeadVideoId(), items.size(), cancel});
+    }
+}

pom.xml

@@ -128,11 +128,6 @@
             </exclusions>
         </dependency>
 
-        <dependency>
-            <groupId>com.google.protobuf</groupId>
-            <artifactId>protobuf-java</artifactId>
-            <version>2.5.0</version>
-        </dependency>
         <dependency>
             <groupId>com.aliyun.openservices</groupId>
             <artifactId>aliyun-log-logback-appender</artifactId>
@@ -204,17 +199,7 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-pool2</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.aliyun.openservices</groupId>
-            <artifactId>ons-client</artifactId>
-            <version>${aliyun.rocketmq.version}</version>
-        </dependency>
-        <!-- druid数据源 -->
-        <dependency>
-            <groupId>com.alibaba</groupId>
-            <artifactId>druid</artifactId>
-            <version>1.1.23</version>
-        </dependency>
+
         <!--<dependency>-->
         <!--<groupId>com.googlecode.aviator</groupId>-->
         <!--<artifactId>aviator</artifactId>-->