Merge remote-tracking branch 'origin/ocpm_bid_engine_scores' into test

# Conflicts:
#	ad-engine-commons/pom.xml
#	ad-engine-service/pom.xml

ad-engine-commons/pom.xml

@@ -10,7 +10,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>ad-engine-commons</artifactId>
-    <version>1.0.1</version>
+    <version>1.0.3</version>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>1.8</maven.compiler.source>
@@ -25,7 +25,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.0.0</version>
+            <version>1.0.3</version>
         </dependency>
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/AdActionFeature.java

@@ -1,121 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-import lombok.Data;
-
-@Data
-public class AdActionFeature {
-    private double adView=0d;
-    private double adClick=0d;
-    private double adConversion=0d;
-
-    private double ctr=0d;
-    private double cvr=0d;
-
-    private double ceilLog(Double key) {
-        return Math.ceil(Math.log(key + 1));
-    }
-
-    private double bucketRatioFeature(Double key) {
-        long bucket = Math.round(Math.log((key + 1) * 50));
-        if (bucket > 50)
-            bucket = 50;
-        return (double) bucket;
-    }
-
-
-    public void setOriginAdView(Object key) {
-        if (key == null) {
-            this.adView = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "0");
-            this.adView = Double.valueOf(formateKey);
-        }
-    }
-
-    public void setOriginAdClick(Object key) {
-        if (key == null) {
-            this.adClick = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "0");
-            this.adClick = Double.valueOf(formateKey);
-        }
-    }
-
-    public void setOriginAdConversion(Object key) {
-        if (key == null) {
-            this.adConversion = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "0");
-            this.adConversion = Double.valueOf(formateKey);
-        }
-    }
-
-    // redis中保存原始值  server段统一处理
-    public void setOriginCtr(Object key) {
-        if (key == null) {
-            this.ctr = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "0");
-            this.ctr = Double.valueOf(formateKey);
-        }
-    }
-
-    public void setOriginCvr(Object key) {
-        if (key == null) {
-            this.cvr = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "0");
-            this.cvr = Double.valueOf(formateKey);
-        }
-    }
-
-
-
-
-    public void setAdView(Object key) {
-        if (key == null) {
-            this.adView = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "0");
-            this.adView = ceilLog(Double.valueOf(formateKey));
-        }
-    }
-
-    public void setAdClick(Object key) {
-        if (key == null) {
-            this.adClick = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "0");
-            this.adClick = ceilLog(Double.valueOf(formateKey));
-        }
-    }
-
-    public void setAdConversion(Object key) {
-        if (key == null) {
-            this.adConversion = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "0");
-            this.adConversion = ceilLog(Double.valueOf(formateKey));
-        }
-    }
-
-    public void setCtr(Object key) {
-        if (key == null) {
-            this.ctr = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "0");
-            this.ctr = bucketRatioFeature(Double.valueOf(formateKey));
-        }
-    }
-
-    public void setCvr(Object key) {
-        if (key == null) {
-            this.cvr = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "0");
-            this.cvr = bucketRatioFeature(Double.valueOf(formateKey));
-        }
-    }
-
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/AdItemBytesFeature.java

@@ -1,87 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-import lombok.Data;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@Data
-public class AdItemBytesFeature {
-
-    private final byte[] adId;
-
-    //private final byte[] adCode;
-
-    private final byte[] advertiserId;
-
-    //private final byte[] advertiserCode;
-
-    private final byte[] campaignId;
-
-    //private final byte[] campaignCode;
-
-    private final byte[] creative;
-
-    //private final byte[] creativeCode;
-
-
-    private Map<String, byte[]> day1_cnt_features;
-    // 3天内统计量
-    private Map<String, byte[]> day3_cnt_features;
-    // 7天内统计量
-    private Map<String, byte[]> day7_cnt_features;
-    // 3个月统计量
-    private Map<String, byte[]> month3_cnt_features;
-
-
-    public AdItemBytesFeature(AdItemFeature feature) {
-
-        adId = feature.getAdId().getBytes();
-        // adCode = feature.getAdCode().getBytes();
-        advertiserId = feature.getAdvertiserId().getBytes();
-        //advertiserCode = feature.getAdvertiserCode().getBytes();
-        campaignId = feature.getCampaignId().getBytes();
-        //campaignCode = feature.getCampaignCode().getBytes();
-        creative = feature.getCreativeId().getBytes();
-        // creativeCode = feature.getCreativeCode().getBytes();
-
-
-
-        // 1 day statistic
-        day1_cnt_features = new HashMap<String, byte[]>();
-        day1_cnt_features.put("view", String.valueOf(feature.getDay1_cnt_features().getAdView()).getBytes());
-        day1_cnt_features.put("click", String.valueOf(feature.getDay1_cnt_features().getAdClick()).getBytes());
-        day1_cnt_features.put("conversion", String.valueOf(feature.getDay1_cnt_features().getAdConversion()).getBytes());
-        day1_cnt_features.put("ctr", String.valueOf(feature.getDay1_cnt_features().getCtr()).getBytes());
-        day1_cnt_features.put("cvr", String.valueOf(feature.getDay1_cnt_features().getCvr()).getBytes());
-
-
-        // 3 day statistic
-        day3_cnt_features = new HashMap<String, byte[]>();
-        day3_cnt_features.put("view", String.valueOf(feature.getDay3_cnt_features().getAdView()).getBytes());
-        day3_cnt_features.put("click", String.valueOf(feature.getDay3_cnt_features().getAdClick()).getBytes());
-        day3_cnt_features.put("conversion", String.valueOf(feature.getDay3_cnt_features().getAdConversion()).getBytes());
-        day3_cnt_features.put("ctr", String.valueOf(feature.getDay3_cnt_features().getCtr()).getBytes());
-        day3_cnt_features.put("cvr", String.valueOf(feature.getDay3_cnt_features().getCvr()).getBytes());
-
-
-        // 7 day statistic
-        day7_cnt_features = new HashMap<String, byte[]>();
-        day7_cnt_features.put("view", String.valueOf(feature.getDay7_cnt_features().getAdView()).getBytes());
-        day7_cnt_features.put("click", String.valueOf(feature.getDay7_cnt_features().getAdClick()).getBytes());
-        day7_cnt_features.put("conversion", String.valueOf(feature.getDay7_cnt_features().getAdConversion()).getBytes());
-        day7_cnt_features.put("ctr", String.valueOf(feature.getDay7_cnt_features().getCtr()).getBytes());
-        day7_cnt_features.put("cvr", String.valueOf(feature.getDay7_cnt_features().getCvr()).getBytes());
-
-
-        // 3 month statisic
-        month3_cnt_features = new HashMap<String, byte[]>();
-        month3_cnt_features.put("view", String.valueOf(feature.getMonth3_cnt_features().getAdView()).getBytes());
-        month3_cnt_features.put("click", String.valueOf(feature.getMonth3_cnt_features().getAdClick()).getBytes());
-        month3_cnt_features.put("conversion", String.valueOf(feature.getMonth3_cnt_features().getAdConversion()).getBytes());
-        month3_cnt_features.put("ctr", String.valueOf(feature.getMonth3_cnt_features().getCtr()).getBytes());
-        month3_cnt_features.put("cvr", String.valueOf(feature.getMonth3_cnt_features().getCvr()).getBytes());
-
-    }
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/AdItemFeature.java

@@ -1,124 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-import javax.validation.constraints.NotNull;
-
-@Getter
-@NoArgsConstructor
-public class AdItemFeature {
-    @Setter
-    @NotNull
-    private String adId = "0";
-
-    @Setter
-    @NotNull
-    private String adCode = "0";
-
-    @Setter
-    @NotNull
-    private String advertiserId = "0";
-    @Setter
-    @NotNull
-    private String advertiserCode = "0";
-
-    @Setter
-    @NotNull
-    private String campaignId = "0";
-
-    @Setter
-    @NotNull
-    private String campaignCode = "0";
-
-    @Setter
-    @NotNull
-    private String creativeId = "0";
-
-    @Setter
-    @NotNull
-    private String creativeCode = "0";
-
-
-
-    // 当天统计量信息
-    private AdActionFeature day1_cnt_features=new AdActionFeature();
-    // 3天内统计量
-    private AdActionFeature day3_cnt_features=new AdActionFeature();
-    // 7天内统计量
-    private AdActionFeature day7_cnt_features=new AdActionFeature();
-    // 3个月统计量
-    private AdActionFeature month3_cnt_features=new AdActionFeature();
-
-
-
-    public void setAdId(String key){
-        if(key == null){
-            this.adId = "0";
-        } else {
-            this.adId = key;
-        }
-    }
-
-
-    public void setAdvertiserId(String key){
-        if(key == null){
-            this.advertiserId = "0";
-        } else {
-            this.advertiserId = key;
-        }
-    }
-
-    public void setCampaignId(String key){
-        if(key == null){
-            this.campaignId = "0";
-        } else {
-            this.campaignId = key;
-        }
-    }
-
-
-    public void setCreativeId(String key){
-        if(key == null){
-            this.creativeId = "0";
-        } else {
-            this.creativeId = key;
-        }
-    }
-
-
-
-    public void setDay1_cnt_features(AdActionFeature feature) {
-        this.day1_cnt_features = feature;
-    }
-
-
-    public void setDay3_cnt_features(AdActionFeature feature) {
-        this.day3_cnt_features = feature;
-
-    }
-
-    public void setDay7_cnt_features(AdActionFeature feature) {
-        this.day7_cnt_features = feature;
-
-    }
-
-    public void setMonth3_cnt_features(AdActionFeature feature) {
-        this.month3_cnt_features = feature;
-
-    }
-
-    public String getKey() {
-        return this.adId;
-    }
-
-    public String getValue() {
-        Gson gson = new GsonBuilder().serializeSpecialFloatingPointValues().create();
-        return gson.toJson(this);
-    }
-
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/AdRankItem.java

@@ -1,40 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-import lombok.Data;
-
-import java.util.HashMap;
-import java.util.Map;
-
-
-@Data
-public class AdRankItem implements Comparable<AdRankItem> {
-    public long adId;
-    private double score; // 记录最终的score
-
-    // 记录Item侧用到的特征
-    private AdItemFeature itemFeature;
-
-    public AdRankItem() {
-        //TODO
-    }
-
-    private Map<String, Double> rankerScore = new HashMap<>();
-    private Map<String, Integer> rankerIndex = new HashMap<>();
-
-
-    @Override
-    public int compareTo(AdRankItem o) {
-        if (o == null) {
-            return -1;
-        }
-        if (score > o.score) {
-            return -1;
-        } else if (score < o.score) {
-            return 1;
-        } else {
-            return 0;
-        }
-    }
-
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/AdRequestContext.java

@@ -1,102 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-@Getter
-@NoArgsConstructor
-public class AdRequestContext {
-
-    // 机型等信息
-    private String apptype="-1";
-    private String machineinfoBrand="-1";
-    private String machineinfoModel="-1";
-    private String machineinfoSdkversion="-1";
-    private String machineinfoWechatversion="-1";
-
-    // 时间等信息
-    private String day="-1";
-    private String week="-1";
-    private String hour="-1";
-    private String region="-1";
-    private String city="-1";
-
-    // video信息
-//    private String headVideoId;
-//    private String PageSource;
-
-    // position 信息
-    private String ownAdPositionId;
-
-
-
-    public void setApptype(String apptype) {
-        this.apptype = apptype;
-        if (apptype == null)
-            this.apptype = "-1";
-    }
-
-    public void setMachineinfoBrand(String machineinfoBrand) {
-        this.machineinfoBrand = machineinfoBrand;
-        if (machineinfoBrand == null)
-            this.machineinfoBrand = "-1";
-    }
-
-    public void setMachineinfoModel(String machineinfoModel) {
-        this.machineinfoModel = machineinfoModel;
-        if (machineinfoModel == null)
-            this.machineinfoModel = "-1";
-    }
-
-
-    public void setMachineinfoWchatversion(String machineinfo_wechatversion) {
-        this.machineinfoWechatversion = machineinfo_wechatversion;
-        if (machineinfo_wechatversion == null)
-            this.machineinfoWechatversion = "-1";
-    }
-
-
-    public void setMachineinfoSdkversion(String machineinfo_sdkversion) {
-        this.machineinfoSdkversion = machineinfo_sdkversion;
-        if (machineinfoSdkversion == null)
-            this.machineinfoSdkversion = "-1";
-    }
-
-
-
-    public void setHour(String hour) {
-        this.hour = hour;
-        if (hour == null)
-            this.hour = "-1";
-    }
-
-
-    public void setDay(String day) {
-        this.day = day;
-        if (day == null)
-            this.day = "-1";
-    }
-
-    public void setWeek(String week) {
-        this.week = week;
-        if (week == null)
-            this.week = "-1";
-    }
-
-
-    public void setRegion(String region) {
-        this.region = region;
-        if (region == null)
-            this.region = "-1";
-    }
-
-
-    public void setCity(String city) {
-        this.city = city;
-        if (city == null)
-            this.city = "-1";
-    }
-
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/AdRequestContextBytesFeature.java

@@ -1,48 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-
-import lombok.Data;
-
-@Data
-public class AdRequestContextBytesFeature {
-    private final byte[] apptype;
-    private final byte[] machineinfo_brand;
-    private final byte[] machineinfo_model;
-    private final byte[] machineinfo_sdkversion;
-    private final byte[] machineinfo_wechatversion;
-
-    // 时间等信息
-    private final byte[] day;
-    private final byte[] week;
-    private final byte[] hour;
-    private final byte[] region;
-    private final byte[] city;
-
-    // position 等信息
-    // private final byte[] headVideoId;
-    // private final byte[] pageSource;
-
-    // position 信息
-    // private final byte[] ownAdPositionId;
-
-
-    public AdRequestContextBytesFeature(AdRequestContext requestContext) {
-        apptype = requestContext.getApptype().getBytes();
-        machineinfo_brand = requestContext.getMachineinfoBrand().getBytes();
-        machineinfo_model = requestContext.getMachineinfoModel().getBytes();
-        machineinfo_sdkversion = requestContext.getMachineinfoSdkversion().getBytes();
-        machineinfo_wechatversion = requestContext.getMachineinfoWechatversion().getBytes();
-
-        day = requestContext.getDay().getBytes();
-        week = requestContext.getWeek().getBytes();
-        hour = requestContext.getHour().getBytes();
-        region = requestContext.getRegion().getBytes();
-        city = requestContext.getCity().getBytes();
-
-        // headVideoId = requestContext.getHeadVideoId().getBytes();
-        // pageSource = requestContext.getPageSource().getBytes();
-        // ownAdPositionId = requestContext.getOwnAdPositionId().getBytes();
-    }
-
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/UserAdBytesFeature.java

@@ -1,154 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-import lombok.Data;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@Data
-public class UserAdBytesFeature {
-
-    private final byte[] mid;
-
-    private final byte[] gender;
-    private final byte[] user_type;
-    private final byte[] gmt_create;
-    private final byte[] tags;
-    private final byte[] category_name;
-    private final byte[] isvip;
-    private final byte[] isreward;
-    private final byte[] isad;
-    private final byte[] isgood;
-
-    private final byte[] first_up_datetime;
-    private final byte[] last_up_datetime;
-    private final byte[] next_to_last_up_datetime;
-    private final byte[] videos; // 上传视频数量
-    private final byte[] idols; // 关注人数
-    private final byte[] fans;  // 粉丝数
-    private final byte[] play_count;  // 播放人数
-    private final byte[] play_count_total;  // 累计播放次数
-
-    private final byte[] total_reward;
-    private final byte[] currentday_reward;
-    private final byte[] reward_person;
-    private final byte[] total_reward_times;
-
-    private final byte[] reward_videos;
-    private final byte[] total_price;
-    private final byte[] currentday_price;
-    private final byte[] total_price_times;
-    private final byte[] total_price_person;
-    private final byte[] total_price_videos;
-    private final byte[] cgrain_user_type;
-    private final byte[] identity_tagname;
-
-
-    private final byte[] operation_tags;
-    private final byte[] identity_tag_id;
-    private final byte[] identity_create_time;
-    private final byte[] country;
-    private final byte[] province;
-    private final byte[] city;
-
-
-    // 当天统计量信息
-    private Map<String, byte[]> day1_cnt_features;
-    // 3天内统计量
-    private Map<String, byte[]> day3_cnt_features;
-    // 7天内统计量
-    private Map<String, byte[]> day7_cnt_features;
-    // 3个月统计量
-    private Map<String, byte[]> month3_cnt_features;
-
-
-    public UserAdBytesFeature(UserAdFeature feature) {
-        this.mid = feature.getMid().getBytes();
-
-
-        this.gender = feature.getGender().getBytes();
-        this.user_type = feature.getUser_type().getBytes();
-        this.gmt_create = feature.getGmt_create().getBytes();
-        this.tags = feature.getTags().getBytes() ;
-        this.category_name = feature.getCategory_name().getBytes();
-        this.isvip = feature.getIsvip().getBytes();
-        this.isreward = feature.getIsreward().getBytes();
-        this.isad = feature.getIsad().getBytes();
-        this.isgood = feature.getIsgood().getBytes();
-
-        this.first_up_datetime = feature.getFirst_up_datetime().getBytes();
-        this.last_up_datetime = feature.getLast_up_datetime().getBytes();
-        this.next_to_last_up_datetime = feature.getNext_to_last_up_datetime().getBytes();
-        this.videos = feature.getVideos().getBytes(); // 上传视频数量
-        this.idols = feature.getIdols().getBytes(); // 关注人数
-        this.fans = feature.getFans().getBytes();  // 粉丝数
-        this.play_count = feature.getPlay_count().getBytes();  // 播放人数
-        this.play_count_total = feature.getPlay_count_total().getBytes();  // 累计播放次数
-
-        this.total_reward = feature.getTotal_reward().getBytes();
-        this.currentday_reward = feature.getCurrentday_reward().getBytes();
-        this.reward_person = feature.getReward_person().getBytes();
-        this.total_reward_times = feature.getTotal_reward_times().getBytes();
-
-        this.reward_videos = feature.getReward_videos().getBytes();
-        this.total_price = feature.getTotal_price().getBytes();
-        this.currentday_price = feature.getCurrentday_price().getBytes();
-        this.total_price_times = feature.getTotal_price_times().getBytes();
-        this.total_price_person = feature.getTotal_price_person().getBytes();
-        this.total_price_videos = feature.getTotal_price_videos().getBytes();
-        this.cgrain_user_type = feature.getCgrain_user_type().getBytes();
-        this.identity_tagname = feature.getIdentity_tagname().getBytes();
-
-
-        this.operation_tags = feature.getOperation_tags().getBytes();
-        this.identity_tag_id = feature.getIdentity_tag_id().getBytes();
-        this.identity_create_time = feature.getIdentity_create_time().getBytes();
-        this.country = feature.getCountry().getBytes();
-        this.province = feature.getProvince().getBytes();
-        this.city = feature.getCity().getBytes();
-
-
-
-        // 1 day statistic
-        day1_cnt_features = new HashMap<String, byte[]>();
-        day1_cnt_features.put("view", String.valueOf(feature.getDay1_cnt_features().getAdView()).getBytes());
-        day1_cnt_features.put("click", String.valueOf(feature.getDay1_cnt_features().getAdClick()).getBytes());
-        day1_cnt_features.put("conversion", String.valueOf(feature.getDay1_cnt_features().getAdConversion()).getBytes());
-        day1_cnt_features.put("ctr", String.valueOf(feature.getDay1_cnt_features().getCtr()).getBytes());
-        day1_cnt_features.put("cvr", String.valueOf(feature.getDay1_cnt_features().getCvr()).getBytes());
-
-
-
-
-        // 3 day statistic
-        day3_cnt_features = new HashMap<String, byte[]>();
-        day3_cnt_features.put("view", String.valueOf(feature.getDay3_cnt_features().getAdView()).getBytes());
-        day3_cnt_features.put("click", String.valueOf(feature.getDay3_cnt_features().getAdClick()).getBytes());
-        day3_cnt_features.put("conversion", String.valueOf(feature.getDay3_cnt_features().getAdConversion()).getBytes());
-        day3_cnt_features.put("ctr", String.valueOf(feature.getDay3_cnt_features().getCtr()).getBytes());
-        day3_cnt_features.put("cvr", String.valueOf(feature.getDay3_cnt_features().getCvr()).getBytes());
-
-
-        // 7 day statistic
-        day7_cnt_features = new HashMap<String, byte[]>();
-        day7_cnt_features.put("view", String.valueOf(feature.getDay7_cnt_features().getAdView()).getBytes());
-        day7_cnt_features.put("click", String.valueOf(feature.getDay7_cnt_features().getAdClick()).getBytes());
-        day7_cnt_features.put("conversion", String.valueOf(feature.getDay7_cnt_features().getAdConversion()).getBytes());
-        day7_cnt_features.put("ctr", String.valueOf(feature.getDay7_cnt_features().getCtr()).getBytes());
-        day7_cnt_features.put("cvr", String.valueOf(feature.getDay7_cnt_features().getCvr()).getBytes());
-
-
-        // 3 month statisic
-        month3_cnt_features = new HashMap<String, byte[]>();
-        month3_cnt_features.put("view", String.valueOf(feature.getMonth3_cnt_features().getAdView()).getBytes());
-        month3_cnt_features.put("click", String.valueOf(feature.getMonth3_cnt_features().getAdClick()).getBytes());
-        month3_cnt_features.put("conversion", String.valueOf(feature.getMonth3_cnt_features().getAdConversion()).getBytes());
-        month3_cnt_features.put("ctr", String.valueOf(feature.getMonth3_cnt_features().getCtr()).getBytes());
-        month3_cnt_features.put("cvr", String.valueOf(feature.getMonth3_cnt_features().getCvr()).getBytes());
-
-
-
-    }
-
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/UserAdFeature.java

@@ -1,113 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-import javax.validation.constraints.NotNull;
-
-@Getter
-@NoArgsConstructor
-@Setter
-@NotNull
-public class UserAdFeature {
-
-    private String mid = "0";
-
-    private String gender = "0";
-
-    private String user_type = "0";
-
-    private String gmt_create = "0";
-
-    private String tags = "0";
-
-    private String category_name = "0";
-
-    private String isvip = "0";
-
-    private String isreward = "0";
-
-    private String isad = "0";
-
-    private String isgood = "0";
-
-    private String first_up_datetime = "0";
-    private String last_up_datetime = "0";
-    private String next_to_last_up_datetime = "0";
-    private String videos = "0"; // 上传视频数量
-    private String idols = "0"; // 关注人数
-    private String fans = "0";  // 粉丝数
-    private String play_count = "0";  // 播放人数
-    private String play_count_total = "0";  // 累计播放次数
-
-    private String total_reward = "0";
-    private String currentday_reward = "0";
-    private String reward_person = "0";
-    private String total_reward_times = "0";
-
-    private String reward_videos = "0";
-    private String total_price = "0";
-    private String currentday_price = "0";
-    private String total_price_times = "0";
-    private String total_price_person = "0";
-    private String total_price_videos = "0";
-    private String cgrain_user_type = "0";
-    private String identity_tagname = "0";
-
-
-    private String operation_tags = "0";
-    private String identity_tag_id = "0";
-    private String identity_create_time = "0";
-    private String country = "0";
-    private String province = "0";
-    private String city = "0";
-
-
-    // 当天统计量信息
-    private AdActionFeature day1_cnt_features=new AdActionFeature();
-    // 3天内统计量
-    private AdActionFeature day3_cnt_features=new AdActionFeature();
-    // 7天内统计量
-    private AdActionFeature day7_cnt_features=new AdActionFeature();
-    // 3个月统计量
-    private AdActionFeature month3_cnt_features=new AdActionFeature();
-
-
-    public void setDay1_cnt_features(AdActionFeature key) {
-        this.day1_cnt_features = key;
-        if (key == null)
-            this.day1_cnt_features = new AdActionFeature();
-    }
-
-    public void setDay3_cnt_features(AdActionFeature key) {
-        this.day3_cnt_features = key;
-        if (key == null)
-            this.day3_cnt_features = new AdActionFeature();
-    }
-
-    public void setDay7_cnt_features(AdActionFeature key) {
-        this.day7_cnt_features = key;
-        if (key == null)
-            this.day7_cnt_features = new AdActionFeature();
-    }
-
-    public void setMonth3_cnt_features(AdActionFeature key) {
-        this.month3_cnt_features = key;
-        if (key == null)
-            this.month3_cnt_features = new AdActionFeature();
-    }
-
-
-    public String getKey() {
-        return this.mid;
-    }
-
-    public String getValue() {
-        Gson gson = new GsonBuilder().serializeSpecialFloatingPointValues().create();
-        return gson.toJson(this);
-    }
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/UserVideoActionFeature.java

@@ -1,108 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-import lombok.Data;
-
-@Data
-public class UserVideoActionFeature {
-    private double exp_cnt;
-    private double click_cnt;
-    private double share_cnt;
-    private double return_cnt;
-
-    private double ctr;
-    private double str;
-    private double rov;
-    private double ros;
-
-    private double ceilLog(Double key) {
-        return Math.ceil(Math.log(key));
-    }
-
-    private double bucketRatioFeature(Double key) {
-        long bucket = Math.round(Math.log(key * 100));
-        if( bucket > 100)
-            bucket = 100;
-        return (double) bucket;
-    }
-
-
-    public void setExp_cnt(Object key){
-        if(key == null ) {
-            this.exp_cnt = 0.0;
-        } else {
-            String formateKey = key.toString().replace("\\N", "-1");
-            this.exp_cnt = ceilLog(Double.valueOf(formateKey));
-        }
-    }
-
-    public void setClick_cnt(Object key){
-        if(key == null ){
-            this.click_cnt = 0.0 ;
-        } else {
-            String formateKey = key.toString().replace("\\N", "-1");
-            this.click_cnt = ceilLog(Double.valueOf(formateKey));
-        }
-    }
-    public void setShare_cnt(Object key){
-        if(key == null ){
-            this.share_cnt = 0.0 ;
-        } else {
-            String formateKey = key.toString().replace("\\N", "-1");
-            this.share_cnt = ceilLog(Double.valueOf(formateKey));
-        }
-    }
-    public void setReturn_cnt(Object key){
-        if(key == null ){
-            this.return_cnt = 0.0 ;
-        } else {
-            String formateKey = key.toString().replace("\\N", "-1");
-            this.return_cnt = ceilLog(Double.valueOf(formateKey));
-        }
-    }
-
-    public void setCtr(Object key){
-        if(key == null ){
-            this.ctr = 0.0 ;
-        } else {
-            String formateKey = key.toString().replace("\\N", "-1");
-            this.ctr = bucketRatioFeature(Double.valueOf(formateKey));
-        }
-    }
-
-    public void setStr(Object key){
-        if(key == null ){
-            this.str = 0.0 ;
-        } else {
-            String formateKey = key.toString().replace("\\N", "-1");
-            this.str = bucketRatioFeature(Double.valueOf(formateKey));
-        }
-    }
-
-    public void setRov(Object key){
-        if(key == null ){
-            this.rov = 0.0 ;
-        } else {
-            String formateKey = key.toString().replace("\\N", "-1");
-            this.rov = bucketRatioFeature(Double.valueOf(formateKey));
-        }
-    }
-
-    public void setRos(Object key){
-        if(key == null ){
-            this.ros = 0.0 ;
-        } else {
-            String formateKey = key.toString().replace("\\N", "-1");
-            this.ros = bucketRatioFeature(Double.valueOf(formateKey));
-        }
-    }
-
-
-
-
-
-
-
-
-
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/base/UserVideoFeature.java

@@ -1,86 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.base;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-@Getter
-@NoArgsConstructor
-public class UserVideoFeature {
-
-    private String uid;
-    // 当天统计量信息
-    private AdActionFeature day1_cnt_features;
-    // 3天内统计量
-    private AdActionFeature day3_cnt_features;
-    // 7天内统计量
-    private AdActionFeature day7_cnt_features;
-    // 3个月统计量
-    private AdActionFeature month3_cnt_features;
-    // 用户行为周期
-    private String user_cycle_bucket_7days;
-    private String user_cycle_bucket_30days;
-    private String user_share_bucket_30days;
-
-    public void setUid(String key){
-        this.uid = key;
-        if(key == null)
-            this.uid = "0";
-    }
-
-
-    public void setDay1_cnt_features(AdActionFeature key){
-        this.day1_cnt_features = key;
-        if(key == null)
-            this.day1_cnt_features = new AdActionFeature();
-    }
-
-    public void setDay3_cnt_features(AdActionFeature key){
-        this.day3_cnt_features = key;
-        if(key == null)
-            this.day3_cnt_features = new AdActionFeature();
-    }
-
-    public void setDay7_cnt_features(AdActionFeature key){
-        this.day7_cnt_features = key;
-        if(key == null)
-            this.day7_cnt_features = new AdActionFeature();
-    }
-
-    public void setMonth3_cnt_features(AdActionFeature key) {
-        this.month3_cnt_features = key;
-        if(key == null)
-            this.month3_cnt_features = new AdActionFeature();
-    }
-
-
-    public void setUser_cycle_bucket_7days(String key){
-        this.user_cycle_bucket_7days = key;
-        if(key == null)
-            this.user_cycle_bucket_7days = "0";
-    }
-
-    public void setUser_cycle_bucket_30days(String key){
-        this.user_cycle_bucket_30days = key;
-        if(key == null)
-            this.user_cycle_bucket_30days = "0";
-    }
-
-    public void setUser_share_bucket_30days(String key){
-        this.user_share_bucket_30days = key;
-        if(key == null)
-            this.user_share_bucket_30days = "0";
-    }
-
-
-    public String getKey() {
-        return this.uid;
-    }
-
-    public String getValue(){
-        Gson gson = new GsonBuilder().serializeSpecialFloatingPointValues().create();
-        return gson.toJson(this);
-    }
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/enums/VlogAdFeatureGroup.java

@@ -6,7 +6,6 @@ public enum VlogAdFeatureGroup {
     MACHINEINFO_MODEL,
     MACHINEINFO_PLATFORM,
     MACHINEINFO_SDKVERSION,
-    MACHINEINFO_SYSTEM,
     MACHINEINFO_WECHATVERSION,
 
 
@@ -22,7 +21,9 @@ public enum VlogAdFeatureGroup {
     ADVERID,
     CAMPAIGIN,
 
-
+    //City
+    CREATIVE_AND_CITY,
+    ADVERID_AND_CITY,
 
 
     USER_1DAY_EXP,
@@ -67,6 +68,70 @@ public enum VlogAdFeatureGroup {
     ITEM_3MONTH_CVR,
 
 
+    CREATIVE_1DAY_EXP,
+    CREATIVE_1DAY_CLICK,
+    CREATIVE_1DAY_CTR,
+    CREATIVE_1DAY_CVR,
+
+    CREATIVE_3DAY_EXP,
+    CREATIVE_3DAY_CLICK,
+    CREATIVE_3DAY_CTR,
+    CREATIVE_3DAY_CVR,
+
+    CREATIVE_7DAY_EXP,
+    CREATIVE_7DAY_CLICK,
+    CREATIVE_7DAY_CTR,
+    CREATIVE_7DAY_CVR,
+
+    CREATIVE_3MONTH_EXP,
+    CREATIVE_3MONTH_CLICK,
+    CREATIVE_3MONTH_CTR,
+    CREATIVE_3MONTH_CVR,
+
+
+
+    ADVERID_1DAY_EXP,
+    ADVERID_1DAY_CLICK,
+    ADVERID_1DAY_CTR,
+    ADVERID_1DAY_CVR,
+
+    ADVERID_3DAY_EXP,
+    ADVERID_3DAY_CLICK,
+    ADVERID_3DAY_CTR,
+    ADVERID_3DAY_CVR,
+
+    ADVERID_7DAY_EXP,
+    ADVERID_7DAY_CLICK,
+    ADVERID_7DAY_CTR,
+    ADVERID_7DAY_CVR,
+
+    ADVERID_3MONTH_EXP,
+    ADVERID_3MONTH_CLICK,
+    ADVERID_3MONTH_CTR,
+    ADVERID_3MONTH_CVR,
+
+
+    // cross feature
+    CREATIVE_AND_HOUR,
+    ADVERID_AND_HOUR,
+    CREATIVE_AND_REGION,
+    ADVERID_AND_REGION,
+    CREATIVE_AND_MACHINEINFO_MODEL,
+    ADVERID_AND_MACHINEINFO_MODEL,
+
+    CREATIVE_AND_MACHINEINFO_BRAND,
+    ADVERID_AND_MACHINEINFO_BRAND,
+
+    CREATIVE_AND_MACHINEINFO_SDK,
+    ADVERID_AND_MACHINEINFO_SDK,
+
+    CREATIVE_AND_USER_3MONTH_CTR,
+    ADVERID_AND_USER_3MONTH_CTR,
+
+    USER_3MONTH_CTR_AND_ITEM_7DAY_CTR,
+    USER_3MONTH_CTR_AND_ITEM_3DAY_CTR,
+
+
     ;
 
 

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/AbstractScorer.java

@@ -1,11 +1,11 @@
 package com.tzld.piaoquan.ad.engine.commons.score;
 
 
-import com.tzld.piaoquan.ad.engine.commons.base.UserVideoFeature;
 import com.tzld.piaoquan.ad.engine.commons.score.model.Model;
 import com.tzld.piaoquan.ad.engine.commons.score.model.ModelManager;
-import com.tzld.piaoquan.ad.engine.commons.base.AdRankItem;
-import com.tzld.piaoquan.ad.engine.commons.base.UserAdFeature;
+
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/BaseCalibrationScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+import com.tzld.piaoquan.ad.engine.commons.score.model.CalibrationModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseCalibrationScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseCalibrationScorer.class);
+
+    public BaseCalibrationScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(CalibrationModel.class);
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/BaseMergeScorer.java

@@ -0,0 +1,19 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+import com.tzld.piaoquan.ad.engine.commons.score.model.LRModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseMergeScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseMergeScorer.class);
+
+    public BaseMergeScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/BaseThompsonSamplingScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+import com.tzld.piaoquan.ad.engine.commons.score.model.ThompsonSamplingModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseThompsonSamplingScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseThompsonSamplingScorer.class);
+
+    public BaseThompsonSamplingScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(ThompsonSamplingModel.class);
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScoreParam.java

@@ -1,6 +1,6 @@
 package com.tzld.piaoquan.ad.engine.commons.score;
 
-import com.tzld.piaoquan.ad.engine.commons.base.AdRequestContext;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import lombok.Data;
 
 /**

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerPipeline.java

@@ -1,8 +1,7 @@
 package com.tzld.piaoquan.ad.engine.commons.score;
 
-import com.tzld.piaoquan.ad.engine.commons.base.AdRankItem;
-import com.tzld.piaoquan.ad.engine.commons.base.UserAdFeature;
-import com.tzld.piaoquan.ad.engine.commons.base.UserVideoFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerUtils.java

@@ -23,9 +23,14 @@ public final class ScorerUtils {
 
     public static String BASE_CONF = "feeds_score_config_baseline.conf";
 
+    public static String THOMPSON_CONF = "feeds_score_config_thompson.conf";
+
+
     public static void warmUp() {
         log.info("scorer warm up ");
         ScorerUtils.init(BASE_CONF);
+        ScorerUtils.init(THOMPSON_CONF);
+
     }
 
     private ScorerUtils() {

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/BytesGroup.java

@@ -1,35 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.score.feature;
-
-
-public class BytesGroup {
-    private int id;
-    private String name;
-    private byte[] nameBytes;
-    private byte[] buffer;
-
-    public BytesGroup(int id, String name, byte[] nameBytes) {
-        this.id = id;
-        this.name = name;
-        this.nameBytes = nameBytes;
-    }
-
-    public int getId() {
-        return id;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    public byte[] getNameBytes() {
-        return nameBytes;
-    }
-
-    public byte[] getBuffer() {
-        return buffer;
-    }
-
-    public void setBuffer(byte[] buffer) {
-        this.buffer = buffer;
-    }
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/BytesUtils.java

@@ -1,192 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.score.feature;
-
-
-import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Extract features from user, item & context info. Returns 64-bit murmurhash of feature string as results.
- */
-public class BytesUtils {
-    private static final byte[] SEPARATOR = "_".getBytes();
-    private static final byte[] FEATURE_SEPARATOR = "#".getBytes();
-    private static final int MAX_FEATURE_BYTES_LENGTH = 512;
-    private static final long SEED = 11L;
-    private BytesGroup[] groups;
-
-    /**
-     * 一个种特殊的List,在尝试写入null的时候回默默地扔掉.
-     * @param <E> List的元素类型.
-     */
-    public static class NullRejectingArrayList<E> extends ArrayList<E> {
-        public NullRejectingArrayList(int capacity) {
-            super(capacity);
-        }
-
-        public NullRejectingArrayList() {
-            super();
-        }
-
-        @Override
-        public boolean add(E e) {
-            return e != null && super.add(e);
-        }
-    }
-
-    public BytesUtils(BytesGroup[] groups) {
-        this.groups = groups;
-        for (BytesGroup g : groups) {
-            byte[] buffer = prepareBuffer(g.getName(), g.getNameBytes());
-            groups[g.getId()].setBuffer(buffer);
-        }
-    }
-
-    public byte[] prepareBuffer(String name, byte[] nameBytes) {
-
-        byte[] buffer = new byte[MAX_FEATURE_BYTES_LENGTH];
-        System.arraycopy(nameBytes, 0, buffer, 0, nameBytes.length);
-        System.arraycopy(FEATURE_SEPARATOR, 0, buffer, nameBytes.length, 1);
-        return buffer;
-    }
-
-    public BaseFeature baseFea(byte[] buffer, int length) {
-        long hash = FeatureHash.MurmurHash64(buffer, 0, length, SEED);
-
-        // debug中查看 String fea = new String(buffer, 0, length);
-        // 初始化protobuf并赋值
-        BaseFeature.Builder tmp = BaseFeature.newBuilder();
-        tmp.setIdentifier(hash);
-        return tmp.build();
-    }
-
-    public BaseFeature makeFea(int id, byte[] value) {
-        byte[] buffer = groups[id].getBuffer();
-        if (buffer == null || value == null) {
-            return null;
-        }
-
-        final int nameLength = groups[id].getNameBytes().length + 1;
-        final int length = nameLength + value.length;
-        System.arraycopy(value, 0, buffer, nameLength, value.length);
-        return baseFea(buffer, length);
-    }
-
-    public BaseFeature makeFea(int id, final byte[] p1, final byte[] p2) {
-        byte[] buffer = groups[id].getBuffer();
-        if (buffer == null || p1 == null || p2 == null) {
-            return null;
-        }
-
-        final int nameLength = groups[id].getNameBytes().length + 1;
-        final int length = nameLength + p1.length + 1 + p2.length;
-
-        System.arraycopy(p1, 0, buffer, nameLength, p1.length);
-        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length, 1);
-        System.arraycopy(p2, 0, buffer, nameLength + p1.length + 1, p2.length);
-        return baseFea(buffer, length);
-    }
-
-    public BaseFeature makeFea(int id, final byte[] p1, final byte[] p2, final byte[] p3) {
-        byte[] buffer = groups[id].getBuffer();
-        if (buffer == null || p1 == null || p2 == null || p3 == null) {
-            return null;
-        }
-
-        final int nameLength = groups[id].getNameBytes().length + 1;
-        final int length = nameLength + p1.length + 1 + p2.length + 1 + p3.length;
-        System.arraycopy(p1, 0, buffer, nameLength, p1.length);
-        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length, 1);
-        System.arraycopy(p2, 0, buffer, nameLength + p1.length + 1, p2.length);
-        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length + 1 + p2.length, 1);
-        System.arraycopy(p3, 0, buffer, nameLength + p1.length + 1 + p2.length + 1, p3.length);
-
-        return baseFea(buffer, length);
-    }
-
-    public BaseFeature makeFea(int id, final byte[] p1, final byte[] p2, final byte[] p3, final byte[] p4) {
-        byte[] buffer = groups[id].getBuffer();
-        if (buffer == null || p1 == null || p2 == null || p3 == null || p4 == null) {
-            return null;
-        }
-
-        final int nameLength = groups[id].getNameBytes().length + 1;
-        final int length = nameLength + p1.length + 1 + p2.length + 1 + p3.length + 1 + p4.length;
-        System.arraycopy(p1, 0, buffer, nameLength, p1.length);
-        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length, 1);
-        System.arraycopy(p2, 0, buffer, nameLength + p1.length + 1, p2.length);
-        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length + 1 + p2.length, 1);
-        System.arraycopy(p3, 0, buffer, nameLength + p1.length + 1 + p2.length + 1, p3.length);
-        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length + 1 + p2.length + 1 + p3.length, 1);
-        System.arraycopy(p4, 0, buffer, nameLength + p1.length + 1 + p2.length + 1 + p3.length + 1, p4.length);
-
-        return baseFea(buffer, length);
-    }
-
-    public List<BaseFeature> makeFea(int id, byte[][] list) {
-        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(list.length);
-        for (byte[] t: list) {
-            result.add(makeFea(id, t));
-        }
-        return result;
-    }
-
-    public List<BaseFeature> makeFea(int id, byte[][] left, byte[] right) {
-        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length);
-        for (byte[] l: left) {
-            result.add(makeFea(id, l, right));
-        }
-        return result;
-    }
-
-    public List<BaseFeature> makeFea(int id, byte[][] left, byte[] right1, byte[] right2) {
-        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length);
-        for (byte[] l: left) {
-            result.add(makeFea(id, l, right1, right2));
-        }
-        return result;
-    }
-
-    public List<BaseFeature> makeFea(int id, byte[][] left, byte[] right1, byte[] right2, byte[] right3) {
-        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length);
-        for (byte[] l: left) {
-            result.add(makeFea(id, l, right1, right2, right3));
-        }
-        return result;
-    }
-
-    public List<BaseFeature> makeFea(int id, byte[] left, byte[][] right) {
-        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(right.length);
-        for (byte[] r : right) {
-            result.add(makeFea(id, left, r));
-        }
-        return result;
-    }
-
-    public List<BaseFeature> makeFea(int id, byte[] left1, byte[] left2, byte[][] right) {
-        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(right.length);
-        for (byte[] r : right) {
-            result.add(makeFea(id, left1, left2, r));
-        }
-        return result;
-    }
-
-    public List<BaseFeature> makeFea(int id, byte[] left1, byte[] left2, byte[] left3, byte[][] right) {
-        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(right.length);
-        for (byte[] r : right) {
-            result.add(makeFea(id, left1, left2, left3, r));
-        }
-        return result;
-    }
-
-    public List<BaseFeature> makeFea(int id, byte[][] left, byte[][] right) {
-        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length * right.length);
-        for (byte[] l: left) {
-            for (byte[] r: right) {
-                result.add(makeFea(id, l, r));
-            }
-        }
-        return result;
-    }
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/FeatureHash.java

@@ -1,230 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.score.feature;
-
-
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-
-public class FeatureHash {
-    public static Charset CharSetUTF8 = Charset.forName("UTF-8");
-
-    public static long getUInt32(byte a, byte b, byte c, byte d) {
-        return (d << 24 | (c & 0xFF) << 16 | (b & 0xFF) << 8 | (a & 0xFF));
-    }
-
-    public static long hash64(byte[] data) {
-        return MurmurHash64A(ByteBuffer.wrap(data), 0, data.length, 11L);
-    }
-
-    public static long MurmurHash64A(ByteBuffer buffer, int from, int len, long seed) {
-        final long m = 0xc6a4a7935bd1e995L;
-        final int r = 47;
-
-        long h = (seed) ^ (len * m);
-        int longLength = len / 8;
-
-        for (int i = 0; i < longLength; ++i) {
-            final int bytePos = from + i * 8;
-            long k = buffer.getLong(bytePos);
-
-            k *= m;
-            k ^= k >> r;
-            k *= m;
-            h ^= k;
-            h *= m;
-        }
-
-        final int remainingPos = len & ~7;
-        switch (len % 8) {
-            case 7: h ^= (long)(buffer.get(remainingPos + 6) & 0xFF) << 48;
-            case 6: h ^= (long)(buffer.get(remainingPos + 5) & 0xFF) << 40;
-            case 5: h ^= (long)(buffer.get(remainingPos + 4) & 0xFF) << 32;
-            case 4: h ^= (long)(buffer.get(remainingPos + 3) & 0xFF) << 24;
-            case 3: h ^= (long)(buffer.get(remainingPos + 2) & 0xFF) << 16;
-            case 2: h ^= (long)(buffer.get(remainingPos + 1) & 0xFF) << 8;
-            case 1:
-                h ^= (long)(buffer.get(remainingPos) & 0xFF);
-                h *= m;
-        }
-
-        h ^= h >>> r;
-        h *= m;
-        h ^= h >>> r;
-        return h;
-    }
-
-    public static long MurmurHash32(byte data[], int len, long seed) {
-        long m = 0x5bd1e995L;
-        int r = 24;
-
-        long h = seed ^ len;
-
-        int offset = 0;
-        while (len >= 4) {
-            long k = getUInt32(data[offset], data[offset + 1], data[offset + 2], data[offset + 3]);
-
-            k *= m;
-            k &= 0xFFFFFFFFL;
-            k ^= k >> r;
-            k *= m;
-            k &= 0xFFFFFFFFL;
-
-            h *= m;
-            h &= 0xFFFFFFFFL;
-            h ^= k;
-
-            offset += 4;
-            len -= 4;
-        }
-
-        // Handle the last few bytes of the input array
-        switch (len) {
-            case 3: h ^= data[offset + 2] << 16;
-            case 2: h ^= data[offset + 1] << 8;
-            case 1: h ^= data[offset];
-                h *= m;
-                h &= 0xFFFFFFFFL;
-        } ;
-
-        // Do a few final mixes of the hash to ensure the last few
-        // bytes are well-incorporated.
-
-        h ^= h >> 13;
-        h *= m;
-        h &= 0xFFFFFFFFL;
-        h ^= h >> 15;
-
-        return h;
-    }
-
-    // 64-bit hash for 32-bit platforms
-    public static long MurmurHash64(byte[] buffer, int start, int len, long seed) {
-        final long m = 0x5bd1e995L;
-        final int r = 24;
-        final int original = len;
-
-        long h1 = (seed) ^ len;
-        long h2 = (seed >> 32);
-
-        int offset = start;
-        while (len >= 8) {
-            long k1 = getUInt32(buffer[offset], buffer[offset + 1], buffer[offset + 2], buffer[offset + 3]);
-            // long k1 = buffer.getInt(offset);
-
-            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
-            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
-            offset += 4;
-
-            long k2 = getUInt32(buffer[offset], buffer[offset + 1], buffer[offset + 2], buffer[offset + 3]);
-            // long k2 = buffer.getInt(offset);
-            k2 *= m; k2 &= 0xFFFFFFFFL; k2 ^= k2 >> r; k2 *= m; k2 &= 0xFFFFFFFFL;
-            h2 *= m; h2 &= 0xFFFFFFFFL; h2 ^= k2;
-
-            offset += 4;
-            len -= 8;
-        }
-
-        if (len >= 4) {
-            long k1 = getUInt32(buffer[offset], buffer[offset + 1], buffer[offset + 2], buffer[offset + 3]);
-            // long k1 = buffer.getInt(offset);
-            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
-            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
-            offset += 4;
-            len -= 4;
-        }
-
-        switch (len) {
-            case 3: h2 ^= (buffer[offset + 2] & 0xFF) << 16;
-            case 2: h2 ^= (buffer[offset + 1] & 0xFF) << 8;
-            case 1: h2 ^= (buffer[offset] & 0xFF);
-                h2 *= m;
-                h2 &= 0xFFFFFFFFL;
-        } ;
-
-        h1 ^= h2 >> 18;
-        h1 *= m; h1 &= 0xFFFFFFFFL;
-        h2 ^= h1 >> 22;
-        h2 *= m; h2 &= 0xFFFFFFFFL;
-        h1 ^= h2 >> 17;
-        h1 *= m; h1 &= 0xFFFFFFFFL;
-        h2 ^= h1 >> 19;
-        h2 *= m; h2 &= 0xFFFFFFFFL;
-
-        /*BigInteger ans = BigInteger.valueOf(h1).shiftLeft(32).or(BigInteger.valueOf(h2));
-        return ans.longValue();*/
-        //System.err.println("feature: " + new String(buffer, 0, original) + " length: " + original + " hash: " + (h1 << 32 | h2) + " daze");
-        return h1 << 32 | h2;
-    }
-
-    // 64-bit hash for 32-bit platforms
-    public static BigInteger MurmurHash64(byte data[], int len, long seed) {
-        long m = 0x5bd1e995L;
-        int r = 24;
-
-        long h1 = (seed) ^ len;
-        long h2 = (seed >> 32);
-
-        int offset = 0;
-        while (len >= 8) {
-            long k1 = getUInt32(data[offset], data[offset + 1], data[offset + 2], data[offset + 3]);
-            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
-            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
-
-            long k2 = getUInt32(data[offset + 4], data[offset + 5], data[offset + 6], data[offset + 7]);
-            k2 *= m; k2 &= 0xFFFFFFFFL; k2 ^= k2 >> r; k2 *= m; k2 &= 0xFFFFFFFFL;
-            h2 *= m; h2 &= 0xFFFFFFFFL; h2 ^= k2;
-
-            offset += 8;
-            len -= 8;
-        }
-
-        if (len >= 4) {
-            long k1 = getUInt32(data[offset], data[offset + 1], data[offset + 2], data[offset + 3]);
-            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
-            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
-            offset += 4;
-            len -= 4;
-        }
-
-        switch (len) {
-            case 3: h2 ^= (data[offset + 2] & 0xFF) << 16;
-            case 2: h2 ^= (data[offset + 1] & 0xFF) << 8;
-            case 1: h2 ^= (data[offset] & 0xFF);
-                h2 *= m;
-                h2 &= 0xFFFFFFFFL;
-        } ;
-
-        h1 ^= h2 >> 18;
-        h1 *= m; h1 &= 0xFFFFFFFFL;
-        h2 ^= h1 >> 22;
-        h2 *= m; h2 &= 0xFFFFFFFFL;
-        h1 ^= h2 >> 17;
-        h1 *= m; h1 &= 0xFFFFFFFFL;
-        h2 ^= h1 >> 19;
-        h2 *= m; h2 &= 0xFFFFFFFFL;
-
-        BigInteger ans = BigInteger.valueOf(h1).shiftLeft(32).or(BigInteger.valueOf(h2));
-        return ans;
-    }
-
-    public static String hash(String input) {
-        byte[] tt = input.getBytes(CharSetUTF8);
-        return MurmurHash64(tt, tt.length, 11L).toString();
-    }
-
-    public static Long hashToLong(String input) {
-        byte[] tt = input.getBytes(CharSetUTF8);
-        return MurmurHash64(tt, tt.length, 11L).longValue();
-    }
-
-    /** the constant 2^64 */
-    private static final BigInteger TWO_64 = BigInteger.ONE.shiftLeft(64);
-
-    public static String asUnsignedLongString(long l) {
-        BigInteger b = BigInteger.valueOf(l);
-        if (b.signum() < 0) {
-            b = b.add(TWO_64);
-        }
-        return b.toString();
-    }
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/FeatureUsage.java

@@ -1,43 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.score.feature;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigObject;
-import com.typesafe.config.ConfigValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.HashSet;
-import java.util.Set;
-
-public class FeatureUsage implements Serializable {
-    private final Logger LOGGER = LoggerFactory.getLogger(FeatureUsage.class);
-    private Set<String> hash;
-
-    public FeatureUsage() {
-        hash = new HashSet<String>();
-    }
-
-    public FeatureUsage(String confName) {
-        // note that these fields are NOT lazy, because if we're going to get any exceptions, we want to get them on startup.
-        Config config = ConfigFactory.load(confName);
-        config.checkValid(config, "features");
-        ConfigObject conf = config.getConfig("features").root();
-        LOGGER.info("create_feature_usage: " + confName);
-        hash = new HashSet<String>();
-        for (ConfigObject.Entry<String, ConfigValue> it : conf.entrySet()) {
-            String key = it.getKey();
-            Config cf = ((ConfigObject) it.getValue()).toConfig();
-            LOGGER.info("key: " + key + ", value: " + cf.toString());
-            boolean disable = cf.getBoolean("disable");
-            if (disable) {
-                hash.add(key);
-            }
-        }
-    }
-
-    public boolean disable(String fea) {
-        return hash.contains(fea);
-    }
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/LRBytesFeatureExtractorBase.java

@@ -1,67 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.score.feature;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ListMultimap;
-import com.tzld.piaoquan.ad.engine.commons.base.AdItemBytesFeature;
-import com.tzld.piaoquan.ad.engine.commons.base.AdRequestContextBytesFeature;
-import com.tzld.piaoquan.ad.engine.commons.base.UserAdBytesFeature;
-import com.tzld.piaoquan.ad.engine.commons.enums.VlogAdFeatureGroup;
-import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
-import com.tzld.piaoquan.recommend.server.gen.recommend.FeatureGroup;
-import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
-
-import java.util.List;
-
-
-public abstract class LRBytesFeatureExtractorBase {
-    private static final double DEFAULT_USER_CTR_GROUP = 10.0;
-    private static final double DEFAULT_ARTICLE_CTR_GROUP = 100.0;
-
-
-    private BytesUtils utils;
-    //Feature Group & Features
-    ListMultimap<FeatureGroup, BaseFeature> features = ArrayListMultimap.create();
-    int groupCount;
-
-
-    LRBytesFeatureExtractorBase() {
-        groupCount = VlogAdFeatureGroup.values().length;
-        BytesGroup[] groups = new BytesGroup[groupCount];
-        for (VlogAdFeatureGroup g: VlogAdFeatureGroup.values()) {
-            groups[g.ordinal()] = new BytesGroup(g.ordinal(),
-                    g.getGroupName(), g.getGroupNameBytes());
-        }
-        utils = new BytesUtils(groups);
-    }
-
-    private FeatureGroup makeGroup(VlogAdFeatureGroup group){
-        FeatureGroup.Builder g = FeatureGroup.newBuilder();
-        g.setType("1");
-        g.setName(group.getGroupName());
-        g.setId(group.ordinal());
-        return g.build();
-    };
-
-
-    void makeFea(VlogAdFeatureGroup group, byte[] value) {
-        FeatureGroup featureGroup = makeGroup(group);
-        BaseFeature feature = utils.makeFea(group.ordinal(), value);
-        features.put(featureGroup, feature);
-    }
-
-    void makeFea(VlogAdFeatureGroup group, byte[][] list) {
-        FeatureGroup g = makeGroup(group);
-        List<BaseFeature> featureList = utils.makeFea(group.ordinal(), list);
-        features.putAll(g, featureList);
-    }
-
-    public ListMultimap<FeatureGroup, BaseFeature> getFeatures() {
-        return features;
-    }
-
-    public abstract LRSamples single(UserAdBytesFeature userBytesFeature,
-                                     AdItemBytesFeature videoBytesFeature,
-                                     AdRequestContextBytesFeature requestContextBytesFeature);
-
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/feature/VlogAdCtrLRFeatureExtractor.java

@@ -1,133 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.score.feature;
-
-import com.tzld.piaoquan.ad.engine.commons.base.AdItemBytesFeature;
-import com.tzld.piaoquan.ad.engine.commons.base.AdRequestContextBytesFeature;
-import com.tzld.piaoquan.ad.engine.commons.base.UserAdBytesFeature;
-import com.tzld.piaoquan.ad.engine.commons.enums.VlogAdFeatureGroup;
-import com.tzld.piaoquan.recommend.server.gen.recommend.FeatureGroup;
-import com.tzld.piaoquan.recommend.server.gen.recommend.GroupedFeature;
-import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
-import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
-
-
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class VlogAdCtrLRFeatureExtractor extends LRBytesFeatureExtractorBase {
-
-    public VlogAdCtrLRFeatureExtractor() {
-        super();
-    }
-
-    // TODO
-    // 补充待抽取的context feature
-    public void getContextFeatures(AdRequestContextBytesFeature requestContextBytes) {
-        makeFea(VlogAdFeatureGroup.MACHINEINFO_BRAND, requestContextBytes.getMachineinfo_brand());
-        makeFea(VlogAdFeatureGroup.MACHINEINFO_MODEL, requestContextBytes.getMachineinfo_model());
-        makeFea(VlogAdFeatureGroup.MACHINEINFO_SDKVERSION, requestContextBytes.getMachineinfo_sdkversion());
-        makeFea(VlogAdFeatureGroup.MACHINEINFO_WECHATVERSION, requestContextBytes.getMachineinfo_brand());
-
-        makeFea(VlogAdFeatureGroup.DAY, requestContextBytes.getWeek());
-        makeFea(VlogAdFeatureGroup.WEEK, requestContextBytes.getWeek());
-        makeFea(VlogAdFeatureGroup.HOUR, requestContextBytes.getHour());
-
-    }
-
-    //TODO
-    public void getUserFeatures(UserAdBytesFeature userAdBytesFeature) {
-
-        // 1day features
-        makeFea(VlogAdFeatureGroup.USER_1DAY_EXP, userAdBytesFeature.getDay1_cnt_features().get("view"));
-        makeFea(VlogAdFeatureGroup.USER_1DAY_CLICK, userAdBytesFeature.getDay1_cnt_features().get("click"));
-        makeFea(VlogAdFeatureGroup.USER_1DAY_CTR, userAdBytesFeature.getDay1_cnt_features().get("ctr"));
-        makeFea(VlogAdFeatureGroup.USER_1DAY_CVR, userAdBytesFeature.getDay1_cnt_features().get("cvr"));
-
-        // 3day features
-        makeFea(VlogAdFeatureGroup.USER_3DAY_EXP, userAdBytesFeature.getDay3_cnt_features().get("view"));
-        makeFea(VlogAdFeatureGroup.USER_3DAY_CLICK, userAdBytesFeature.getDay3_cnt_features().get("click"));
-        makeFea(VlogAdFeatureGroup.USER_3DAY_CTR, userAdBytesFeature.getDay3_cnt_features().get("ctr"));
-        makeFea(VlogAdFeatureGroup.USER_3DAY_CVR, userAdBytesFeature.getDay3_cnt_features().get("cvr"));
-
-        // 7day features
-        makeFea(VlogAdFeatureGroup.USER_7DAY_EXP, userAdBytesFeature.getDay7_cnt_features().get("view"));
-        makeFea(VlogAdFeatureGroup.USER_7DAY_CLICK, userAdBytesFeature.getDay7_cnt_features().get("click"));
-        makeFea(VlogAdFeatureGroup.USER_7DAY_CTR, userAdBytesFeature.getDay7_cnt_features().get("ctr"));
-        makeFea(VlogAdFeatureGroup.USER_7DAY_CVR, userAdBytesFeature.getDay7_cnt_features().get("cvr"));
-
-        // 3month features
-        makeFea(VlogAdFeatureGroup.USER_3MONTH_EXP, userAdBytesFeature.getMonth3_cnt_features().get("view"));
-        makeFea(VlogAdFeatureGroup.USER_3MONTH_CLICK, userAdBytesFeature.getMonth3_cnt_features().get("click"));
-        makeFea(VlogAdFeatureGroup.USER_3MONTH_CTR, userAdBytesFeature.getMonth3_cnt_features().get("ctr"));
-        makeFea(VlogAdFeatureGroup.USER_3MONTH_CVR, userAdBytesFeature.getMonth3_cnt_features().get("cvr"));
-
-
-    }
-
-    public void getItemFeature(AdItemBytesFeature item) {
-
-        // Vdieo
-        makeFea(VlogAdFeatureGroup.ADID, item.getAdId());
-        makeFea(VlogAdFeatureGroup.CREATIVE, item.getCreative());
-        makeFea(VlogAdFeatureGroup.CAMPAIGIN, item.getCampaignId());
-        makeFea(VlogAdFeatureGroup.ADVERID, item.getAdvertiserId());
-
-
-
-        // 1day features
-        makeFea(VlogAdFeatureGroup.ITEM_1DAY_EXP, item.getDay1_cnt_features().get("view"));
-        makeFea(VlogAdFeatureGroup.ITEM_1DAY_CLICK, item.getDay1_cnt_features().get("click"));
-        makeFea(VlogAdFeatureGroup.ITEM_1DAY_CTR, item.getDay1_cnt_features().get("ctr"));
-        makeFea(VlogAdFeatureGroup.ITEM_1DAY_CVR, item.getDay1_cnt_features().get("cvr"));
-
-        // 3day features
-        makeFea(VlogAdFeatureGroup.ITEM_3DAY_EXP, item.getDay1_cnt_features().get("view"));
-        makeFea(VlogAdFeatureGroup.ITEM_3DAY_CLICK, item.getDay1_cnt_features().get("click"));
-        makeFea(VlogAdFeatureGroup.ITEM_3DAY_CTR, item.getDay1_cnt_features().get("ctr"));
-        makeFea(VlogAdFeatureGroup.ITEM_3DAY_CVR, item.getDay1_cnt_features().get("cvr"));
-
-
-        // 7day features
-        makeFea(VlogAdFeatureGroup.ITEM_7DAY_EXP, item.getDay1_cnt_features().get("view"));
-        makeFea(VlogAdFeatureGroup.ITEM_7DAY_CLICK, item.getDay1_cnt_features().get("click"));
-        makeFea(VlogAdFeatureGroup.ITEM_7DAY_CTR, item.getDay1_cnt_features().get("ctr"));
-        makeFea(VlogAdFeatureGroup.ITEM_7DAY_CVR, item.getDay1_cnt_features().get("cvr"));
-
-
-        // 3month features
-        makeFea(VlogAdFeatureGroup.ITEM_3MONTH_EXP, item.getMonth3_cnt_features().get("view"));
-        makeFea(VlogAdFeatureGroup.ITEM_3MONTH_CLICK, item.getMonth3_cnt_features().get("click"));
-        makeFea(VlogAdFeatureGroup.ITEM_3MONTH_CTR, item.getMonth3_cnt_features().get("ctr"));
-        makeFea(VlogAdFeatureGroup.ITEM_3MONTH_CVR, item.getMonth3_cnt_features().get("cvr"));
-
-    }
-
-    @Override
-    public synchronized LRSamples single(UserAdBytesFeature userAdBytesFeature,
-                                         AdItemBytesFeature videoBytesFeature,
-                                         AdRequestContextBytesFeature requestContextBytesFeature) {
-        features.clear();
-        // extract features
-        getUserFeatures(userAdBytesFeature);
-        getContextFeatures(requestContextBytesFeature);
-        getItemFeature(videoBytesFeature);
-
-        LRSamples.Builder lr = com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples.newBuilder();
-        lr.setGroupNum(groupCount);
-        List<FeatureGroup> keys = new ArrayList<>(features.keySet());
-        int count = 0;
-        for(FeatureGroup group : keys) {
-            List<BaseFeature> fea = features.get(group);
-            GroupedFeature.Builder gf = GroupedFeature.newBuilder();
-            gf.setGroup(group);
-            gf.setCount(fea.size());
-            gf.addAllFeatures(fea);
-            count += fea.size();
-            lr.addFeatures(gf);
-        }
-        lr.setCount(count);
-        return lr.build();
-    }
-
-
-}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/CalibrationModel.java

@@ -0,0 +1,76 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdActionFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import org.apache.commons.math3.distribution.BetaDistribution;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.TreeMap;
+
+
+public class CalibrationModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25;  // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(CalibrationModel.class);
+
+    private TreeMap<Double, Double> calibrationModel;
+
+    public CalibrationModel() {
+        //配置不同环境的hdfs conf
+        this.calibrationModel = new TreeMap<>();
+    }
+
+    public TreeMap<Double, Double> getCalibrationModel() {
+        return this.calibrationModel;
+    }
+
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+        TreeMap<Double, Double> bins = new TreeMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+            bins.put(Double.parseDouble(items[0]), Double.parseDouble(items[1]));
+        }
+
+        this.calibrationModel = bins;
+        LOGGER.info("[MODELLOAD] calibration model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+
+    @Override
+    public int getModelSize() {
+        if (this.calibrationModel == null)
+            return 0;
+        int sum = this.calibrationModel.size();
+        return sum;
+    }
+
+    public double score(AdRankItem adRankItem, String ctrOrCVR) {
+        double score = 0.0f;
+        if (ctrOrCVR.equals("ctr")) {
+            double upperBound = this.calibrationModel.floorKey(adRankItem.getCtr());
+            score = this.calibrationModel.get(upperBound);
+        }
+        if (ctrOrCVR.equals("cvr")) {
+            double upperBound = this.calibrationModel.floorKey(adRankItem.getCvr());
+            score = this.calibrationModel.get(upperBound);
+        }
+        return score;
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/LRModel.java

@@ -1,8 +1,8 @@
 package com.tzld.piaoquan.ad.engine.commons.score.model;
 
-import com.tzld.piaoquan.recommend.server.gen.recommend.BaseFeature;
-import com.tzld.piaoquan.recommend.server.gen.recommend.GroupedFeature;
-import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
+import com.tzld.piaoquan.recommend.feature.model.sample.BaseFeature;
+import com.tzld.piaoquan.recommend.feature.model.sample.GroupedFeature;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
 import it.unimi.dsi.fastutil.longs.Long2FloatMap;
 import it.unimi.dsi.fastutil.longs.Long2FloatOpenHashMap;
 import org.slf4j.Logger;

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/ThompsonSamplingModel.java

@@ -0,0 +1,96 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdActionFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.model.sample.BaseFeature;
+import com.tzld.piaoquan.recommend.feature.model.sample.GroupedFeature;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
+import it.unimi.dsi.fastutil.longs.Long2FloatMap;
+import it.unimi.dsi.fastutil.longs.Long2FloatOpenHashMap;
+import org.apache.commons.math3.distribution.BetaDistribution;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class ThompsonSamplingModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25;  // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(ThompsonSamplingModel.class);
+
+    // key = adid, value = <exp, click, conversation>
+    private Map<Long, AdActionFeature> thompsonSamplingModel;
+
+    public ThompsonSamplingModel() {
+        //配置不同环境的hdfs conf
+        this.thompsonSamplingModel = new HashMap<>();
+    }
+
+    public Map<Long, AdActionFeature> getLrModel() {
+        return this.thompsonSamplingModel;
+    }
+
+
+    public void putFeature(Map<Long, AdActionFeature> model, Long key, Double expose, Double click, Double conversation) {
+        AdActionFeature adActionFeature = new AdActionFeature();
+        adActionFeature.setOriginAdView(expose);
+        adActionFeature.setOriginAdClick(click);
+        adActionFeature.setOriginAdConversion(conversation);
+        model.put(key, adActionFeature);
+    }
+
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+        Map<Long, AdActionFeature> initModel = new HashMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 3) {
+                continue;
+            }
+            putFeature(initModel, new BigInteger(items[0].trim()).longValue(), Double.valueOf(items[1].trim()).doubleValue(), Double.valueOf(items[2].trim()).doubleValue(), Double.valueOf(items[3].trim()).doubleValue());
+        }
+
+        this.thompsonSamplingModel = initModel;
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+
+    @Override
+    public int getModelSize() {
+        if (this.thompsonSamplingModel == null)
+            return 0;
+        int sum = this.thompsonSamplingModel.size();
+        return sum;
+    }
+
+    public double score(AdRankItem adRankItem, String ctrOrCVR) {
+        double score = 0.0f;
+        AdActionFeature adActionFeature = this.thompsonSamplingModel.get(adRankItem.getAdId());
+        if(ctrOrCVR.equals("ctr"))
+            score = this.betaSampler(adActionFeature.getAdClick(), adActionFeature.getAdView());
+        if(ctrOrCVR.equals("cvr"))
+            score = this.betaSampler(adActionFeature.getAdConversion(), adActionFeature.getAdView());
+        return score;
+    }
+
+    public double betaSampler(double alpha, double beta) {
+        BetaDistribution betaSample = new BetaDistribution(alpha, beta);
+        return betaSample.sample();
+    }
+
+
+}

ad-engine-server/pom.xml

@@ -15,7 +15,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>ad-engine-service</artifactId>
-            <version>1.0.0</version>
+            <version>1.0.3</version>
         </dependency>
 
     </dependencies>

ad-engine-server/src/main/java/com/tzld/piaoquan/ad/engine/server/controller/AdRecommendController.java

@@ -1,10 +1,14 @@
 package com.tzld.piaoquan.ad.engine.server.controller;
 
 import com.tzld.piaoquan.ad.engine.service.score.RankService;
+import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformBidCreativeDTO;
+import com.tzld.piaoquan.ad.engine.service.score.param.BidRankRecommendRequestParam;
 import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
@@ -18,18 +22,31 @@ public class AdRecommendController {
 
     private final static Logger log = LoggerFactory.getLogger(AdRecommendController.class);
     @Autowired
+    @Qualifier("rankServiceImpl")
     RankService rankService;
 
     @RequestMapping("/top1/basic")
     public Map<String,Object> adRecommendTop1Basic(@RequestBody RankRecommendRequestParam request){
-        Long adId= rankService.adItemRank(request);
+        AdRankItem rankResult = rankService.adItemRank(request);
         HashMap map =new HashMap();
         map.put("code","0");
         map.put("msg","success");
         HashMap contentMap=new HashMap<>();
-        contentMap.put("adId",adId);
+        contentMap.put("adId", rankResult.getAdId());
+        contentMap.put("adScore", rankResult.getScore());
         map.put("content",contentMap);
         return map;
     }
 
+    @RequestMapping("/top1/bid/basic")
+    public Map<String,Object> adRecommendTop1BidBasic(@RequestBody BidRankRecommendRequestParam request){
+        AdPlatformBidCreativeDTO rankResult = rankService.adBidRank(request);
+        HashMap map =new HashMap();
+        map.put("code","0");
+        map.put("msg","success");
+        map.put("content",rankResult);
+
+        return map;
+    }
+
 }

ad-engine-server/src/main/resources/feeds_score_config_baseline.conf

@@ -1,7 +1,28 @@
 scorer-config = {
-  related-score-config = {
+  lr-ctr-score-config = {
     scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogAdCtrLRScorer"
     scorer-priority = 99
     model-path = "ad_ctr_model/model_ad_ctr.txt"
   }
+  lr-cvr-score-config = {
+      scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogAdCvrLRScorer"
+      scorer-priority = 99
+      model-path = "ad_cvr_model/model_ad_cvr.txt"
+  }
+  lr-ctr-calibretion-config = {
+      scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogAdCtrCalibrationScorer"
+      scorer-priority = 99
+      model-path = "ad_ctr_calibration/model_ctr_calibration.txt"
+  }
+
+  lr-cvr-calibretion-config = {
+      scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogAdCvrCalibrationScorer"
+      scorer-priority = 99
+      model-path = "ad_cvr_calibration/model_cvr_calibration.txt"
+  }
+  lr-ecpm-merge-config = {
+      scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogMergeEcpmScorer"
+      scorer-priority = 99
+  }
+
 }

ad-engine-server/src/main/resources/feeds_score_config_thompson.conf

@@ -0,0 +1,14 @@
+scorer-config = {
+  lr-ctr-score-config = {
+    scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogAdThompsonScorer"
+    scorer-priority = 99
+    model-path = "ad_thompson_model/model_ad_thompson.txt"
+  }
+
+  lr-ecpm-merge-config = {
+        scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogMergeEcpmScorer"
+        scorer-priority = 99
+  }
+
+
+}

ad-engine-service/pom.xml

@@ -10,6 +10,8 @@
     </parent>
 
     <artifactId>ad-engine-service</artifactId>
+    <version>1.0.3</version>
+
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -21,13 +23,13 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>ad-engine-commons</artifactId>
-            <version>1.0.1</version>
+            <version>1.0.3</version>
         </dependency>
 
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.0.0</version>
+            <version>1.0.3</version>
         </dependency>
 
     </dependencies>

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/param/RuleParamHelper.java

@@ -204,7 +204,9 @@ public class RuleParamHelper {
             }
         }
 
-        String thresholdKeyName = thresholdKeyNamePrefix + abtestId + ":" + abtestConfigTag + ":" + abTestCode + ":" + midGroup;
+       String thresholdKeyName = thresholdKeyNamePrefix + abtestId + ":" + abtestConfigTag + ":" + abTestCode + ":" + midGroup;
+        // String thresholdKeyName = thresholdKeyNamePrefix + "ad-engine" + ":" + abtestConfigTag + ":" + abTestCode + ":" + midGroup;
+
         String thresholdStr = redisHelper.get(thresholdKeyName);
         double threshold = (thresholdStr != null) ? Double.parseDouble(thresholdStr) : 0.0;
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/FeatureRemoteService.java

@@ -1,8 +1,8 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
-import com.tzld.piaoquan.ad.engine.commons.base.AdActionFeature;
-import com.tzld.piaoquan.ad.engine.commons.base.AdRankItem;
-import com.tzld.piaoquan.ad.engine.commons.base.UserAdFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdActionFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
 import com.tzld.piaoquan.ad.engine.commons.util.CommonCollectionUtils;
 import com.tzld.piaoquan.ad.engine.service.score.convert.FeatureConvert;
 import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
@@ -88,7 +88,7 @@ public class FeatureRemoteService {
 
         feature.setAdId(Long.parseLong(proto.getAdId()));
         feature.setItemFeature(FeatureConvert.convertToJava(proto));
-
+        System.out.println(feature);
         return feature;
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService.java

@@ -1,8 +1,20 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
+import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformBidCreativeDTO;
+import com.tzld.piaoquan.ad.engine.service.score.param.BidRankRecommendRequestParam;
 import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+
+import java.util.List;
+import java.util.Map;
 
 public interface RankService {
 
-    Long adItemRank( RankRecommendRequestParam request);
+    AdRankItem adItemRank(RankRecommendRequestParam request);
+    AdPlatformBidCreativeDTO adBidRank(BidRankRecommendRequestParam request);
+
+    List<AdRankItem> rank (ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems, String configFile);
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCtrCalibrationScorer.java

@@ -0,0 +1,143 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+
+import com.tzld.piaoquan.ad.engine.commons.score.BaseCalibrationScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.CalibrationModel;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+
+
+//@Service
+public class VlogAdCtrCalibrationScorer extends BaseCalibrationScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogAdCtrCalibrationScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+    public VlogAdCtrCalibrationScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userFeature,
+                                    final List<AdRankItem> rankItems) {
+
+        if (userFeature == null || CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        CalibrationModel model = (CalibrationModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<AdRankItem> result = rankItems;
+        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
+
+        LOGGER.debug("calibration ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final List<AdRankItem> items,
+                                        final AdRequestContext requestContext,
+                                        final UserAdFeature user) {
+        long startTime = System.currentTimeMillis();
+        CalibrationModel model = (CalibrationModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
+            }
+        }
+
+        LOGGER.debug("calibration ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+
+    /**
+     * 计算 predict ecpm
+     */
+    public double calcScore(final CalibrationModel model,
+                            final AdRankItem item) {
+        double pctr = 0.0;
+        try {
+            pctr = model.score(item, "ctr");
+        } catch (Exception e) {
+            LOGGER.error("score error for doc={} exception={}", new Object[]{
+                    item.getAdId(), ExceptionUtils.getFullStackTrace(e)});
+        }
+        item.setCtr(pctr);
+        return pctr;
+    }
+
+
+    /**
+     * 并行打分 ecpm
+     *
+     * @param items
+     * @param model
+     */
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final CalibrationModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            items.get(fIndex).setScore(0.0);   //设置为原始值为0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex));
+                    } catch (Exception e) {
+                        LOGGER.error("calibration exception: [{}] [{}]", items.get(fIndex).adId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("ecpm Score {}, Total: {}, Cancel: {}", new Object[]{items.size(), cancel});
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCtrLRScorer.java

@@ -1,13 +1,15 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
 
-import com.tzld.piaoquan.ad.engine.commons.base.*;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.*;
+import com.tzld.piaoquan.recommend.feature.domain.ad.feature.VlogAdCtrLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
 import com.tzld.piaoquan.ad.engine.commons.score.BaseLRModelScorer;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
-import com.tzld.piaoquan.ad.engine.commons.score.feature.VlogAdCtrLRFeatureExtractor;
 import com.tzld.piaoquan.ad.engine.commons.score.model.LRModel;
-import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
+
+
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
@@ -60,8 +62,8 @@ public class VlogAdCtrLRScorer extends BaseLRModelScorer {
     }
 
     private List<AdRankItem> rankByJava(final List<AdRankItem> items,
-                                      final AdRequestContext requestContext,
-                                      final UserAdFeature user) {
+                                        final AdRequestContext requestContext,
+                                        final UserAdFeature user) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
@@ -76,12 +78,10 @@ public class VlogAdCtrLRScorer extends BaseLRModelScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
             }
         }
 
-        Collections.sort(items);
-
         LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
         LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
                 System.currentTimeMillis() - startTime);
@@ -119,14 +119,8 @@ public class VlogAdCtrLRScorer extends BaseLRModelScorer {
                 LOGGER.error("score error for doc={} exception={}", new Object[]{
                         item.getAdId(), ExceptionUtils.getFullStackTrace(e)});
             }
-            // 增加实时特征后打开在线存储日志逻辑
-            //
-            // CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
-            // samples.setLr_samples(lrSamples);
-            // item.setSamples(samples);
-            //
         }
-        item.setScore(pro);
+        item.setCtr(pro);
         return pro;
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCvrCalibrationScorer.java

@@ -0,0 +1,143 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+
+import com.tzld.piaoquan.ad.engine.commons.score.BaseCalibrationScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.CalibrationModel;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+
+
+//@Service
+public class VlogAdCvrCalibrationScorer extends BaseCalibrationScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogAdCvrCalibrationScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+    public VlogAdCvrCalibrationScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userFeature,
+                                    final List<AdRankItem> rankItems) {
+
+        if (userFeature == null || CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        CalibrationModel model = (CalibrationModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<AdRankItem> result = rankItems;
+        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
+
+        LOGGER.debug("calibration ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final List<AdRankItem> items,
+                                        final AdRequestContext requestContext,
+                                        final UserAdFeature user) {
+        long startTime = System.currentTimeMillis();
+        CalibrationModel model = (CalibrationModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
+            }
+        }
+
+        LOGGER.debug("calibration ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+
+    /**
+     * 计算 predict ecpm
+     */
+    public double calcScore(final CalibrationModel model,
+                            final AdRankItem item) {
+        double pcvr = 0.0;
+        try {
+            pcvr = model.score(item, "cvr");
+        } catch (Exception e) {
+            LOGGER.error("score error for doc={} exception={}", new Object[]{
+                    item.getAdId(), ExceptionUtils.getFullStackTrace(e)});
+        }
+        item.setCvr(pcvr);
+        return pcvr;
+    }
+
+
+    /**
+     * 并行打分 ecpm
+     *
+     * @param items
+     * @param model
+     */
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final CalibrationModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            items.get(fIndex).setScore(0.0);   //设置为原始值为0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex));
+                    } catch (Exception e) {
+                        LOGGER.error("calibration exception: [{}] [{}]", items.get(fIndex).adId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("ecpm Score {}, Total: {}, Cancel: {}", new Object[]{items.size(), cancel});
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCvrLRScorer.java

@@ -0,0 +1,181 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+
+import com.tzld.piaoquan.ad.engine.commons.score.BaseLRModelScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.LRModel;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.*;
+import com.tzld.piaoquan.recommend.feature.domain.ad.feature.VlogAdCtrLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.*;
+
+
+//@Service
+public class VlogAdCvrLRScorer extends BaseLRModelScorer {
+
+    private final static int CORE_POOL_SIZE = 64;
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogAdCvrLRScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+    private static final double defaultUserCtrGroupNumber = 10.0;
+    private static final int enterFeedsScoreRatio = 10;
+    private static final int enterFeedsScoreNum = 20;
+
+
+    public VlogAdCvrLRScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+
+    @Override
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userFeature,
+                                    final List<AdRankItem> rankItems) {
+
+        if (userFeature == null || CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<AdRankItem> result = rankItems;
+        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final List<AdRankItem> items,
+                                        final AdRequestContext requestContext,
+                                        final UserAdFeature user) {
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // userAdBytes
+        UserAdBytesFeature userInfoBytes = null;
+        userInfoBytes = new UserAdBytesFeature(user);
+
+        // 所有都参与打分，按照cvr排序
+        multipleScore(items, userInfoBytes, requestContext, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
+            }
+        }
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+
+    /**
+     * 计算 predict cvr
+     */
+    public double calcScore(final LRModel lrModel,
+                            final AdRankItem item,
+                            final UserAdBytesFeature userInfoBytes,
+                            final AdRequestContext requestContext) {
+
+        LRSamples lrSamples = null;
+        VlogAdCtrLRFeatureExtractor bytesFeatureExtractor;
+        bytesFeatureExtractor = new VlogAdCtrLRFeatureExtractor();
+
+        try {
+            AdItemBytesFeature newsInfoBytes = new AdItemBytesFeature(item.getItemFeature());
+            lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
+                    new AdRequestContextBytesFeature(requestContext));
+        } catch (Exception e) {
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getMid()), item.getAdId(),
+                    ExceptionUtils.getFullStackTrace(e)});
+        }
+
+
+        double pro = 0.0;
+        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
+            try {
+                pro = lrModel.score(lrSamples);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", new Object[]{
+                        item.getAdId(), ExceptionUtils.getFullStackTrace(e)});
+            }
+        }
+        item.setCvr(pro);
+        return pro;
+    }
+
+
+    /**
+     * 并行打分
+     *
+     * @param items
+     * @param userInfoBytes
+     * @param requestContext
+     * @param model
+     */
+    private void multipleScore(final List<AdRankItem> items,
+                                  final UserAdBytesFeature userInfoBytes,
+                                  final AdRequestContext requestContext,
+                                  final LRModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).adId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", requestContext.getApptype(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{requestContext.getApptype(), items.size(), cancel});
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdThompsonScorer.java

@@ -0,0 +1,157 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+
+import com.tzld.piaoquan.ad.engine.commons.score.BaseThompsonSamplingScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.LRModel;
+import com.tzld.piaoquan.ad.engine.commons.score.model.ThompsonSamplingModel;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.*;
+import com.tzld.piaoquan.recommend.feature.domain.ad.feature.VlogAdCtrLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.*;
+
+
+//@Service
+public class VlogAdThompsonScorer extends BaseThompsonSamplingScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogAdThompsonScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+    public VlogAdThompsonScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userFeature,
+                                    final List<AdRankItem> rankItems) {
+
+        if (userFeature == null || CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<AdRankItem> result = rankItems;
+        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
+
+        LOGGER.debug("thompson sampling ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final List<AdRankItem> items,
+                                        final AdRequestContext requestContext,
+                                        final UserAdFeature user) {
+        long startTime = System.currentTimeMillis();
+        ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
+            }
+        }
+
+        LOGGER.debug("thompson ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[thompson ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+
+    /**
+     * 计算 predict ecpm
+     */
+    public double calcScore(final ThompsonSamplingModel model,
+                            final AdRankItem item) {
+
+
+        double pctr = 0.0;
+        double pcvr = 0.0;
+        double score = 0.0;
+        double ecpm = 0.0;
+
+        try {
+            pctr = model.score(item, "ctr");
+            pcvr = model.score(item, "cvr");
+            ecpm = item.getCpa() * item.getBid1() * item.getBid2() * pctr * pcvr;
+        } catch (Exception e) {
+            LOGGER.error("score error for doc={} exception={}", new Object[]{
+                    item.getAdId(), ExceptionUtils.getFullStackTrace(e)});
+        }
+        item.setCtr(pctr);
+        item.setCvr(pcvr);
+        item.setEcpm1(ecpm);
+        item.setScore(ecpm);
+        return score;
+    }
+
+
+    /**
+     * 并行打分 ecpm
+     * @param items
+     * @param model
+     */
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final ThompsonSamplingModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            items.get(fIndex).setScore(0.0);   //设置为原始值为0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex));
+                    } catch (Exception e) {
+                        LOGGER.error("thompson exception: [{}] [{}]", items.get(fIndex).adId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("ecpm Score {}, Total: {}, Cancel: {}", new Object[]{ items.size(), cancel});
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogMergeEcpmScorer.java

@@ -0,0 +1,66 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+
+import com.tzld.piaoquan.ad.engine.commons.score.BaseLRModelScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.LRModel;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.List;
+
+
+//@Service
+public class VlogMergeEcpmScorer extends BaseLRModelScorer {
+
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogMergeEcpmScorer.class);
+
+
+    public VlogMergeEcpmScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+
+    @Override
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userFeature,
+                                    final List<AdRankItem> rankItems) {
+
+
+        long startTime = System.currentTimeMillis();
+        List<AdRankItem> result = mergetEcpm(rankItems);
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+
+    public List<AdRankItem> mergetEcpm(List<AdRankItem> items) {
+        long startTime = System.currentTimeMillis();
+
+        for (AdRankItem item : items) {
+            try {
+                double cpa = item.getCpa();
+                double bid1 = item.getBid1();
+                double bid2 = item.getBid2();
+                double pctr = item.getCtr();
+                double pcvr = item.getCvr();
+                double ecpm = cpa * bid1 * bid2 * pcvr * pctr * 1000;
+                item.setEcpm1(ecpm);
+                item.setScore(ecpm);
+            } catch (Exception e) {
+                LOGGER.error("merge Ecpm Score: {} error", "");
+                item.setCtr(0.0);
+            }
+            LOGGER.debug("merge Ecpm ranker , score: {}->{}", "");
+        }
+        Collections.sort(items);
+        return items;
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/convert/FeatureConvert.java

@@ -1,8 +1,8 @@
 package com.tzld.piaoquan.ad.engine.service.score.convert;
 
-import com.tzld.piaoquan.ad.engine.commons.base.AdActionFeature;
-import com.tzld.piaoquan.ad.engine.commons.base.AdItemFeature;
-import com.tzld.piaoquan.ad.engine.commons.base.UserAdFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdActionFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdItemFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
 import com.tzld.piaoquan.recommend.feature.model.feature.*;
 
 /**
@@ -13,90 +13,27 @@ public class FeatureConvert {
     public static UserAdFeatureProto convertToProto(UserAdFeature userAdFeature) {
         UserAdFeatureProto.Builder builder = UserAdFeatureProto.newBuilder();
         builder.setMid(userAdFeature.getMid());
-        builder.setGender(userAdFeature.getGender());
-        builder.setUserType(userAdFeature.getUser_type());
-        builder.setGmtCreate(userAdFeature.getGmt_create());
-        builder.setTags(userAdFeature.getTags());
-        builder.setCategoryName(userAdFeature.getCategory_name());
-        builder.setIsvip(userAdFeature.getIsvip());
-        builder.setIsreward(userAdFeature.getIsreward());
-        builder.setIsad(userAdFeature.getIsad());
-        builder.setIsgood(userAdFeature.getIsgood());
-        builder.setFirstUpDatetime(userAdFeature.getFirst_up_datetime());
-        builder.setLastUpDatetime(userAdFeature.getLast_up_datetime());
-        builder.setNextToLastUpDatetime(userAdFeature.getNext_to_last_up_datetime());
-        builder.setVideos(userAdFeature.getVideos());
-        builder.setIdols(userAdFeature.getIdols());
-        builder.setFans(userAdFeature.getFans());
-        builder.setPlayCount(userAdFeature.getPlay_count());
-        builder.setPlayCountTotal(userAdFeature.getPlay_count_total());
-        builder.setTotalReward(userAdFeature.getTotal_reward());
-        builder.setCurrentdayReward(userAdFeature.getCurrentday_reward());
-        builder.setRewardPerson(userAdFeature.getReward_person());
-        builder.setTotalRewardTimes(userAdFeature.getTotal_reward_times());
-        builder.setRewardVideos(userAdFeature.getReward_videos());
-        builder.setTotalPrice(userAdFeature.getTotal_price());
-        builder.setCurrentdayPrice(userAdFeature.getCurrentday_price());
-        builder.setTotalPriceTimes(userAdFeature.getTotal_price_times());
-        builder.setTotalPricePerson(userAdFeature.getTotal_price_person());
-        builder.setTotalPriceVideos(userAdFeature.getTotal_price_videos());
-        builder.setCgrainUserType(userAdFeature.getCgrain_user_type());
-        builder.setIdentityTagname(userAdFeature.getIdentity_tagname());
-        builder.setOperationTags(userAdFeature.getOperation_tags());
-        builder.setIdentityTagId(userAdFeature.getIdentity_tag_id());
-        builder.setIdentityCreateTime(userAdFeature.getIdentity_create_time());
-        builder.setCountry(userAdFeature.getCountry());
-        builder.setProvince(userAdFeature.getProvince());
-        builder.setCity(userAdFeature.getCity());
 
         builder.setDay1CntFeatures(convertToProto(userAdFeature.getDay1_cnt_features()));
         builder.setDay3CntFeatures(convertToProto(userAdFeature.getDay3_cnt_features()));
         builder.setDay7CntFeatures(convertToProto(userAdFeature.getDay7_cnt_features()));
         builder.setMonth3CntFeatures(convertToProto(userAdFeature.getMonth3_cnt_features()));
 
+        //TODO
+
         return builder.build();
     }
 
-    public static UserAdFeature convertToJava(UserAdFeatureProto userAdFeatureProto){
-            UserAdFeature userAdFeature = new UserAdFeature();
-            userAdFeature.setMid(userAdFeatureProto.getMid());
-            userAdFeature.setGender(userAdFeatureProto.getGender());
-            userAdFeature.setUser_type(userAdFeatureProto.getUserType());
-            userAdFeature.setGmt_create(userAdFeatureProto.getGmtCreate());
-            userAdFeature.setTags(userAdFeatureProto.getTags());
-            userAdFeature.setCategory_name(userAdFeatureProto.getCategoryName());
-            userAdFeature.setIsvip(userAdFeatureProto.getIsvip());
-            userAdFeature.setIsreward(userAdFeatureProto.getIsreward());
-            userAdFeature.setIsad(userAdFeatureProto.getIsad());
-            userAdFeature.setIsgood(userAdFeatureProto.getIsgood());
-            userAdFeature.setFirst_up_datetime(userAdFeatureProto.getFirstUpDatetime());
-            userAdFeature.setLast_up_datetime(userAdFeatureProto.getLastUpDatetime());
-            userAdFeature.setNext_to_last_up_datetime(userAdFeatureProto.getNextToLastUpDatetime());
-            userAdFeature.setVideos(userAdFeatureProto.getVideos());
-            userAdFeature.setIdols(userAdFeatureProto.getIdols());
-            userAdFeature.setFans(userAdFeatureProto.getFans());
-            userAdFeature.setPlay_count(userAdFeatureProto.getPlayCount());
-            userAdFeature.setPlay_count_total(userAdFeatureProto.getPlayCountTotal());
-            userAdFeature.setTotal_reward(userAdFeatureProto.getTotalReward());
-            userAdFeature.setCurrentday_reward(userAdFeatureProto.getCurrentdayReward());
-            userAdFeature.setReward_person(userAdFeatureProto.getRewardPerson());
-            userAdFeature.setTotal_price_times(userAdFeatureProto.getTotalRewardTimes());
-            userAdFeature.setReward_videos(userAdFeatureProto.getRewardVideos());
-            userAdFeature.setTotal_price(userAdFeatureProto.getTotalPrice());
-            userAdFeature.setCurrentday_price(userAdFeatureProto.getCurrentdayPrice());
-            userAdFeature.setTotal_price_times(userAdFeatureProto.getTotalPriceTimes());
-            userAdFeature.setTotal_price_person(userAdFeatureProto.getTotalPricePerson());
-            userAdFeature.setTotal_price_videos(userAdFeatureProto.getTotalPriceVideos());
-            userAdFeature.setCgrain_user_type(userAdFeatureProto.getCgrainUserType());
-            userAdFeature.setIdentity_tagname(userAdFeatureProto.getIdentityTagname());
-            userAdFeature.setOperation_tags(userAdFeatureProto.getOperationTags());
-            userAdFeature.setIdentity_tag_id(userAdFeatureProto.getIdentityTagId());
-            userAdFeature.setIdentity_create_time(userAdFeatureProto.getIdentityCreateTime());
-            userAdFeature.setDay1_cnt_features(convertToJava(userAdFeatureProto.getDay1CntFeatures()));
-            userAdFeature.setDay3_cnt_features(convertToJava(userAdFeatureProto.getDay3CntFeatures()));
-            userAdFeature.setDay7_cnt_features(convertToJava(userAdFeatureProto.getDay7CntFeatures()));
-            userAdFeature.setMonth3_cnt_features(convertToJava(userAdFeatureProto.getMonth3CntFeatures()));
-            return userAdFeature;
+    public static UserAdFeature convertToJava(UserAdFeatureProto userAdFeatureProto) {
+        UserAdFeature userAdFeature = new UserAdFeature();
+        userAdFeature.setMid(userAdFeatureProto.getMid());
+
+        userAdFeature.setDay1_cnt_features(convertToJava(userAdFeatureProto.getDay1CntFeatures()));
+        userAdFeature.setDay3_cnt_features(convertToJava(userAdFeatureProto.getDay3CntFeatures()));
+        userAdFeature.setDay7_cnt_features(convertToJava(userAdFeatureProto.getDay7CntFeatures()));
+        userAdFeature.setMonth3_cnt_features(convertToJava(userAdFeatureProto.getMonth3CntFeatures()));
+
+        return userAdFeature;
     }
 
     public static AdActionFeatureProto convertToProto(AdActionFeature adActionFeature) {
@@ -108,6 +45,7 @@ public class FeatureConvert {
         builder.setCvr(adActionFeature.getCvr());
         return builder.build();
     }
+
     public static AdActionFeature convertToJava(AdActionFeatureProto adActionFeatureProto) {
         AdActionFeature adActionFeature = new AdActionFeature();
         adActionFeature.setAdView(adActionFeatureProto.getAdView());
@@ -127,13 +65,29 @@ public class FeatureConvert {
         adItemFeature.setCampaignId(adItemFeatureProto.getCampaignId());
         adItemFeature.setCampaignCode(adItemFeatureProto.getCampaignCode());
         adItemFeature.setCreativeId(adItemFeatureProto.getCreativeId());
+        //TODO 补充creative advertiser等维度信息
         adItemFeature.setDay1_cnt_features(convertToJava(adItemFeatureProto.getDay1CntFeatures()));
         adItemFeature.setDay3_cnt_features(convertToJava(adItemFeatureProto.getDay3CntFeatures()));
         adItemFeature.setDay7_cnt_features(convertToJava(adItemFeatureProto.getDay7CntFeatures()));
         adItemFeature.setMonth3_cnt_features(convertToJava(adItemFeatureProto.getMonth3CntFeatures()));
+
+        adItemFeature.setCreative_1day_cnt_features(convertToJava(adItemFeatureProto.getCreative1DayCntFeatures()));
+        adItemFeature.setCreative_3day_cnt_features(convertToJava(adItemFeatureProto.getCreative3DayCntFeatures()));
+        adItemFeature.setCreative_7day_cnt_features(convertToJava(adItemFeatureProto.getCreative7DayCntFeatures()));
+        adItemFeature.setCreative_3month_cnt_features(convertToJava(adItemFeatureProto.getCreative3MonthCntFeatures()));
+
+        adItemFeature.setAdvertiser_1day_cnt_features(convertToJava(adItemFeatureProto.getAdvertiser1DayCntFeatures()));
+        adItemFeature.setAdvertiser_3day_cnt_features(convertToJava(adItemFeatureProto.getAdvertiser3DayCntFeatures()));
+        adItemFeature.setAdvertiser_7day_cnt_features(convertToJava(adItemFeatureProto.getAdvertiser7DayCntFeatures()));
+        adItemFeature.setAdvertiser_3month_cnt_features(convertToJava(adItemFeatureProto.getAdvertiser3MonthCntFeatures()));
+
+
+
         return adItemFeature;
+
     }
 
+
     public static AdItemFeatureProto convertToProto(AdItemFeature adItemFeature) {
         AdItemFeatureProto.Builder builder = AdItemFeatureProto.newBuilder();
         builder.setAdId(adItemFeature.getAdId());
@@ -148,6 +102,17 @@ public class FeatureConvert {
         builder.setDay3CntFeatures(convertToProto(adItemFeature.getDay3_cnt_features()));
         builder.setDay7CntFeatures(convertToProto(adItemFeature.getDay7_cnt_features()));
         builder.setMonth3CntFeatures(convertToProto(adItemFeature.getMonth3_cnt_features()));
+
+        builder.setCreative1DayCntFeatures(convertToProto(adItemFeature.getCreative_1day_cnt_features()));
+        builder.setCreative3DayCntFeatures(convertToProto(adItemFeature.getCreative_3day_cnt_features()));
+        builder.setCreative7DayCntFeatures(convertToProto(adItemFeature.getCreative_7day_cnt_features()));
+        builder.setCreative3MonthCntFeatures(convertToProto(adItemFeature.getCreative_3month_cnt_features()));
+
+        builder.setAdvertiser1DayCntFeatures(convertToProto(adItemFeature.getAdvertiser_1day_cnt_features()));
+        builder.setAdvertiser3DayCntFeatures(convertToProto(adItemFeature.getAdvertiser_3day_cnt_features()));
+        builder.setAdvertiser7DayCntFeatures(convertToProto(adItemFeature.getAdvertiser_7day_cnt_features()));
+        builder.setAdvertiser3MonthCntFeatures(convertToProto(adItemFeature.getAdvertiser_3month_cnt_features()));
+
         return builder.build();
     }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/convert/RequestConvert.java

@@ -1,6 +1,7 @@
 package com.tzld.piaoquan.ad.engine.service.score.convert;
 
-import com.tzld.piaoquan.ad.engine.commons.base.AdRequestContext;
+import com.tzld.piaoquan.ad.engine.service.score.param.RecommendRequestParam;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
 
@@ -9,7 +10,7 @@ import java.time.format.DateTimeFormatter;
 
 public class RequestConvert {
 
-    public static ScoreParam requestConvert(RankRecommendRequestParam request){
+    public static ScoreParam requestConvert(RecommendRequestParam request){
         AdRequestContext context=new AdRequestContext();
         context.setApptype(request.getAppType().toString());
         context.setMachineinfoBrand(request.getMachineInfo().getBrand());

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/dto/AdPlatformBidCreativeDTO.java

@@ -0,0 +1,48 @@
+package com.tzld.piaoquan.ad.engine.service.score.dto;
+
+import com.alibaba.fastjson.JSON;
+import lombok.Builder;
+import lombok.Data;
+import lombok.ToString;
+
+@Builder
+@Data
+@ToString
+public class AdPlatformBidCreativeDTO {
+
+    private Long creativeId;
+
+    private String creativeCode;
+
+    private Integer bidType;
+
+    private Double pctr;
+
+    private Double pcvr;
+
+    private Double cpa;
+
+    private Double bid1;
+
+    private Double bid2;
+
+    private Double ecpm1;
+
+    private Double ecpm2;
+
+    public static void main(String[] args) {
+        System.out.println(JSON.toJSONString(AdPlatformBidCreativeDTO.builder()
+                .creativeId(3366L)
+                .creativeCode("CREATIVE_17036704789851370")
+                .bidType(2)
+                .pctr(3.2)
+                .pcvr(1.2)
+                .cpa(40.1)
+                .bid1(1.3)
+                .bid2(2.1)
+                .ecpm1(0.1)
+                .ecpm2(0.2)
+                .build()));
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -1,8 +1,12 @@
 package com.tzld.piaoquan.ad.engine.service.score.impl;
 
-import com.tzld.piaoquan.ad.engine.commons.base.AdItemFeature;
-import com.tzld.piaoquan.ad.engine.commons.base.AdRankItem;
-import com.tzld.piaoquan.ad.engine.commons.base.UserAdFeature;
+import com.alibaba.fastjson.JSONObject;
+import com.google.gson.JsonObject;
+import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformBidCreativeDTO;
+import com.tzld.piaoquan.ad.engine.service.score.param.BidRankRecommendRequestParam;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdItemFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.CommonCollectionUtils;
@@ -12,11 +16,13 @@ import com.tzld.piaoquan.ad.engine.service.score.convert.RequestConvert;
 import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
 import java.util.*;
+import java.util.stream.Collectors;
 
 @Service
 public class RankServiceImpl implements RankService {
@@ -26,7 +32,10 @@ public class RankServiceImpl implements RankService {
     @Autowired
     FeatureRemoteService featureRemoteService;
 
-    public Long adItemRank( RankRecommendRequestParam request){
+    @Autowired
+    RankServiceThompsonImpl rankServiceThompson;
+
+    public AdRankItem adItemRank(RankRecommendRequestParam request){
 
         ScoreParam param= RequestConvert.requestConvert(request);
         UserAdFeature userAdFeature=featureRemoteService.getUserAdFeature(request.getMid());
@@ -38,23 +47,84 @@ public class RankServiceImpl implements RankService {
                         CommonCollectionUtils.toList(request.getAdIdList(),id->id.toString())
                 );
         //兜底方案
+        List<AdRankItem> rankResult;
         if(rankItems==null|| rankItems.size()==0){
-            AdItemFeature feature=new AdItemFeature();
-            feature.setAdId(request.getAdIdList().get(0).toString());
-            AdRankItem adRankItem=new AdRankItem();
-            adRankItem.setAdId(request.getAdIdList().get(0));
-            adRankItem.setItemFeature(feature);
-            rankItems=new ArrayList<>();
-            rankItems.add(adRankItem);
+            rankResult=rankServiceThompson.rank(param, userAdFeature, rankItems,null);
+        }else {
+            rankResult=rank(param, userAdFeature, rankItems,ScorerUtils.BASE_CONF);
         }
-        List<AdRankItem> rankResult = ScorerUtils
-                .getScorerPipeline(
-                        ScorerUtils.BASE_CONF)
-                .scoring(param, userAdFeature, rankItems);
+        log.info("svc=adItemRank rankResult={}", JSONObject.toJSONString(rankResult));
         if (!CollectionUtils.isEmpty(rankResult)) {
-           return rankResult.get(0).getAdId();
+           return rankResult.get(0);
         }else {
-            return null;
+            //空返回值
+            return new AdRankItem();
         }
     }
+
+    @Override
+    public AdPlatformBidCreativeDTO adBidRank(BidRankRecommendRequestParam request) {
+
+        ScoreParam param= RequestConvert.requestConvert(request);
+        UserAdFeature userAdFeature=featureRemoteService.getUserAdFeature(request.getMid());
+        if(userAdFeature==null){
+            userAdFeature=new UserAdFeature();
+        }
+        Map<String,List<AdPlatformBidCreativeDTO>> groupMap=request
+                                        .getCreativeList()
+                                            .stream()
+                .                                  collect(Collectors.groupingBy(creativeDTO -> creativeDTO.getCreativeId()+""));
+
+        List<AdRankItem> rankItems=featureRemoteService.
+                getAllAdFeatureList(
+                        new ArrayList<>(groupMap.keySet())
+                );
+        for(AdRankItem item:rankItems){
+            try {
+                AdPlatformBidCreativeDTO dto=groupMap.get(item.getAdId()+"").get(0);
+                item.setBid1(dto.getBid1());
+                item.setBid2(dto.getBid2());
+                item.setCpa(dto.getCpa());
+            }catch (Exception e){
+                e.printStackTrace();
+            }
+        }
+        List<AdRankItem> rankResult;
+        if(rankItems==null|| rankItems.size()==0){
+            rankResult=rankServiceThompson.rank(param, userAdFeature, rankItems,null);
+        }else {
+            rankResult=rank(param, userAdFeature, rankItems,ScorerUtils.BASE_CONF);
+        }
+        log.info("svc=adBidRank rankResult={}", JSONObject.toJSONString(rankResult));
+
+        AdRankItem topItem=rankResult.get(0);
+
+        AdPlatformBidCreativeDTO result=AdPlatformBidCreativeDTO
+                .builder()
+                .build();
+        BeanUtils.copyProperties(topItem,result);
+        result.setCreativeId(topItem.getAdId());
+        result.setCreativeCode(groupMap.get(topItem.getAdId()+"").get(0).getCreativeCode());
+        if(rankResult.size()<=1){
+            //经验值 待定
+            result.setEcpm2(topItem.getEcpm1()*0.9);
+        }else {
+            result.setEcpm2(rankResult.get(1).getEcpm1());
+        }
+
+
+        rank(param, userAdFeature, rankItems,ScorerUtils.BASE_CONF);
+        return result;
+    }
+
+
+
+    public List<AdRankItem> rank (ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems,String configFile){
+        List<AdRankItem> rankResult = ScorerUtils
+                .getScorerPipeline(configFile)
+                .scoring(param, userAdFeature, rankItems);
+        return rankResult;
+    }
+
+
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceThompsonImpl.java

@@ -0,0 +1,65 @@
+package com.tzld.piaoquan.ad.engine.service.score.impl;
+
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
+import com.tzld.piaoquan.ad.engine.commons.util.CommonCollectionUtils;
+import com.tzld.piaoquan.ad.engine.service.score.FeatureRemoteService;
+import com.tzld.piaoquan.ad.engine.service.score.RankService;
+import com.tzld.piaoquan.ad.engine.service.score.convert.RequestConvert;
+import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformBidCreativeDTO;
+import com.tzld.piaoquan.ad.engine.service.score.param.BidRankRecommendRequestParam;
+import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdItemFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Service
+public class RankServiceThompsonImpl implements RankService {
+
+    private final static Logger log = LoggerFactory.getLogger(RankServiceThompsonImpl.class);
+
+    @Autowired
+    FeatureRemoteService featureRemoteService;
+
+    public AdRankItem adItemRank(RankRecommendRequestParam request) {
+
+        ScoreParam param = RequestConvert.requestConvert(request);
+        UserAdFeature userAdFeature = new UserAdFeature();
+        List<AdRankItem> rankItems = featureRemoteService.getAllAdFeatureList(
+                CommonCollectionUtils.toList(request.getAdIdList(), id -> id.toString())
+        );
+        List<AdRankItem> rankResult = ScorerUtils
+                .getScorerPipeline(ScorerUtils.THOMPSON_CONF)
+                .scoring(param, userAdFeature, rankItems);
+
+        if (!CollectionUtils.isEmpty(rankResult)) {
+            return rankResult.get(0);
+        } else {
+            return null;
+        }
+    }
+
+    @Override
+    public AdPlatformBidCreativeDTO adBidRank(BidRankRecommendRequestParam request) {
+        return null;
+    }
+
+    public List<AdRankItem> rank (ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems,String configFile){
+        if(configFile==null){
+            configFile=ScorerUtils.THOMPSON_CONF;
+        }
+        List<AdRankItem> rankResult = ScorerUtils
+                .getScorerPipeline(configFile)
+                .scoring(param, userAdFeature, rankItems);
+        return rankResult;
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/param/BidRankRecommendRequestParam.java

@@ -0,0 +1,13 @@
+package com.tzld.piaoquan.ad.engine.service.score.param;
+
+import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformBidCreativeDTO;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class BidRankRecommendRequestParam extends RecommendRequestParam{
+
+     List<AdPlatformBidCreativeDTO> creativeList;
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/param/RankRecommendRequestParam.java

@@ -5,10 +5,7 @@ import lombok.Data;
 import java.util.List;
 
 @Data
-public class RankRecommendRequestParam {
-     MachineInfoParam machineInfo;
-     Long appType;
-     String mid;
+public class RankRecommendRequestParam extends RecommendRequestParam{
      List<Long> adIdList;
 
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/param/RecommendRequestParam.java

@@ -0,0 +1,12 @@
+package com.tzld.piaoquan.ad.engine.service.score.param;
+
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+public class RecommendRequestParam {
+    MachineInfoParam machineInfo;
+    Long appType;
+    String mid;
+}