1

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/VideoAdThompsonModel.java

@@ -0,0 +1,113 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+import com.tzld.piaoquan.ad.engine.commons.score.rankitem.VideoAdThompsonRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdActionFeature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import org.apache.commons.math3.distribution.BetaDistribution;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.math.BigDecimal;
+import java.util.HashMap;
+import java.util.Map;
+
+public class VideoAdThompsonModel extends Model{
+    private static final Logger LOGGER = LoggerFactory.getLogger(VideoAdThompsonModel.class);
+
+    private Map<String, AdActionFeature> featureCache = new HashMap<>();
+    private Map<Long, AdActionFeature> adActionFeatureMap = new HashMap<>();
+
+    @Override
+    public int getModelSize() {
+        return featureCache.size();
+    }
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws Exception {
+        Map<String, AdActionFeature> initModel = new HashMap<>();
+        Map<Long, AdActionFeature> tempAdActionFeatureMap =new HashMap<>();
+        try (BufferedReader reader = new BufferedReader(in)) {
+            String line = null;
+            int size = 0;
+            while ((line = reader.readLine()) != null) {
+                String[] items = line.split("\t");
+                if (items.length < 5) {
+                    continue;
+                }
+//                headvideoid   STRING
+//    ,advertiserid STRING
+//    ,adid         STRING
+//    ,creativeid   STRING
+//    ,conversion   STRING
+//    ,view         STRING
+//    ,click        STRING
+//    ,cpa          STRING
+                double click=new BigDecimal(items[7].trim()).doubleValue();
+                double conversion=new BigDecimal(items[7].trim()).doubleValue();
+                double view= new BigDecimal(items[6].trim()).doubleValue();
+                Long creativeId=new BigDecimal(items[4].trim()).longValue();
+                AdActionFeature adActionFeature=tempAdActionFeatureMap.getOrDefault(creativeId,new AdActionFeature());
+                adActionFeature.setAdClick(adActionFeature.getAdClick()+click);
+                adActionFeature.setAdConversion(adActionFeature.getAdConversion()+conversion);
+                adActionFeature.setAdView(adActionFeature.getAdView()+view);
+                tempAdActionFeatureMap.put(creativeId,adActionFeature);
+                this.putFeature(
+                        initModel,
+                        new BigDecimal(items[0].trim()).longValue(),
+                        creativeId,
+                        view,
+                        click,
+                        conversion
+                );
+                size++;
+            }
+            this.featureCache = initModel;
+            this.adActionFeatureMap=tempAdActionFeatureMap;
+            LOGGER.info("[UNION THOMPSON SAMPLING MODEL LOAD] mode load over, size {}", size);
+        } catch (
+                Exception e) {
+            LOGGER.info("[UNION THOMPSON SAMPLING MODEL LOAD] mode load error ", e);
+        } finally {
+            in.close();
+        }
+
+        return true;
+    }
+
+    private void putFeature(Map<String, AdActionFeature> modelCache, Long videoId, Long creativeId, Double view, Double click, Double conversion) {
+        AdActionFeature adActionFeature =  new AdActionFeature();
+        adActionFeature.setAdView(view);
+        adActionFeature.setAdClick(click);
+        adActionFeature.setAdConversion(conversion);
+        modelCache.put(videoId+"_"+creativeId, adActionFeature);
+
+    }
+
+    public double ctrScore(VideoAdThompsonRankItem item) {
+        AdActionFeature adActionFeature = this.featureCache.getOrDefault(item.getVideoId()+"_"+item.getAdId(),  new AdActionFeature());
+        if(adActionFeature.getAdView()<1000){
+            adActionFeature=this.adActionFeatureMap.getOrDefault(item.getAdId(), new AdActionFeature());
+        }
+        int alpha_cvr = (int) adActionFeature.getAdClick() + 10;
+        int beta_cvr = 100 + (int) adActionFeature.getAdView() - (int) adActionFeature.getAdClick();
+        return this.betaSampler(alpha_cvr, beta_cvr);
+    }
+
+    public double cvrScore(VideoAdThompsonRankItem item) {
+        AdActionFeature adActionFeature = this.featureCache.getOrDefault(item.getVideoId()+"_"+item.getAdId(),  new AdActionFeature());
+        if(adActionFeature.getAdView()<1000){
+            adActionFeature=this.adActionFeatureMap.getOrDefault(item.getAdId(), new AdActionFeature());
+        }
+        int alpha_cvr = (int) adActionFeature.getAdConversion() + 10;
+        int beta_cvr = 100 + (int) adActionFeature.getAdClick() - (int) adActionFeature.getAdConversion();
+        return this.betaSampler(alpha_cvr, beta_cvr);
+    }
+
+    public double betaSampler(double alpha, double beta) {
+        BetaDistribution betaSample = new BetaDistribution(alpha, beta);
+        return betaSample.sample();
+
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/rankitem/VideoAdThompsonRankItem.java

@@ -0,0 +1,15 @@
+package com.tzld.piaoquan.ad.engine.commons.score.rankitem;
+
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+
+public class VideoAdThompsonRankItem extends AdRankItem {
+    public long getVideoId() {
+        return videoId;
+    }
+
+    public void setVideoId(long videoId) {
+        this.videoId = videoId;
+    }
+
+    public long videoId;
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VideoAdThompsonScorer.java

@@ -0,0 +1,152 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+import com.tzld.piaoquan.ad.engine.commons.score.BaseThompsonSamplingScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.ThompsonSamplingModel;
+import com.tzld.piaoquan.ad.engine.commons.score.model.VideoAdThompsonModel;
+import com.tzld.piaoquan.ad.engine.commons.score.rankitem.VideoAdThompsonRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+
+public class VideoAdThompsonScorer extends BaseThompsonSamplingScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VideoAdThompsonScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(8);
+
+    public VideoAdThompsonScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userFeature,
+                                    final List<AdRankItem> rankItems) {
+
+        if (userFeature == null || CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        VideoAdThompsonModel model = (VideoAdThompsonModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<AdRankItem> result = rankItems;
+        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
+
+        LOGGER.debug("thompson sampling ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final List<AdRankItem> items,
+                                        final AdRequestContext requestContext,
+                                        final UserAdFeature user) {
+        long startTime = System.currentTimeMillis();
+        VideoAdThompsonModel model = (VideoAdThompsonModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, model);
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
+            }
+        }
+
+        LOGGER.debug("thompson ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[thompson ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+
+    /**
+     * 计算 predict ecpm
+     */
+    public double calcScore(final VideoAdThompsonModel model,
+                            final AdRankItem item) {
+
+
+        double pctr = 0.0;
+        double pcvr = 0.0;
+        double ecpm = 0.0;
+        VideoAdThompsonRankItem  rankItem=(VideoAdThompsonRankItem)item;
+        try {
+            pctr = model.ctrScore(rankItem);
+            pcvr = model.cvrScore(rankItem);
+            ecpm = item.getCpa() * item.getBid1() * item.getBid2() * pctr * pcvr;
+        } catch (Exception e) {
+            LOGGER.error("score error for doc={} exception={}", new Object[]{
+                    item.getAdId(), ExceptionUtils.getFullStackTrace(e)});
+        }
+        item.setTf_ctr(pctr);
+        item.setTf_cvr(pcvr);
+        item.setEcpm1(ecpm);
+        item.setScore(ecpm);
+        return ecpm;
+    }
+
+
+    /**
+     * 并行打分 ecpm
+     * @param items
+     * @param model
+     */
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final VideoAdThompsonModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            items.get(fIndex).setScore(0.0);   //设置为原始值为0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex));
+                    } catch (Exception e) {
+                        LOGGER.error("thompson exception: [{}] [{}]", items.get(fIndex).adId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("ecpm Score {}, Total: {}, Cancel: {}", new Object[]{ items.size(), cancel});
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -168,7 +168,124 @@ public class RankServiceImpl implements RankService {
         }
     }
 
+    public AdRankItem videoAdThompsonItemRank(RankRecommendRequestParam request){
+        ScoreParam param= RequestConvert.requestConvert(request);
+        LocalDateTime currentTime = LocalDateTime.now();
+        int currentHour = currentTime.getHour();
+        int dayOfWeek = currentTime.getDayOfWeek().getValue();
+        param.getRequestContext().setHour(currentHour+"");
+        param.getRequestContext().setWeek(dayOfWeek+"");
+        param.getRequestContext().setRegion(request.getRegion().replace("省",""));
+        param.getRequestContext().setCity(request.getCity().replace("市",""));
+        param.getRequestContext().setDay(currentTime.format(dateFormatter));
+
+        UserAdFeature userAdFeature=featureRemoteService.getUserAdFeature(request.getMid());
+        if(userAdFeature==null){
+            userAdFeature=new UserAdFeature();
+        }
+        Map<Long,List<AdPlatformCreativeDTO>> groupMap=request
+                .getAdIdList()
+                .stream()
+                .collect(Collectors.groupingBy(creativeDTO -> creativeDTO.getCreativeId()));
+        Map<Long, AdRankItem> cache=adCreativeFeatureContainer.getAll(new ArrayList<>(groupMap.keySet()));
+        List<AdRankItem> rankItems=Collections.emptyList();
+        if(!cache.isEmpty()){
+            rankItems=new LinkedList<>(cache.values());
+        }
+        //避免recommend-feature出问题
+        if(rankItems==null|| rankItems.size()==0){
+            rankItems=new LinkedList<>();
+            for(Long adId:groupMap.keySet()){
+                AdRankItem item=new AdRankItem();
+                item.setAdId(adId);
+                item.setItemFeature(new AdItemFeature());
+                rankItems.add(item);
+            }
+        }
+        boolean inCpcPidExp=false;
+        if (request.getAdAbExpArr() != null && request.getAdAbExpArr().size() != 0) {
+            for (Map<String, Object> map : request.getAdAbExpArr() ) {
+                if (map.getOrDefault("abExpCode", "").equals(cpcPidExpCode)) {
+                    inCpcPidExp = true;
+                }
+            }
+        }
+        double lambda=-1d;
+        if(inCpcPidExp){
+            for(AdRankItem item:rankItems){
+                try {
+                    AdPlatformCreativeDTO dto=groupMap.get(item.getAdId()).get(0);
+                    item.setBid1(dto.getBid1());
+                    item.setBid2(dto.getBid2());
+                    item.setCpa(dto.getCpa());
+                    item.setPidLambda(1d);
+                }catch (Exception e){
+                    log.error("rankItems info error itemId={}",item.getAdId());
+                    e.printStackTrace();
+                }
+            }
+        }else {
+            for(AdRankItem item:rankItems){
+                try {
+                    AdPlatformCreativeDTO dto=groupMap.get(item.getAdId()).get(0);
+                    item.setBid1(dto.getBid1());
+                    item.setBid2(dto.getBid2());
+                    lambda= PidLambdaForCpcContainer.getPidLambda(item.getAdId());
+                    if(lambda<0){
+                        item.setCpa(dto.getCpa());
+                        item.setPidLambda(1);
+                    }else {
+                        if(dto.getCpa()>1&&lambda<=1){
+                            lambda=2d;
+                        }
+                        item.setCpa(lambda);
+                        item.setPidLambda(1d);
+                    }
+                    item.setCpa(dto.getCpa());
+                    item.setPidLambda(1d);
+                }catch (Exception e){
+                    log.error("rankItems info error itemId={}",item.getAdId());
+                    e.printStackTrace();
+                }
+            }
+        }
+
+        // 兜底方案
+        List<AdRankItem> rankResult;
+        if (inCpcPidExp) {
+            rankResult = rank(param, userAdFeature, rankItems, ScorerUtils.CVR_ADJUSTING);
+        } else {
+            rankResult = rank(param, userAdFeature, rankItems, ScorerUtils.BASE_CONF);
+        }
 
+        if (!CollectionUtils.isEmpty(rankResult)) {
+            JSONObject object=new JSONObject();
+            object.put("mid",request.getMid());
+            object.put("adid",rankResult.get(0).getAdId());
+            object.put("type",rankResult.get(0).getScore_type());
+            object.put("pctr",rankResult.get(0).getCtr());
+            object.put("pcvr",rankResult.get(0).getCvr());
+            object.put("score",rankResult.get(0).getScore());
+            object.put("pidLambda",rankResult.get(0).getPidLambda());
+            object.put("lrsamples",rankResult.get(0).getLrSampleString());
+            object.put("dataTime",currentTime.format(timeFormatter));
+            object.put("creativeId",rankResult.get(0).getAdId());
+            log.info("svc=adItemRank {}", JSONObject.toJSONString(object));
+            object.remove("lrsamples");
+            if(inCpcPidExp){
+                AdPlatformCreativeDTO dto=groupMap.get(rankResult.get(0).getAdId()).get(0);
+                object.put("cpa",dto.getCpa()*dto.getBid1());
+                object.put("oCpa",dto.getCpa());
+                log.info("svc=cpc_pid obj={}", JSONObject.toJSONString(object));
+            }else {
+                log.info("svc=pid_log obj={}", JSONObject.toJSONString(object));
+            }
+            return rankResult.get(0);
+        }else {
+            //空返回值
+            return new AdRankItem();
+        }
+    }
 
     @Override
     public AdPlatformCreativeDTO adBidRank(BidRankRecommendRequestParam request) {
@@ -265,18 +382,6 @@ public class RankServiceImpl implements RankService {
                     item.setCpa(lambda);
                     item.setPidLambda(1d);
                 }
-//                if(lambda<0){
-//                    item.setCpa(dto.getCpa());
-//                    item.setPidLambda(dto.getCpa()*0.6);
-//                }else {
-//                    if(dto.getCpa()>1&&lambda<=1){
-//                        lambda=2d;
-//                    }
-//                    item.setCpa(dto.getCpa());
-//                    item.setPidLambda(lambda);
-//                }
-//                item.setCpa(dto.getCpa());
-//                item.setPidLambda(PidLambdaContainer.getPidLambda(item.getAdId()));
                 rankItems.add(item);
             }
             rankResult=rankServiceThompson.rank(param, userAdFeature, rankItems,null);
@@ -362,21 +467,11 @@ public class RankServiceImpl implements RankService {
         double lambda=-1d;
         for(AdRankItem item:rankItems){
             try {
-//                AdPlatformBidCreativeDTO dto=groupMap.get(item.getAdId()+"").get(0);
+
                 AdPlatformCreativeDTO dto=groupMap.get(item.getAdId()).get(0);
                 item.setBid1(dto.getBid1());
                 item.setBid2(dto.getBid2());
                 lambda=PidLambdaV2Container.getPidLambda(item.getAdId());
-//                if(lambda<0){
-//                    item.setCpa(dto.getCpa());
-//                    item.setPidLambda(0.6);
-//                }else {
-//                    if(dto.getCpa()>1&&lambda<=1){
-//                        lambda=2d;
-//                    }
-//                    item.setCpa(lambda);
-//                    item.setPidLambda(1d);
-//                }
                 if(lambda<0){
                     item.setCpa(dto.getCpa());
                     item.setPidLambda(dto.getCpa()*0.6);
@@ -403,16 +498,7 @@ public class RankServiceImpl implements RankService {
                 item.setAdId(dto.getCreativeId());
                 item.setItemFeature(new AdItemFeature());
                 lambda=PidLambdaV2Container.getPidLambda(item.getAdId());
-//                if(lambda<0){
-//                    item.setCpa(dto.getCpa());
-//                    item.setPidLambda(0.6);
-//                }else {
-//                    if(dto.getCpa()>1&&lambda<=1){
-//                        lambda=2d;
-//                    }
-//                    item.setCpa(lambda);
-//                    item.setPidLambda(1d);
-//                }
+
                 if(lambda<0){
                     item.setCpa(dto.getCpa());
                     item.setPidLambda(dto.getCpa()*0.6);
@@ -423,8 +509,6 @@ public class RankServiceImpl implements RankService {
                     item.setCpa(dto.getCpa());
                     item.setPidLambda(lambda);
                 }
-//                item.setCpa(dto.getCpa());
-//                item.setPidLambda(PidLambdaContainer.getPidLambda(item.getAdId()));
                 rankItems.add(item);
             }
             rankResult=rankServiceThompson.rank(param, userAdFeature, rankItems,null);
@@ -443,8 +527,6 @@ public class RankServiceImpl implements RankService {
         result.setPcvr(topItem.getCvr());
         result.setCreativeCode(groupMap.get(topItem.getAdId()).get(0).getCreativeCode());
         double realECpm=0d;
-        //经验值 待定
-//        realECpm=topItem.getEcpm1();
         realECpm=topItem.getEcpm2();
 
         if(realECpm>cpmMax/1000d){
@@ -464,7 +546,6 @@ public class RankServiceImpl implements RankService {
         object.put("pcvr",topItem.getCvr());
         object.put("lrsamples",topItem.getLrSampleString());
         object.put("pidLambda",topItem.getPidLambda());
-        //临时加入供pid v2使用
         object.put("realECpm",realECpm);
         object.put("creativeId",result.getCreativeId());
         //CPA还原

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceThompsonImpl.java

@@ -45,6 +45,7 @@ public class RankServiceThompsonImpl implements RankService {
         }
     }
 
+
     public AdRankItem adItemRankV2(RankRecommendRequestParam request) {
 
         ScoreParam param = RequestConvert.requestConvert(request);