dev 1.0 with rank

ad-engine-commons/pom.xml

@@ -10,7 +10,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>ad-engine-commons</artifactId>
-    <version>1.0.0.release</version>
+    <version>1.0.0</version>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>1.8</maven.compiler.source>

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/VlogAdCtrLRScorer.java

@@ -1,186 +0,0 @@
-package com.tzld.piaoquan.ad.engine.commons.score;
-
-
-import com.tzld.piaoquan.ad.engine.commons.base.*;
-import com.tzld.piaoquan.ad.engine.commons.score.feature.VlogAdCtrLRFeatureExtractor;
-import com.tzld.piaoquan.ad.engine.commons.score.model.LRModel;
-import com.tzld.piaoquan.recommend.server.gen.recommend.LRSamples;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.*;
-
-
-//@Service
-public class VlogAdCtrLRScorer extends BaseLRModelScorer {
-
-    private final static int CORE_POOL_SIZE = 64;
-
-    private static final int LOCAL_TIME_OUT = 150;
-    private final static Logger LOGGER = LoggerFactory.getLogger(VlogAdCtrLRScorer.class);
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
-    private static final double defaultUserCtrGroupNumber = 10.0;
-    private static final int enterFeedsScoreRatio = 10;
-    private static final int enterFeedsScoreNum = 20;
-
-
-    public VlogAdCtrLRScorer(ScorerConfigInfo configInfo) {
-        super(configInfo);
-    }
-
-
-    @Override
-    public List<AdRankItem> scoring(final ScoreParam param,
-                                    final UserAdFeature userFeature,
-                                    final List<AdRankItem> rankItems) {
-
-        if (userFeature == null || CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<AdRankItem> result = rankItems;
-        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
-
-        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<AdRankItem> rankByJava(final List<AdRankItem> items,
-                                      final AdRequestContext requestContext,
-                                      final UserAdFeature user) {
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        // userAdBytes
-        UserAdBytesFeature userInfoBytes = null;
-        userInfoBytes = new UserAdBytesFeature(user);
-
-        // 所有都参与打分，按照ctr排序
-        multipleCtrScore(items, userInfoBytes, requestContext, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
-            }
-        }
-
-        Collections.sort(items);
-
-        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-
-    /**
-     * 计算 predict ctr
-     */
-    public double calcScore(final LRModel lrModel,
-                            final AdRankItem item,
-                            final UserAdBytesFeature userInfoBytes,
-                            final AdRequestContext requestContext) {
-
-        LRSamples lrSamples = null;
-        VlogAdCtrLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new VlogAdCtrLRFeatureExtractor();
-
-        try {
-            AdItemBytesFeature newsInfoBytes = new AdItemBytesFeature(item.getItemFeature());
-            lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
-                    new AdRequestContextBytesFeature(requestContext));
-        } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getMid()), item.getAdId(),
-                    ExceptionUtils.getFullStackTrace(e)});
-        }
-
-
-        double pro = 0.0;
-        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
-            try {
-                pro = lrModel.score(lrSamples);
-            } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getAdId(), ExceptionUtils.getFullStackTrace(e)});
-            }
-            // 增加实时特征后打开在线存储日志逻辑
-            //
-            // CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
-            // samples.setLr_samples(lrSamples);
-            // item.setSamples(samples);
-            //
-        }
-        item.setScore(pro);
-        return pro;
-    }
-
-
-    /**
-     * 并行打分
-     *
-     * @param items
-     * @param userInfoBytes
-     * @param requestContext
-     * @param model
-     */
-    private void multipleCtrScore(final List<AdRankItem> items,
-                                  final UserAdBytesFeature userInfoBytes,
-                                  final AdRequestContext requestContext,
-                                  final LRModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
-                    } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).adId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", requestContext.getApptype(),
-                            ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{requestContext.getApptype(), items.size(), cancel});
-    }
-}

ad-engine-server/src/main/java/com/tzld/piaoquan/ad/engine/server/controller/AdRecommendController.java

@@ -23,7 +23,6 @@ public class AdRecommendController {
     @RequestMapping("/top1/basic")
     public Map<String,Object> adRecommendTop1Basic(@RequestBody RankRecommendRequestParam request){
         Long adId= rankService.adItemRank(request);
-//        scorer.scoring(param,userFeature,rankItems).get(0);
         HashMap map =new HashMap();
         map.put("code","0");
         map.put("msg","success");

ad-engine-server/src/main/resources/feeds_score_config_baseline.conf

@@ -2,6 +2,6 @@ scorer-config = {
   related-score-config = {
     scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogAdCtrLRScorer"
     scorer-priority = 99
-    model-path = "model/model_new.txt"
+    model-path = "model/model_ad_ctr.txt"
   }
 }

ad-engine-service/pom.xml

@@ -29,6 +29,11 @@
             <artifactId>recommend-feature-client</artifactId>
             <version>1.0.0</version>
         </dependency>
+        <dependency>
+            <groupId>com.tzld.piaoquan</groupId>
+            <artifactId>ad-engine-commons</artifactId>
+            <version>1.0.0</version>
+        </dependency>
 
     </dependencies>
 </project>

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/convert/FeatureConvert.java

@@ -92,6 +92,10 @@ public class FeatureConvert {
             userAdFeature.setOperation_tags(userAdFeatureProto.getOperationTags());
             userAdFeature.setIdentity_tag_id(userAdFeatureProto.getIdentityTagId());
             userAdFeature.setIdentity_create_time(userAdFeatureProto.getIdentityCreateTime());
+            userAdFeature.setDay1_cnt_features(convertToJava(userAdFeatureProto.getDay1CntFeatures()));
+            userAdFeature.setDay3_cnt_features(convertToJava(userAdFeatureProto.getDay3CntFeatures()));
+            userAdFeature.setDay7_cnt_features(convertToJava(userAdFeatureProto.getDay7CntFeatures()));
+            userAdFeature.setMonth3_cnt_features(convertToJava(userAdFeatureProto.getMonth3CntFeatures()));
             return userAdFeature;
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/convert/RequestConvert.java

@@ -4,6 +4,9 @@ import com.tzld.piaoquan.ad.engine.commons.base.AdRequestContext;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
 
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+
 public class RequestConvert {
 
     public static ScoreParam requestConvert(RankRecommendRequestParam request){
@@ -13,8 +16,13 @@ public class RequestConvert {
         context.setMachineinfoModel(request.getMachineInfo().getModel());
         context.setMachineinfoSdkversion(request.getMachineInfo().getSdkVersion());
         context.setMachineinfoWchatversion(request.getMachineInfo().getWeChatVersion());
+        LocalDateTime date=LocalDateTime.now();
+        context.setHour(date.getHour()+"");
+        context.setDay(date.format(DateTimeFormatter.ofPattern("yyyyMMdd")));
+        context.setWeek(date.getDayOfWeek().getValue()+"");
         ScoreParam scoreParam=new ScoreParam();
         scoreParam.setRequestContext(context);
         return scoreParam;
     }
+
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -22,8 +22,6 @@ import java.util.*;
 public class RankServiceImpl implements RankService {
 
     private final static Logger log = LoggerFactory.getLogger(RankServiceImpl.class);
-    @Autowired
-    ApplicationContext applicationContext;
 
     @Autowired
     FeatureRemoteService featureRemoteService;
@@ -42,9 +40,7 @@ public class RankServiceImpl implements RankService {
         List<AdRankItem> rankResult = ScorerUtils
                 .getScorerPipeline(
                         ScorerUtils.BASE_CONF)
-//                .scoring(convert(param), userFeature, rankItems);
                 .scoring(param, userAdFeature, rankItems);
-//        Collections.sort(rankResult, Comparator.comparingDouble(ad -> -ad.getScore()));
         if (!CollectionUtils.isEmpty(rankResult)) {
            return rankResult.get(0).getAdId();
         }else {