Merge branch 'cid_layer_copc' of algorithm/ad-engine into master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/helper/CreativeUserLayerDataHelper.java

@@ -0,0 +1,105 @@
+package com.tzld.piaoquan.ad.engine.commons.helper;
+
+import com.alibaba.fastjson.JSONObject;
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.model.OSSObject;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.*;
+
+@Slf4j
+@Component
+public class CreativeUserLayerDataHelper {
+    @Autowired
+    private OSS ossClient;
+
+    private static final String BUCKET_NAME = "art-recommend";
+    private static final String filePath = "ad_engine_strategy/cid_layer_copc.txt";
+    private static final String keyFormat = "%s:%s:%s";
+
+    private volatile static Map<String, Double> cidMap = Collections.emptyMap();
+
+    // 服务启动时初始化数据
+    @PostConstruct
+    public void init() {
+        Map<String, Double> map = readTextFromOss(filePath);
+        cidMap = Collections.unmodifiableMap(map);
+    }
+
+    // 每5分钟更新一次数据
+    @Scheduled(fixedRate = 5 * 60 * 1000)
+    public void scheduledUpdate() {
+        Map<String, Double> map = readTextFromOss(filePath);
+        cidMap = Collections.unmodifiableMap(map);
+    }
+
+    public static Double getCopc(String model, String cid, String layer) {
+        if (null != cidMap) {
+            String key = String.format(keyFormat, model, cid, layer);
+            if (cidMap.containsKey(key)) {
+                return cidMap.get(key);
+            }
+            key = String.format(keyFormat, model, cid, "sum");
+            if (cidMap.containsKey(key)) {
+                return cidMap.get(key);
+            }
+        }
+        return null;
+    }
+
+    private HashMap<String, Double> readTextFromOss(String ossFilePath) {
+        try {
+            List<JSONObject> list = readOssFile(ossFilePath);
+            return parseCidData(list);
+        } catch (Exception e) {
+            log.error("get modelVersionPath error", e);
+        }
+        return new HashMap<>();
+    }
+
+    private HashMap<String, Double> parseCidData(List<JSONObject> list) {
+        HashMap<String, Double> map = new HashMap<>();
+        if (null != list) {
+            for (JSONObject json : list) {
+                String model = json.getString("model");
+                String cid = json.getString("cid");
+                String layer = json.getString("layer");
+                Double copc = json.getDoubleValue("final_copc");
+                String key = String.format(keyFormat, model, cid, layer);
+                map.put(key, copc);
+            }
+        }
+        return map;
+    }
+
+    private List<JSONObject> readOssFile(String ossFilePath) {
+        if (!ossClient.doesObjectExist(BUCKET_NAME, ossFilePath)) {
+            log.error("ossFilePath not exist: {}", ossFilePath);
+            return null;
+        }
+        List<JSONObject> ossJson = new ArrayList<>();
+        try (OSSObject ossObj = ossClient.getObject(BUCKET_NAME, ossFilePath)) {
+            BufferedReader reader = new BufferedReader(new InputStreamReader(ossObj.getObjectContent()));
+            String line = reader.readLine();
+            String[] columns = line.split("\t");
+            while ((line = reader.readLine()) != null) {
+                String[] values = line.split("\t");
+                JSONObject jsonObj = new JSONObject();
+                for (int i = 0; i < columns.length; i++) {
+                    jsonObj.put(columns[i], values[i]);
+                }
+                ossJson.add(jsonObj);
+            }
+        } catch (Exception e) {
+            log.error("load oss file error: {}, ", ossFilePath, e);
+        }
+        log.info("load oss file: {}, size: {}", ossFilePath, ossJson.size());
+        return ossJson;
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy847.java

@@ -3,7 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.score.strategy;
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
-import com.tzld.piaoquan.ad.engine.commons.helper.AnomalousCidDataHelper;
+import com.tzld.piaoquan.ad.engine.commons.helper.CreativeUserLayerDataHelper;
 import com.tzld.piaoquan.ad.engine.commons.helper.DnnCidDataHelper;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
@@ -343,6 +343,12 @@ public class RankStrategyBy847 extends RankStrategyBasic {
 
         String calibModelName = paramsMap.getOrDefault("calibModelName", "dnnV3");
         calculateCtcvrScore(result, request, scoreParam, calibModelName, reqFeature);
+
+        double minValidCopc = NumberUtils.toDouble(paramsMap.getOrDefault("minValidCopc", "0.8"));
+        double maxValidCopc = NumberUtils.toDouble(paramsMap.getOrDefault("maxValidCopc", "10"));
+        double minCopc = NumberUtils.toDouble(paramsMap.getOrDefault("minCopc", "0.2"));
+        double maxCopc = NumberUtils.toDouble(paramsMap.getOrDefault("maxCopc", "2.5"));
+        calibrationCidCtcvr(result, calibModelName, reqFeature, minValidCopc, maxValidCopc, minCopc, maxCopc);
         if (CollectionUtils.isEmpty(result)) {
             log.error("calculateCtcvrScore result is empty");
         }
@@ -363,7 +369,7 @@ public class RankStrategyBy847 extends RankStrategyBasic {
         double expLowerWeight = NumberUtils.toDouble(paramsMap.getOrDefault("expLowerWeight", "0.2"));
         double expUpperWeight = NumberUtils.toDouble(paramsMap.getOrDefault("expUpperWeight", "1.0"));
         double expScale = NumberUtils.toDouble(paramsMap.getOrDefault("expScale", "10.0"));
-        double anomWeight = NumberUtils.toDouble(paramsMap.getOrDefault("anomWeight", "1.0"));
+        int openH5 = NumberUtils.toInt(paramsMap.getOrDefault("openH5", "0"));
         for (AdRankItem item : result) {
             double bid = item.getCpa();
             if (scoreParam.getExpCodeSet().contains(correctCpaExp1) || scoreParam.getExpCodeSet().contains(correctCpaExp2)) {
@@ -376,12 +382,10 @@ public class RankStrategyBy847 extends RankStrategyBasic {
                 isGuaranteeType = true;
             }
 
-            double anomW = 1.0;
-            boolean h5Page = this.landingH5Page(item);
-            if (h5Page) {
-                if (AnomalousCidDataHelper.contains(item.getAdId())) {
-                    anomW = anomWeight;
-                }
+            // h5 降权
+            double h5Weight = 1;
+            if (openH5 > 0) {
+                h5Weight = this.getH5SuppressWeight(item);
             }
 
             // 控制曝光权重
@@ -397,14 +401,14 @@ public class RankStrategyBy847 extends RankStrategyBasic {
             double layerAndCreativeWeight = getLayerAndCreativeWeight(layerAndCreativeWeightMapKey);
             double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
             double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(isGuaranteedFlow, item.getExt());
-            double score = anomW * expWeight * item.getLrScore() * bid * scoreCoefficient * guaranteeScoreCoefficient * layerAndCreativeWeight;
+            double score = h5Weight * expWeight * item.getLrScore() * bid * scoreCoefficient * guaranteeScoreCoefficient * layerAndCreativeWeight;
             item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
             item.getScoreMap().put("bid", bid);
             item.getScoreMap().put("cpmCoefficient", cpmCoefficient);
             item.getScoreMap().put("scoreCoefficient", scoreCoefficient);
-            item.getScoreMap().put("anom", anomW);
+            item.getScoreMap().put("h5", h5Weight);
             item.getFeatureMap().putAll(userFeatureMap);
             item.getFeatureMap().putAll(sceneFeatureMap);
 
@@ -1009,4 +1013,27 @@ public class RankStrategyBy847 extends RankStrategyBasic {
         double weight = Math.log(exp + 1) / scale;
         return Math.min(Math.max(lowerWeight, weight), upperWeight);
     }
+
+    private void calibrationCidCtcvr(List<AdRankItem> items, String modelName, Map<String, String> reqFeature,
+                                     double minValid, double maxValid,
+                                     double minVal, double maxVal) {
+        try {
+            String layer = reqFeature.get("layer_l4");
+            for (AdRankItem item : items) {
+                String cid = String.valueOf(item.getAdId());
+                Double diff = CreativeUserLayerDataHelper.getCopc(modelName, cid, layer);
+                if (null != diff) {
+                    double newDiff = Math.min(Math.max(minVal, diff), maxVal);
+                    if (newDiff < minValid || newDiff > maxValid) {
+                        double ctcvr = item.getLrScore() * newDiff;
+                        item.getScoreMap().put("cidCtcvrScore", ctcvr);
+                        item.getScoreMap().put("ctcvrScore", ctcvr);
+                        item.setLrScore(ctcvr);
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("calibCidScore error", e);
+        }
+    }
 }