feat:添加定向打分融合

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/impl/LogHubServiceImpl.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.ad.engine.service.log.impl;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Sets;
 import com.tzld.commons.aliyun.log.AliyunLogManager;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
@@ -35,6 +36,8 @@ public class LogHubServiceImpl implements LogHubService {
     @Autowired
     private AliyunLogManager aliyunLogManager;
 
+    private static final Set<String> FEATURE_FIELD_SET = Sets.newHashSet();
+
     @Override
     public void scoreLogUpload(ScoreParam param, List<AdPlatformCreativeDTO> adIdList, List<AdRankItem> rankItems, RecommendRequestParam requestParam, String scoreStrategy, String abCode) {
         logUploadThreadPool.execute(new Runnable() {
@@ -59,16 +62,29 @@ public class LogHubServiceImpl implements LogHubService {
                 }
                 logMap.put("expids", abExpCode);
 
-                // List<JSONObject> scoreResult = new ArrayList<>();
-                // for (AdRankItem rankItem : rankItems) {
-                //     JSONObject json = new JSONObject();
-                //     json.put("cid", rankItem.getAdId());
-                //     json.put("score", rankItem.getScore());
-                //     rankItem.getExt().put("weight", rankItem.getWeight());
-                //     json.put("feature", rankItem.getExt());
-                //     scoreResult.add(json);
-                // }
-                // logMap.put("scoreResult", JSON.toJSONString(scoreResult));
+                List<JSONObject> scoreResult = new ArrayList<>();
+                for (AdRankItem rankItem : rankItems) {
+                    JSONObject json = new JSONObject();
+                    json.put("cid", rankItem.getAdId());
+                    json.put("score", rankItem.getScore());
+                    JSONObject featureJson = new JSONObject();
+                    for (Map.Entry<String, String> entry : rankItem.getFeatureMap().entrySet()) {
+                        if (FEATURE_FIELD_SET.contains(entry.getKey())) {
+                            featureJson.put(entry.getKey(), entry.getValue());
+                        }
+                    }
+                    for (Map.Entry<String, Map<String, String>> entry : rankItem.getMetaFeatureMap().entrySet()) {
+                        if (FEATURE_FIELD_SET.contains(entry.getKey())) {
+                            featureJson.put(entry.getKey(), entry.getValue());
+                        }
+                    }
+                    json.put("feature", featureJson);
+                    scoreResult.add(json);
+                }
+
+                JSONObject result = new JSONObject();
+                result.put("score", scoreResult);
+                logMap.put("result", JSON.toJSONString(result));
 
                 AdRankItem top1 = rankItems.get(0);
                 logMap.put("cid", top1.getAdId());
@@ -76,15 +92,15 @@ public class LogHubServiceImpl implements LogHubService {
 
                 Map<String, String> featureMap = top1.getFeatureMap();
                 featureMap.put("weight", String.valueOf(top1.getWeight()));
-                logMap.put("allFeatureMap", JSON.toJSONString(featureMap));
+                logMap.put("allfeature", JSON.toJSONString(featureMap));
 
-                logMap.put("metaFeatureMap", JSON.toJSONString(top1.getMetaFeatureMap()));
+                logMap.put("metafeature", JSON.toJSONString(top1.getMetaFeatureMap()));
 
                 logMap.put("abcode", param.getAdAbGroup());
                 logMap.put("scorestrategy", scoreStrategy);
                 logMap.put("apptype", context.getApptype());
                 logMap.put("extinfo", new JSONObject());
-
+                logMap.put("scoremap", new JSONObject());
                 // logMap.put("creativeList", JSON.toJSONString(adIdList));
                 // if (Objects.nonNull(requestParam.getStatisticsLog())) {
                 //     extInfo.put("earlyAdIds", requestParam.getStatisticsLog().getEarlyAdIds());

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/container/PredictPidContainer.java

@@ -65,17 +65,17 @@ public class PredictPidContainer {
 
     @PostConstruct
     private void init(){
-        instanceClient();
-        final Runnable task = new Runnable() {
-            public void run() {
-                try {
-                    loadAndCalIfNeed();
-                }catch (Exception e){
-                    e.printStackTrace();
-                }
-            }
-        };
-        scheduler.scheduleAtFixedRate(task, 0, SCHEDULE_PERIOD, TimeUnit.MINUTES); // 10分钟
+        // instanceClient();
+        // final Runnable task = new Runnable() {
+        //     public void run() {
+        //         try {
+        //             loadAndCalIfNeed();
+        //         }catch (Exception e){
+        //             e.printStackTrace();
+        //         }
+        //     }
+        // };
+        // scheduler.scheduleAtFixedRate(task, 0, SCHEDULE_PERIOD, TimeUnit.MINUTES); // 10分钟
     }
 
     private void instanceClient(){

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService680.java

@@ -1,14 +1,11 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
-import com.alibaba.fastjson.JSON;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
-import com.tzld.piaoquan.ad.engine.commons.score.ScorerPipeline;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.ExtractorUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
-import com.tzld.piaoquan.ad.engine.service.score.convert.RequestConvert;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
@@ -56,13 +53,14 @@ public class RankService680 {
         Map<String, Double> midTimeDiffMap = this.parseC1FeatureListToTimeDiffMap(midActionList, ts);
         Map<String, Double> actionStaticMap = this.parseC1FeatureListToActionStaticMap(midActionList);
 
-        Map<String, String> d1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
-        this.handleD1Feature(d1Feature, userFeatureMap);
+        Map<String, String> d1Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+        Map<String, String> d2Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf_rank", new HashMap<>());
+
+        Map<String, Map<String, Double>> vidRankMaps = this.parseD2FeatureMap(d2Feature);
 
         Map<String, String> e1Feature = userFeature.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
         Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
 
-        Map<String, String> d2Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf_rank", new HashMap<>());
 
         List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
         for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
@@ -71,6 +69,7 @@ public class RankService680 {
             adRankItem.setCreativeCode(dto.getCreativeCode());
             adRankItem.setAdVerId(dto.getAdVerId());
             adRankItem.setVideoId(request.getVideoId());
+            adRankItem.setCpa(dto.getCpa());
 
             String cidStr = dto.getCreativeId().toString();
             Map<String, String> cidFeatureMap = new HashMap<>();
@@ -87,7 +86,9 @@ public class RankService680 {
 
             this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
 
-            this.handleD2Feature(d2Feature, cidFeatureMap, cidStr);
+            this.handleD1Feature(d1Feature, cidFeatureMap);
+
+            this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
 
             String title = b1Feature.getOrDefault("cidtitle", "");
             this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
@@ -110,7 +111,7 @@ public class RankService680 {
         List<AdRankItem> items = ScorerUtils.getScorerPipeline(ScorerUtils.LR_ROV_SCORE_20240626)
                 .scoring(new HashMap<>(), userFeatureMap, adRankItems);
 
-        List<AdRankItem> results =new ArrayList<>(items.size());
+        List<AdRankItem> results = new ArrayList<>(items.size());
         for (AdRankItem item : items) {
             AdRankItem result = new AdRankItem();
             result.setAdId(item.getAdId());
@@ -118,20 +119,22 @@ public class RankService680 {
             result.setAdVerId(item.getAdVerId());
             result.setVideoId(item.getVideoId());
             result.setLrScore(item.getLrScore());
-            result.setScore(item.getLrScore());
-            result.getFeatureMap().put("lrModelFeature", JSON.toJSONString(item.getFeatureMap()));
-            result.getFeatureMap().put("userFeature", JSON.toJSONString(userFeatureMap));
 
-            if (MapUtils.isNotEmpty(videoFeature)){
+            result.setScore(item.getLrScore() * item.getCpa());
+
+            result.getFeatureMap().putAll(item.getFeatureMap());
+            result.getFeatureMap().putAll(userFeatureMap);
+
+            if (MapUtils.isNotEmpty(videoFeature)) {
                 result.getMetaFeatureMap().putAll(videoFeature);
             }
-            if (MapUtils.isNotEmpty(userFeature)){
+            if (MapUtils.isNotEmpty(userFeature)) {
                 result.getMetaFeatureMap().putAll(userFeature);
             }
-            if (allAdVerFeature.containsKey(item.getAdVerId())){
+            if (allAdVerFeature.containsKey(item.getAdVerId())) {
                 result.getMetaFeatureMap().putAll(allAdVerFeature.get(item.getAdVerId()));
             }
-            if (allCidFeature.containsKey(String.valueOf(item.getAdId()))){
+            if (allCidFeature.containsKey(String.valueOf(item.getAdId()))) {
                 result.getMetaFeatureMap().putAll(allCidFeature.get(String.valueOf(item.getAdId())));
             }
         }
@@ -246,6 +249,7 @@ public class RankService680 {
                         Tuple5 tuple5 = new Tuple5(rList[1], rList[2], rList[3], rList[4], rList[5]);
                         return new TupleMapEntry<>(rList[0], tuple5);
                     })
+                    // TODO 倒排
                     .sorted((a, b) -> Integer.compare(Integer.parseInt(b.value.f1), Integer.parseInt(a.value.f1)))
                     .collect(Collectors.toList());
         }
@@ -268,25 +272,25 @@ public class RankService680 {
 
     private void handleC1UIFeature(Map<String, Double> midTimeDiffMap, Map<String, Double> midActionStatic, Map<String, String> featureMap, String cid) {
         if (midTimeDiffMap.containsKey("timediff_view_" + cid)) {
-            featureMap.put("timediff_view_" + cid, String.valueOf(midTimeDiffMap.getOrDefault("timediff_view_" + cid, 0.0)));
+            featureMap.put("timediff_view", String.valueOf(midTimeDiffMap.getOrDefault("timediff_view_" + cid, 0.0)));
         }
         if (midTimeDiffMap.containsKey("timediff_click_" + cid)) {
-            featureMap.put("timediff_click_" + cid, String.valueOf(midTimeDiffMap.getOrDefault("timediff_click_" + cid, 0.0)));
+            featureMap.put("timediff_click", String.valueOf(midTimeDiffMap.getOrDefault("timediff_click_" + cid, 0.0)));
         }
         if (midTimeDiffMap.containsKey("timediff_conver_" + cid)) {
-            featureMap.put("timediff_conver_" + cid, String.valueOf(midTimeDiffMap.getOrDefault("timediff_conver_" + cid, 0.0)));
+            featureMap.put("timediff_conver", String.valueOf(midTimeDiffMap.getOrDefault("timediff_conver_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_view_" + cid)) {
-            featureMap.put("actionstatic_view_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)));
+            featureMap.put("actionstatic_view", String.valueOf(midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_click_" + cid)) {
-            featureMap.put("actionstatic_click_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)));
+            featureMap.put("actionstatic_click", String.valueOf(midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_conver_" + cid)) {
-            featureMap.put("actionstatic_conver_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0)));
+            featureMap.put("actionstatic_conver", String.valueOf(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_income_" + cid)) {
-            featureMap.put("actionstatic_income_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0)));
+            featureMap.put("actionstatic_income", String.valueOf(midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
             double ctr = NumUtil.div(
@@ -325,21 +329,11 @@ public class RankService680 {
         }
     }
 
-    private void handleD2Feature(Map<String, String> d2Feature, Map<String, String> featureMap, String cid) {
-        if (MapUtils.isEmpty(d2Feature)) {
+    private void handleD2Feature(Map<String, Map<String, Double>> vidRankMaps, Map<String, String> featureMap, String cid) {
+        if (MapUtils.isEmpty(vidRankMaps)) {
             return;
         }
 
-        Map<String, Map<String, Double>> vidRankMaps = new HashMap<>();
-        for (Map.Entry<String, String> entry : d2Feature.entrySet()) {
-            String key = entry.getKey();
-            String value = entry.getValue();
-            Map<String, Double> valueMap = Arrays.stream(value.split(","))
-                    .map(r -> r.split(":"))
-                    .collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
-            vidRankMaps.put(key, valueMap);
-        }
-
         List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
         List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
 
@@ -393,7 +387,7 @@ public class RankService680 {
             long tsHistory = Long.parseLong(entry.value.f1);
             long click = Long.parseLong(entry.value.f2);
             long conver = Long.parseLong(entry.value.f3);
-            long d = (ts - tsHistory) / 3600 / 21;
+            long d = (ts - tsHistory) / 3600 / 24;
             if (!midTimeDiffMap.containsKey("timediff_view_" + cid)) {
                 midTimeDiffMap.put("timediff_view_" + cid, NumUtil.div(1, d));
             }
@@ -431,38 +425,55 @@ public class RankService680 {
         return midActionStaticsMap;
     }
 
+    private Map<String, Map<String, Double>> parseD2FeatureMap(Map<String, String> d2Feature) {
+        Map<String, Map<String, Double>> vidRankMaps = new HashMap<>();
+        for (Map.Entry<String, String> entry : d2Feature.entrySet()) {
+            String key = entry.getKey();
+            String value = entry.getValue();
+            Map<String, Double> valueMap = Arrays.stream(value.split(","))
+                    .map(r -> r.split(":"))
+                    .collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
+            vidRankMaps.put(key, valueMap);
+        }
+        return vidRankMaps;
+    }
+
     private void readBucketFile() {
-        InputStream resourceStream = RankService680.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
-        if (resourceStream != null) {
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
-                Map<String, double[]> bucketsMap = new HashMap<>();
-                Map<String, Double> bucketsLen = new HashMap<>();
-                String line;
-                while ((line = reader.readLine()) != null) {
-                    // 替换空格和换行符，过滤空行
-                    line = line.replace(" ", "").replaceAll("\n", "");
-                    if (!line.isEmpty()) {
-                        String[] rList = line.split("\t");
-                        if (rList.length == 3) {
-                            String key = rList[0];
-                            double value1 = Double.parseDouble(rList[1]);
-                            bucketsLen.put(key, value1);
-                            double[] value2 = Arrays.stream(rList[2].split(","))
-                                    .mapToDouble(Double::valueOf)
-                                    .toArray();
-                            bucketsMap.put(key, value2);
+        if (MapUtils.isNotEmpty(bucketsMap)) {
+            return;
+        }
+        synchronized (this) {
+            InputStream resourceStream = RankService680.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+            if (resourceStream != null) {
+                try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                    Map<String, double[]> bucketsMap = new HashMap<>();
+                    Map<String, Double> bucketsLen = new HashMap<>();
+                    String line;
+                    while ((line = reader.readLine()) != null) {
+                        // 替换空格和换行符，过滤空行
+                        line = line.replace(" ", "").replaceAll("\n", "");
+                        if (!line.isEmpty()) {
+                            String[] rList = line.split("\t");
+                            if (rList.length == 3) {
+                                String key = rList[0];
+                                double value1 = Double.parseDouble(rList[1]);
+                                bucketsLen.put(key, value1);
+                                double[] value2 = Arrays.stream(rList[2].split(","))
+                                        .mapToDouble(Double::valueOf)
+                                        .toArray();
+                                bucketsMap.put(key, value2);
+                            }
                         }
                     }
+                    this.bucketsMap = bucketsMap;
+                    this.bucketsLen = bucketsLen;
+                } catch (IOException e) {
+                    log.error("something is wrong in parse bucket file:" + e);
                 }
-                this.bucketsMap = bucketsMap;
-                this.bucketsLen = bucketsLen;
-            } catch (IOException e) {
-                log.error("something is wrong in parse bucket file:" + e);
+            } else {
+                log.error("no bucket file");
             }
-        } else {
-            log.error("no bucket file");
         }
-
     }
 
     private Map<String, String> featureBucket(Map<String, String> featureMap) {