Merge branch 'feature_20240621_zhaohaipeng_fm' into test

.gitignore

@@ -34,4 +34,5 @@ build/
 apollo-cache-dir
 sentinel
 weblog
-logs
+logs
+LOG_PATH_IS_UNDEFINED

ad-engine-server/pom.xml

@@ -31,6 +31,7 @@
                     <include>**/*.xml</include>
                     <include>**/*.conf</include>
                     <include>**/*.properties</include>
+                    <include>**/*.txt</include>
                 </includes>
                 <filtering>true</filtering>
             </resource>

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/FeatureService.java

@@ -1,5 +1,7 @@
 package com.tzld.piaoquan.ad.engine.service.feature;
 
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
@@ -71,6 +73,7 @@ public class FeatureService {
         protos.add(genWithMid("alg_mid_feature_share_tags", param.getMid()));
 
         Map<String, String> featureMap = remoteService.getFeature(protos);
+        featureMap = this.featureStrCover(featureMap);
         Feature feature = new Feature();
 
         for (Map.Entry<String, String> entry : featureMap.entrySet()) {
@@ -91,7 +94,7 @@ public class FeatureService {
                     Map<String, Map<String, String>> avFeatureMap = feature.getAdVerFeature().getOrDefault(id, new HashMap<>());
                     avFeatureMap.put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
                     }, Collections.emptyMap()));
-                    feature.getAdVerFeature().put(tableName, avFeatureMap);
+                    feature.getAdVerFeature().put(id, avFeatureMap);
                     break;
                 case "m":
                     feature.getUserFeature().put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
@@ -202,4 +205,19 @@ public class FeatureService {
                 .build();
     }
 
+    private Map<String, String> featureStrCover(Map<String, String> metaFeatureMap) {
+        Map<String, String> newFeatureMap = new HashMap<>();
+        for (Map.Entry<String, String> entry : metaFeatureMap.entrySet()) {
+            String key = entry.getKey();
+            try {
+                Map<String, String> valueMap = JSONUtils.fromJson(entry.getValue(), new TypeToken<Map<String, String>>() {
+                }, Collections.emptyMap());
+                String value = valueMap.getOrDefault("feature", "{}");
+                newFeatureMap.put(key, value);
+            } catch (Exception e) {
+                log.error("featureStrCover error: ", e);
+            }
+        }
+        return newFeatureMap;
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/impl/LogHubServiceImpl.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.ad.engine.service.log.impl;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Sets;
 import com.tzld.commons.aliyun.log.AliyunLogManager;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
@@ -35,6 +36,8 @@ public class LogHubServiceImpl implements LogHubService {
     @Autowired
     private AliyunLogManager aliyunLogManager;
 
+    private static final Set<String> FEATURE_FIELD_SET = Sets.newHashSet();
+
     @Override
     public void scoreLogUpload(ScoreParam param, List<AdPlatformCreativeDTO> adIdList, List<AdRankItem> rankItems, RecommendRequestParam requestParam, String scoreStrategy, String abCode) {
         logUploadThreadPool.execute(new Runnable() {
@@ -59,16 +62,29 @@ public class LogHubServiceImpl implements LogHubService {
                 }
                 logMap.put("expids", abExpCode);
 
-                // List<JSONObject> scoreResult = new ArrayList<>();
-                // for (AdRankItem rankItem : rankItems) {
-                //     JSONObject json = new JSONObject();
-                //     json.put("cid", rankItem.getAdId());
-                //     json.put("score", rankItem.getScore());
-                //     rankItem.getExt().put("weight", rankItem.getWeight());
-                //     json.put("feature", rankItem.getExt());
-                //     scoreResult.add(json);
-                // }
-                // logMap.put("scoreResult", JSON.toJSONString(scoreResult));
+                List<JSONObject> scoreResult = new ArrayList<>();
+                for (AdRankItem rankItem : rankItems) {
+                    JSONObject json = new JSONObject();
+                    json.put("cid", rankItem.getAdId());
+                    json.put("score", rankItem.getScore());
+                    JSONObject featureJson = new JSONObject();
+                    for (Map.Entry<String, String> entry : rankItem.getFeatureMap().entrySet()) {
+                        if (FEATURE_FIELD_SET.contains(entry.getKey())) {
+                            featureJson.put(entry.getKey(), entry.getValue());
+                        }
+                    }
+                    for (Map.Entry<String, Map<String, String>> entry : rankItem.getMetaFeatureMap().entrySet()) {
+                        if (FEATURE_FIELD_SET.contains(entry.getKey())) {
+                            featureJson.put(entry.getKey(), entry.getValue());
+                        }
+                    }
+                    json.put("feature", featureJson);
+                    scoreResult.add(json);
+                }
+
+                JSONObject result = new JSONObject();
+                result.put("score", scoreResult);
+                logMap.put("result", JSON.toJSONString(result));
 
                 AdRankItem top1 = rankItems.get(0);
                 logMap.put("cid", top1.getAdId());
@@ -76,15 +92,15 @@ public class LogHubServiceImpl implements LogHubService {
 
                 Map<String, String> featureMap = top1.getFeatureMap();
                 featureMap.put("weight", String.valueOf(top1.getWeight()));
-                logMap.put("allFeatureMap", JSON.toJSONString(featureMap));
+                logMap.put("allfeature", JSON.toJSONString(featureMap));
 
-                logMap.put("metaFeatureMap", JSON.toJSONString(top1.getMetaFeatureMap()));
+                logMap.put("metafeature", JSON.toJSONString(top1.getMetaFeatureMap()));
 
                 logMap.put("abcode", param.getAdAbGroup());
                 logMap.put("scorestrategy", scoreStrategy);
                 logMap.put("apptype", context.getApptype());
                 logMap.put("extinfo", new JSONObject());
-
+                logMap.put("scoremap", new JSONObject());
                 // logMap.put("creativeList", JSON.toJSONString(adIdList));
                 // if (Objects.nonNull(requestParam.getStatisticsLog())) {
                 //     extInfo.put("earlyAdIds", requestParam.getStatisticsLog().getEarlyAdIds());

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/container/PredictPidContainer.java

@@ -65,17 +65,17 @@ public class PredictPidContainer {
 
     @PostConstruct
     private void init(){
-        instanceClient();
-        final Runnable task = new Runnable() {
-            public void run() {
-                try {
-                    loadAndCalIfNeed();
-                }catch (Exception e){
-                    e.printStackTrace();
-                }
-            }
-        };
-        scheduler.scheduleAtFixedRate(task, 0, SCHEDULE_PERIOD, TimeUnit.MINUTES); // 10分钟
+        // instanceClient();
+        // final Runnable task = new Runnable() {
+        //     public void run() {
+        //         try {
+        //             loadAndCalIfNeed();
+        //         }catch (Exception e){
+        //             e.printStackTrace();
+        //         }
+        //     }
+        // };
+        // scheduler.scheduleAtFixedRate(task, 0, SCHEDULE_PERIOD, TimeUnit.MINUTES); // 10分钟
     }
 
     private void instanceClient(){

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService680.java

@@ -1,14 +1,11 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
-import com.alibaba.fastjson.JSON;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
-import com.tzld.piaoquan.ad.engine.commons.score.ScorerPipeline;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.ExtractorUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
-import com.tzld.piaoquan.ad.engine.service.score.convert.RequestConvert;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
@@ -56,13 +53,14 @@ public class RankService680 {
         Map<String, Double> midTimeDiffMap = this.parseC1FeatureListToTimeDiffMap(midActionList, ts);
         Map<String, Double> actionStaticMap = this.parseC1FeatureListToActionStaticMap(midActionList);
 
-        Map<String, String> d1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
-        this.handleD1Feature(d1Feature, userFeatureMap);
+        Map<String, String> d1Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+        Map<String, String> d2Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf_rank", new HashMap<>());
+
+        Map<String, Map<String, Double>> vidRankMaps = this.parseD2FeatureMap(d2Feature);
 
         Map<String, String> e1Feature = userFeature.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
         Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
 
-        Map<String, String> d2Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf_rank", new HashMap<>());
 
         List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
         for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
@@ -71,6 +69,7 @@ public class RankService680 {
             adRankItem.setCreativeCode(dto.getCreativeCode());
             adRankItem.setAdVerId(dto.getAdVerId());
             adRankItem.setVideoId(request.getVideoId());
+            adRankItem.setCpa(dto.getCpa());
 
             String cidStr = dto.getCreativeId().toString();
             Map<String, String> cidFeatureMap = new HashMap<>();
@@ -87,7 +86,9 @@ public class RankService680 {
 
             this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
 
-            this.handleD2Feature(d2Feature, cidFeatureMap, cidStr);
+            this.handleD1Feature(d1Feature, cidFeatureMap);
+
+            this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
 
             String title = b1Feature.getOrDefault("cidtitle", "");
             this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
@@ -110,35 +111,39 @@ public class RankService680 {
         List<AdRankItem> items = ScorerUtils.getScorerPipeline(ScorerUtils.LR_ROV_SCORE_20240626)
                 .scoring(new HashMap<>(), userFeatureMap, adRankItems);
 
-        List<AdRankItem> results =new ArrayList<>(items.size());
+        List<AdRankItem> result = new ArrayList<>(items.size());
         for (AdRankItem item : items) {
-            AdRankItem result = new AdRankItem();
-            result.setAdId(item.getAdId());
-            result.setCreativeCode(item.getCreativeCode());
-            result.setAdVerId(item.getAdVerId());
-            result.setVideoId(item.getVideoId());
-            result.setLrScore(item.getLrScore());
-            result.setScore(item.getLrScore());
-            result.getFeatureMap().put("lrModelFeature", JSON.toJSONString(item.getFeatureMap()));
-            result.getFeatureMap().put("userFeature", JSON.toJSONString(userFeatureMap));
-
-            if (MapUtils.isNotEmpty(videoFeature)){
-                result.getMetaFeatureMap().putAll(videoFeature);
+            AdRankItem adRankItem = new AdRankItem();
+            adRankItem.setAdId(item.getAdId());
+            adRankItem.setCreativeCode(item.getCreativeCode());
+            adRankItem.setAdVerId(item.getAdVerId());
+            adRankItem.setVideoId(item.getVideoId());
+            adRankItem.setLrScore(item.getLrScore());
+
+            adRankItem.setScore(item.getLrScore() * item.getCpa());
+
+            adRankItem.getFeatureMap().putAll(item.getFeatureMap());
+            adRankItem.getFeatureMap().putAll(userFeatureMap);
+
+            if (MapUtils.isNotEmpty(videoFeature)) {
+                adRankItem.getMetaFeatureMap().putAll(videoFeature);
             }
-            if (MapUtils.isNotEmpty(userFeature)){
-                result.getMetaFeatureMap().putAll(userFeature);
+            if (MapUtils.isNotEmpty(userFeature)) {
+                adRankItem.getMetaFeatureMap().putAll(userFeature);
             }
-            if (allAdVerFeature.containsKey(item.getAdVerId())){
-                result.getMetaFeatureMap().putAll(allAdVerFeature.get(item.getAdVerId()));
+            if (allAdVerFeature.containsKey(item.getAdVerId())) {
+                adRankItem.getMetaFeatureMap().putAll(allAdVerFeature.get(item.getAdVerId()));
             }
-            if (allCidFeature.containsKey(String.valueOf(item.getAdId()))){
-                result.getMetaFeatureMap().putAll(allCidFeature.get(String.valueOf(item.getAdId())));
+            if (allCidFeature.containsKey(String.valueOf(item.getAdId()))) {
+                adRankItem.getMetaFeatureMap().putAll(allCidFeature.get(String.valueOf(item.getAdId())));
             }
+
+            result.add(adRankItem);
         }
 
-        Collections.sort(results);
+        Collections.sort(result);
 
-        return results;
+        return result;
     }
 
     private Feature getFeature(ScoreParam param, RankRecommendRequestParam request) {
@@ -246,6 +251,7 @@ public class RankService680 {
                         Tuple5 tuple5 = new Tuple5(rList[1], rList[2], rList[3], rList[4], rList[5]);
                         return new TupleMapEntry<>(rList[0], tuple5);
                     })
+                    // TODO 倒排
                     .sorted((a, b) -> Integer.compare(Integer.parseInt(b.value.f1), Integer.parseInt(a.value.f1)))
                     .collect(Collectors.toList());
         }
@@ -268,25 +274,25 @@ public class RankService680 {
 
     private void handleC1UIFeature(Map<String, Double> midTimeDiffMap, Map<String, Double> midActionStatic, Map<String, String> featureMap, String cid) {
         if (midTimeDiffMap.containsKey("timediff_view_" + cid)) {
-            featureMap.put("timediff_view_" + cid, String.valueOf(midTimeDiffMap.getOrDefault("timediff_view_" + cid, 0.0)));
+            featureMap.put("timediff_view", String.valueOf(midTimeDiffMap.getOrDefault("timediff_view_" + cid, 0.0)));
         }
         if (midTimeDiffMap.containsKey("timediff_click_" + cid)) {
-            featureMap.put("timediff_click_" + cid, String.valueOf(midTimeDiffMap.getOrDefault("timediff_click_" + cid, 0.0)));
+            featureMap.put("timediff_click", String.valueOf(midTimeDiffMap.getOrDefault("timediff_click_" + cid, 0.0)));
         }
         if (midTimeDiffMap.containsKey("timediff_conver_" + cid)) {
-            featureMap.put("timediff_conver_" + cid, String.valueOf(midTimeDiffMap.getOrDefault("timediff_conver_" + cid, 0.0)));
+            featureMap.put("timediff_conver", String.valueOf(midTimeDiffMap.getOrDefault("timediff_conver_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_view_" + cid)) {
-            featureMap.put("actionstatic_view_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)));
+            featureMap.put("actionstatic_view", String.valueOf(midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_click_" + cid)) {
-            featureMap.put("actionstatic_click_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)));
+            featureMap.put("actionstatic_click", String.valueOf(midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_conver_" + cid)) {
-            featureMap.put("actionstatic_conver_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0)));
+            featureMap.put("actionstatic_conver", String.valueOf(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_income_" + cid)) {
-            featureMap.put("actionstatic_income_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0)));
+            featureMap.put("actionstatic_income", String.valueOf(midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
             double ctr = NumUtil.div(
@@ -325,21 +331,11 @@ public class RankService680 {
         }
     }
 
-    private void handleD2Feature(Map<String, String> d2Feature, Map<String, String> featureMap, String cid) {
-        if (MapUtils.isEmpty(d2Feature)) {
+    private void handleD2Feature(Map<String, Map<String, Double>> vidRankMaps, Map<String, String> featureMap, String cid) {
+        if (MapUtils.isEmpty(vidRankMaps)) {
             return;
         }
 
-        Map<String, Map<String, Double>> vidRankMaps = new HashMap<>();
-        for (Map.Entry<String, String> entry : d2Feature.entrySet()) {
-            String key = entry.getKey();
-            String value = entry.getValue();
-            Map<String, Double> valueMap = Arrays.stream(value.split(","))
-                    .map(r -> r.split(":"))
-                    .collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
-            vidRankMaps.put(key, valueMap);
-        }
-
         List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
         List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
 
@@ -393,7 +389,7 @@ public class RankService680 {
             long tsHistory = Long.parseLong(entry.value.f1);
             long click = Long.parseLong(entry.value.f2);
             long conver = Long.parseLong(entry.value.f3);
-            long d = (ts - tsHistory) / 3600 / 21;
+            long d = (ts - tsHistory) / 3600 / 24;
             if (!midTimeDiffMap.containsKey("timediff_view_" + cid)) {
                 midTimeDiffMap.put("timediff_view_" + cid, NumUtil.div(1, d));
             }
@@ -431,38 +427,55 @@ public class RankService680 {
         return midActionStaticsMap;
     }
 
+    private Map<String, Map<String, Double>> parseD2FeatureMap(Map<String, String> d2Feature) {
+        Map<String, Map<String, Double>> vidRankMaps = new HashMap<>();
+        for (Map.Entry<String, String> entry : d2Feature.entrySet()) {
+            String key = entry.getKey();
+            String value = entry.getValue();
+            Map<String, Double> valueMap = Arrays.stream(value.split(","))
+                    .map(r -> r.split(":"))
+                    .collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
+            vidRankMaps.put(key, valueMap);
+        }
+        return vidRankMaps;
+    }
+
     private void readBucketFile() {
-        InputStream resourceStream = RankService680.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
-        if (resourceStream != null) {
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
-                Map<String, double[]> bucketsMap = new HashMap<>();
-                Map<String, Double> bucketsLen = new HashMap<>();
-                String line;
-                while ((line = reader.readLine()) != null) {
-                    // 替换空格和换行符，过滤空行
-                    line = line.replace(" ", "").replaceAll("\n", "");
-                    if (!line.isEmpty()) {
-                        String[] rList = line.split("\t");
-                        if (rList.length == 3) {
-                            String key = rList[0];
-                            double value1 = Double.parseDouble(rList[1]);
-                            bucketsLen.put(key, value1);
-                            double[] value2 = Arrays.stream(rList[2].split(","))
-                                    .mapToDouble(Double::valueOf)
-                                    .toArray();
-                            bucketsMap.put(key, value2);
+        if (MapUtils.isNotEmpty(bucketsMap)) {
+            return;
+        }
+        synchronized (this) {
+            InputStream resourceStream = RankService680.class.getClassLoader().getResourceAsStream("20240622_ad_bucket_249.txt");
+            if (resourceStream != null) {
+                try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                    Map<String, double[]> bucketsMap = new HashMap<>();
+                    Map<String, Double> bucketsLen = new HashMap<>();
+                    String line;
+                    while ((line = reader.readLine()) != null) {
+                        // 替换空格和换行符，过滤空行
+                        line = line.replace(" ", "").replaceAll("\n", "");
+                        if (!line.isEmpty()) {
+                            String[] rList = line.split("\t");
+                            if (rList.length == 3) {
+                                String key = rList[0];
+                                double value1 = Double.parseDouble(rList[1]);
+                                bucketsLen.put(key, value1);
+                                double[] value2 = Arrays.stream(rList[2].split(","))
+                                        .mapToDouble(Double::valueOf)
+                                        .toArray();
+                                bucketsMap.put(key, value2);
+                            }
                         }
                     }
+                    this.bucketsMap = bucketsMap;
+                    this.bucketsLen = bucketsLen;
+                } catch (IOException e) {
+                    log.error("something is wrong in parse bucket file:" + e);
                 }
-                this.bucketsMap = bucketsMap;
-                this.bucketsLen = bucketsLen;
-            } catch (IOException e) {
-                log.error("something is wrong in parse bucket file:" + e);
+            } else {
+                log.error("no bucket file");
             }
-        } else {
-            log.error("no bucket file");
         }
-
     }
 
     private Map<String, String> featureBucket(Map<String, String> featureMap) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -92,6 +92,15 @@ public class RankServiceImpl implements RankService {
     private AdRankItem rankBy680(RankRecommendRequestParam request) {
         ScoreParam scoreParam = RequestConvert.requestConvert(request);
         List<AdRankItem> adRankItems = fmRankService.adItemRank(request, scoreParam);
+        List<JSONObject> collect = adRankItems.stream().map(i -> {
+            JSONObject json = new JSONObject();
+            json.put("cid", i.getAdId());
+            json.put("score", i.getScore());
+            json.put("metaFeatureMap", i.getMetaFeatureMap());
+            json.put("allFeatureMap", i.getFeatureMap());
+            return json;
+        }).collect(Collectors.toList());
+        log.info("LR模型打分结果: {}", JSON.toJSONString(collect));
         logHubService.scoreLogUpload(scoreParam, request.getAdIdList(), adRankItems, request, "LRModelScore", "680");
         return adRankItems.get(0);
     }
@@ -227,7 +236,7 @@ public class RankServiceImpl implements RankService {
             }
 
             for (AdRankItem adRankItem : rankResult) {
-                adRankItem.getExt().put("userAdFeature", JSON.toJSONString(userAdFeature));
+                // adRankItem.getItemFeature().put("userAdFeature", JSON.toJSONString(userAdFeature));
             }
             // 日志上报
             logHubService.scoreLogUpload(param, request.getAdIdList(), rankResult, request,