feat:添加定向打分参数

ad-engine-commons/pom.xml

@@ -1,7 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project
+        xmlns="http://maven.apache.org/POM/4.0.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>ad-engine</artifactId>
         <groupId>com.tzld.piaoquan</groupId>
@@ -25,7 +26,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.1.30</version>
+            <version>1.1.21</version>
         </dependency>
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
@@ -57,17 +58,29 @@
         </dependency>
 
 
-
-        <!--        <dependency>-->
-<!--            <groupId>net.devh</groupId>-->
-<!--            <artifactId>grpc-client-spring-boot-starter</artifactId>-->
-<!--            <version>2.9.0.RELEASE</version>-->
-<!--        </dependency>-->
-<!--        <dependency>-->
-<!--            <groupId>org.projectlombok</groupId>-->
-<!--            <artifactId>lombok</artifactId>-->
-<!--            <version>1.18.22</version>-->
-<!--        </dependency>-->
+        <dependency>
+            <groupId>org.xm</groupId>
+            <artifactId>similarity</artifactId>
+            <version>1.1</version>
+            <exclusions>
+<!--                <exclusion>-->
+<!--                    <artifactId>logback-access</artifactId>-->
+<!--                    <groupId>ch.qos.logback</groupId>-->
+<!--                </exclusion>-->
+<!--                <exclusion>-->
+<!--                    <artifactId>logback-classic</artifactId>-->
+<!--                    <groupId>ch.qos.logback</groupId>-->
+<!--                </exclusion>-->
+<!--                <exclusion>-->
+<!--                    <artifactId>logback-core</artifactId>-->
+<!--                    <groupId>ch.qos.logback</groupId>-->
+<!--                </exclusion>-->
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
 </project>

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/helper/NewExpHelper.java

@@ -0,0 +1,106 @@
+package com.tzld.piaoquan.ad.engine.commons.helper;
+
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
+import lombok.Data;
+import org.apache.commons.collections4.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+@Component
+public class NewExpHelper {
+    private final static Logger log = LoggerFactory.getLogger(NewExpHelper.class);
+    public static Map<String, Set<String>> appExpIdCache = new HashMap<>();
+    public static Map<String, Exp> expIdAndRangeCache = new HashMap<>();
+
+    public static String flagId;
+
+    @Value("${ad.new.exp.flag.id:0}")
+    private void setFlagId(String flagId) {
+        NewExpHelper.flagId = flagId;
+    }
+
+    @Value("${new.exp,config.v2:[]}")
+    private void setNewExpInfo(String str) {
+        List<ExpConfig> expConfigs = JSONUtils.fromJson(str, new TypeToken<List<ExpConfig>>() {
+        }, Collections.emptyList());
+        for (ExpConfig expConfig : expConfigs) {
+            String appId = expConfig.getAppType();
+            for (Layer layer : expConfig.layers) {
+                if (layer.layerId.equals("ad-server")) {
+                    Set<String> expIdSet = new HashSet<>();
+                    for (Exp exp : layer.getExps()) {
+                        expIdSet.add(exp.getExpId());
+                        expIdAndRangeCache.put(exp.getExpId(), exp);
+                    }
+                    appExpIdCache.put(appId, expIdSet);
+                }
+            }
+        }
+    }
+
+    public static boolean checkInNewExpGroup(String appId, int groupNumber, String expId) {
+        try {
+            if (appExpIdCache.get(appId) == null || !appExpIdCache.get(appId).contains(expId)) {
+                return false;
+            }
+            if (expIdAndRangeCache.get(expId).isAllEnter() && groupNumber < 0) {
+                return true;
+            }
+            return expIdAndRangeCache.get(expId).getRange()[0] <= groupNumber && expIdAndRangeCache.get(expId).getRange()[1] >= groupNumber;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    public static boolean checkInNewExpGroup(String appId, int groupNumber, Collection<String> expIds) {
+        if (CollectionUtils.isEmpty(expIds)) {
+            return false;
+        }
+        for (String expId : expIds) {
+            try {
+                if (appExpIdCache.get(appId) == null || !appExpIdCache.get(appId).contains(expId)) {
+                    continue;
+                }
+                if (expIdAndRangeCache.get(expId).isAllEnter() && groupNumber < 0) {
+                    return true;
+                }
+                if (expIdAndRangeCache.get(expId).getRange()[0] <= groupNumber && expIdAndRangeCache.get(expId).getRange()[1] >= groupNumber) {
+                    return true;
+                }
+            } catch (Exception e) {
+                log.warn("svc=checkInNewExpGroup appId={} groupNumber={} expId={}", appId, groupNumber, expId);
+            }
+        }
+        return false;
+    }
+
+    @Data
+    public static class ExpConfig {
+        private String appType;
+        private List<Layer> layers;
+
+    }
+
+    @Data
+    public static class Layer {
+        private String layerId;
+        private int bucketNum;
+        private List<Exp> exps;
+        private Map<String, String> groupRule;
+
+    }
+
+    @Data
+    public static class Exp {
+        private String expId;
+        private int[] range;
+        private boolean allEnter = false;
+        private Map<String, Object> param;
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/BaseFMModelScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+import com.tzld.piaoquan.ad.engine.commons.score.model.FMModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseFMModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseFMModelScorer.class);
+
+    public BaseFMModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(FMModel.class);
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScoreParam.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.ad.engine.commons.score;
 
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import lombok.Data;
+
 import java.util.*;
 
 /**
@@ -18,8 +19,7 @@ public class ScoreParam {
     private String province;
     private Integer newExpGroup;
     private String pqtId;
-    private Map<String,Object> extraParam=new HashMap<>();
-
+    private Map<String, Object> extraParam = new HashMap<>();
 
 }
 

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/FMModel.java

@@ -0,0 +1,144 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class FMModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(FMModel.class);
+    private Map<String, List<Float>> model;
+
+    public void putFeature(Map<String, List<Float>> model, String[] items) {
+
+        String featureKey = items[0];
+        List<Float> weights = new ArrayList<>();
+        for (int i = 1; i < items.length; i++) {
+            weights.add(Float.valueOf(items[i]));
+        }
+
+        model.put(featureKey, weights);
+    }
+
+    public float getWeight(Map<String, List<Float>> model, String featureKey, int index) {
+        if (!model.containsKey(featureKey)) {
+            return 0.0f;
+        }
+
+        return model.get(featureKey).get(index);
+
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.model == null)
+            return 0;
+        return model.size();
+    }
+
+    public void cleanModel() {
+        this.model = null;
+    }
+
+    public Float score(Map<String, String> featureMap) {
+        float sum = 0.0f;
+
+        if (MapUtils.isNotEmpty(featureMap)) {
+            // 计算 sum w*x
+            float sum0 = 0.0f;
+            for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                float x = NumberUtils.toFloat(e.getValue(), 0.0f);
+                float w = getWeight(this.model, e.getKey(), 0);
+                sum0 += w * x;
+            }
+            sum += sum0;
+
+            // 计算 sum v*v*x*X
+            float sum1 = 0.0f;
+            for (int i = 1; i < 9; i++) {
+                float sum10 = 0.0f;
+                float sum11 = 0.0f;
+                for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                    float x = NumberUtils.toFloat(e.getValue(), 0.0f);
+                    float v = getWeight(this.model, e.getKey(), i);
+                    float d = v * x;
+                    sum10 += d;
+                    sum11 += d * d;
+                }
+                sum1 += sum10 * sum10 - sum11;
+            }
+            sum1 = 0.5f * sum1;
+            float biasW = model.get("bias").get(0);
+            sum = biasW + sum0 + sum1;
+        }
+
+        return (float) (1.0f / (1 + Math.exp(-sum)));
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        Map<String, List<Float>> model = new HashMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", model.size(), curTime);
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 9) {
+                if (items[0].equals("bias")) {
+                    putFeature(model, items);
+                }
+                continue;
+            }
+
+            putFeature(model, items);
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.model = model;
+
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", model.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 9) {
+                continue;
+            }
+            putFeature(model, items);
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", model.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/AbUtil.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.ad.engine.commons.util;
+
+import com.tzld.piaoquan.ad.engine.commons.helper.NewExpHelper;
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class AbUtil {
+
+    public static Set<String> unfoldAllExpCode(List<Map<String, String>> adAbMap) {
+        if (CollectionUtils.isEmpty(adAbMap)) {
+            return new HashSet<>();
+        }
+        return adAbMap.stream()
+                .map(m -> m.get("abExpCode"))
+                .filter(Objects::nonNull)
+                .map(Object::toString)
+                .collect(Collectors.toSet());
+    }
+
+    /**
+     * 判断用户是否在某个实验中，包含在新实验系统和旧实验系统
+     */
+    public static boolean isInAbExp(Collection<String> expCodeSet, Object appType, int newExpGroup, String expCode) {
+        if (CollectionUtils.isEmpty(expCodeSet)) {
+            return false;
+        }
+        if (expCodeSet.contains(expCode)) {
+            return true;
+        }
+
+        if (expCodeSet.contains(NewExpHelper.flagId)) {
+            return NewExpHelper.checkInNewExpGroup(appType.toString(), newExpGroup, expCode);
+        }
+        return false;
+   }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/ExtractorUtils.java

@@ -0,0 +1,200 @@
+package com.tzld.piaoquan.ad.engine.commons.util;
+
+import org.xm.Similarity;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ExtractorUtils {
+
+    public static double sigmoid(double x) {
+        return 1.0 / (1.0 + Math.exp(-x));
+    }
+    public static int findInsertPosition(double[] sortedArray, double target) {
+        int low = 0;
+        int high = sortedArray.length - 1;
+
+        while (low <= high) {
+            int mid = low + (high - low) / 2;
+            double midValue = sortedArray[mid];
+
+            if (midValue < target) {
+                low = mid + 1;
+            } else if (midValue > target) {
+                high = mid - 1;
+            } else {
+                // 找到相等的值，尝试在右侧寻找插入点
+                while (mid < sortedArray.length - 1 && sortedArray[mid + 1] == target) {
+                    mid++;
+                }
+                return mid + 1; // 返回当前mid的下一个位置作为插入点
+            }
+        }
+
+        return low; // 返回low作为插入点
+    }
+    public static Double[] funcC34567ForTags(String tags, String title) {
+        String[] tagsList = tags.split(",");
+        int d1 = 0;
+        List<String> d2 = new ArrayList<>();
+        double d3 = 0.0;
+        double d4 = 0.0;
+
+        for (String tag : tagsList) {
+            if (title.contains(tag)) {
+                d1++;
+                d2.add(tag);
+            }
+            double score = Similarity.conceptSimilarity(tag, title);
+            if (score > d3) {
+                d3 = score;
+            }
+            d4 += score;
+        }
+
+        d4 = (tagsList.length > 0) ? d4 / tagsList.length : d4;
+
+        // 使用数组来返回多个值
+        return new Double[]{(double) d1, d3, d4};
+    }
+    public static Double calDiv(double a, double b){
+        if (a == 0 || b == 0){
+            return 0D;
+        }
+        return a / b;
+    }
+    public static Double calLog(double a){
+        if (a <= 0){
+            return 0D;
+        }
+        return Math.log(a + 1.0);
+    }
+    public static Double division(String s1, String s2, Map<String, String> maps){
+        double rate = 0.0;
+        if (maps.containsKey(s1) && maps.containsKey(s2)){
+            Double d1 = Double.valueOf(maps.get(s1));
+            if (isDoubleEqualToZero(d1)){
+                return rate;
+            }
+            Double d2 = Double.valueOf(maps.get(s2));
+            rate = d2 / d1;
+        }
+        return rate;
+    }
+    public static Double divisionDouble(Double d1, Double d2){
+        double rate = 0.0;
+        if (isDoubleEqualToZero(d1)){
+            return rate;
+        }
+        rate = d2 / d1;
+        return rate;
+    }
+
+    public static boolean isDoubleEqualToZero(double value) {
+        final double epsilon = 1e-10; // 定义一个很小的误差范围
+        // 判断value是否在误差范围内
+        return Math.abs(value) < epsilon;
+    }
+
+
+
+    public static double calculateVariance(List<Double> numbers) {
+        double average = numbers.stream()
+                .mapToDouble(Double::doubleValue)
+                .average()
+                .orElse(0.0);
+
+        double squaredDiffSum = numbers.stream()
+                .mapToDouble(Double::doubleValue)
+                .map(x -> Math.pow(x - average, 2))
+                .average()
+                .orElse(0.0);
+
+        return squaredDiffSum;
+    }
+
+    public static double calculateAverage(List<Double> numbers) {
+        if (numbers == null || numbers.isEmpty()) {
+            return 0.0;
+        }
+        return numbers.stream()
+                .mapToDouble(Number::doubleValue)
+                .average()
+                .orElse(0.0);
+    }
+
+    public static List<Double> calculateDifferences(List<Double> numbers) {
+        List<Double> differences = new ArrayList<>();
+
+        for (int i = 0; i < numbers.size() - 1; i++) {
+            Double diff = 0.0;
+            if (!isDoubleEqualToZero(numbers.get(i))){
+                diff = (numbers.get(i + 1) - numbers.get(i)) / numbers.get(i);
+            }
+            differences.add(diff);
+        }
+
+        return differences;
+    }
+
+    public static List<String> generateHourStrings(String timeString, int N) {
+        LocalDateTime dateTime = LocalDateTime.parse(timeString, DateTimeFormatter.ofPattern("yyyyMMddHH"));
+        List<String> hourStrings = new ArrayList<>();
+        for (int i = 0; i < N; i++) {
+            hourStrings.add(dateTime.minusHours(i).format(DateTimeFormatter.ofPattern("yyyyMMddHH")));
+        }
+
+        return hourStrings;
+    }
+
+    public static String subtractHours(String inputDateTime, int hoursToSubtract) {
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHH");
+        LocalDateTime dateTime = LocalDateTime.parse(inputDateTime, formatter);
+        LocalDateTime subtractedDateTime = dateTime.minusHours(hoursToSubtract);
+        return subtractedDateTime.format(formatter);
+    }
+
+    // 针对0-1的数字，进行分桶。
+    public static Integer ceilLogRate(Double key) {
+        double bucket = Math.ceil(
+                Math.pow(key, 0.2) * 100
+        );
+        if (bucket > 300) {
+            bucket = 300;
+        }
+        if (bucket < 0) {
+            bucket = 0;
+        }
+        return (int)bucket;
+    }
+
+    // 针对大于1的数字，进行分桶。
+    public static int bucketCnt(Double key) {
+        long bucket = Math.round(Math.log((key * 10 + 1.0)) * 10);
+        if (bucket > 300) {
+            bucket = 300;
+        }
+        if (bucket < 0) {
+            bucket = 0;
+        }
+        return (int)bucket;
+    }
+
+    public static void main(String[] args) {
+//        System.out.println(ceilLogRate(0.0002));
+//        System.out.println(ceilLogRate(0.01));
+//        System.out.println(ceilLogRate(0.2));
+//        System.out.println(ceilLogRate(4.));
+//        System.out.println(bucketCnt(1.));
+//        System.out.println(bucketCnt(20.));
+//        System.out.println(bucketCnt(500.));
+//        System.out.println(bucketCnt(50000.));
+
+        System.out.println(generateHourStrings("2024011603", 5));
+
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/NumUtil.java

@@ -0,0 +1,12 @@
+package com.tzld.piaoquan.ad.engine.commons.util;
+
+public class NumUtil {
+
+    public static double div(double d1, double d2) {
+        if (d1 == 0 || d2 == 0) {
+            return 0d;
+        }
+        return d1 / d2;
+    }
+
+}

ad-engine-server/src/main/java/com/tzld/piaoquan/ad/engine/server/controller/AdRecommendController.java

@@ -29,10 +29,10 @@ public class AdRecommendController {
     @RequestMapping("/top1/basic")
     public Map<String, Object> adRecommendTop1Basic(@RequestBody RankRecommendRequestParam request) {
         AdRankItem rankResult = rankService.adItemRank(request);
-        HashMap map = new HashMap();
-        map.put("code", "0");
-        map.put("msg", "success");
-        HashMap contentMap = new HashMap<>();
+        Map<String, Object> map =new HashMap<>();
+        map.put("code","0");
+        map.put("msg","success");
+        Map<String, Object> contentMap=new HashMap<>();
         contentMap.put("adId", rankResult.getAdId());
         contentMap.put("adScore", rankResult.getScore());
         map.put("content", contentMap);

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/Feature.java

@@ -0,0 +1,22 @@
+package com.tzld.piaoquan.ad.engine.service.feature;
+
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Data
+public class Feature {
+    // k1:创意、k2:表、k3:特征、v:特征值
+    private Map<String, Map<String, Map<String, String>>> cidFeature = new HashMap<>();
+
+    // k1:视频、k2:表、v:特征值
+    private Map<String, Map<String, String>> videoFeature = new HashMap<>();
+
+    // k1: 广告主ID、k2:表、k3:特征、v:特征值
+    private Map<String, Map<String, Map<String, String>>> adVerFeature = new HashMap<>();
+
+    // k1:mid、k2:表、v:特征值
+    private Map<String, Map<String, String>> userFeature = new HashMap<>();
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/FeatureService.java

@@ -0,0 +1,205 @@
+package com.tzld.piaoquan.ad.engine.service.feature;
+
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
+import com.tzld.piaoquan.ad.engine.service.remote.FeatureV2RemoteService;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
+import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+@Slf4j
+@Component
+public class FeatureService {
+
+    @Autowired
+    private FeatureV2RemoteService remoteService;
+
+    private static final String cidUkFormat = "c:%s:%s";
+    private static final String cidUkDoubleFieldFormat = "c:%s:%s:%s";
+    private static final String avUkFormat = "av:%s:%s";
+    private static final String vidUkFormat = "v:%s:%s";
+    private static final String midUkFormat = "m:%s:%s";
+
+    public Feature getFeature(Collection<String> cidList, Collection<String> adVerIdList, ScoreParam param) {
+        AdRequestContext context = param.getRequestContext();
+
+        List<FeatureKeyProto> protos = new ArrayList<>();
+        for (String cidStr : cidList) {
+
+            // cid
+            protos.add(genWithCid("alg_cid_feature_basic_info", cidStr));
+            protos.add(genWithCid("alg_cid_feature_cid_action", cidStr));
+
+            // cid + region
+            protos.add(genWithCidAndRegion("alg_cid_feature_region_action", cidStr, context.getRegion()));
+
+            // cid + apptype
+            protos.add(genWithCidAndAppType("alg_cid_feature_app_action", cidStr, context.getApptype()));
+
+            // cid + week
+            protos.add(genWithCidAndWeek("alg_cid_feature_week_action", cidStr, context.getWeek()));
+
+            // cid + hour
+            protos.add(genWithCidAndHour("alg_cid_feature_hour_action", cidStr, context.getHour()));
+
+            // cid + brand
+            protos.add(genWithCidAndBrand("alg_cid_feature_brand_action", cidStr, context.getMachineinfoBrand()));
+
+            // cid + wechatversion
+            protos.add(genWithCidAndWechatVersion("alg_cid_feature_weChatVersion_action", cidStr, context.getMachineinfoWechatversion()));
+
+            // cid + vid
+            protos.add(genWithCidAndVid("alg_cid_feature_vid_cf", cidStr, param.getVideoId().toString()));
+        }
+
+        for (String adVerId : adVerIdList) {
+            // adverid
+            protos.add(genWithAdVerId("alg_cid_feature_adver_action", adVerId));
+        }
+
+        // vid
+        protos.add(genWithVid("alg_cid_feature_vid_cf_rank", param.getVideoId().toString()));
+
+        // mid
+        protos.add(genWithMid("alg_mid_feature_ad_action", param.getMid()));
+        protos.add(genWithMid("alg_mid_feature_return_tags", param.getMid()));
+        protos.add(genWithMid("alg_mid_feature_share_tags", param.getMid()));
+
+        Map<String, String> featureMap = remoteService.getFeature(protos);
+        Feature feature = new Feature();
+
+        for (Map.Entry<String, String> entry : featureMap.entrySet()) {
+            String key = entry.getKey();
+            String value = entry.getValue();
+            String[] split = key.split(":");
+            String prefix = split[0];
+            String tableName = split[1];
+            String id = split[2];
+            switch (prefix) {
+                case "c":
+                    Map<String, Map<String, String>> tableFeatureMap = feature.getCidFeature().getOrDefault(id, new HashMap<>());
+                    tableFeatureMap.put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    feature.getCidFeature().put(id, tableFeatureMap);
+                    break;
+                case "av":
+                    Map<String, Map<String, String>> avFeatureMap = feature.getAdVerFeature().getOrDefault(id, new HashMap<>());
+                    avFeatureMap.put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    feature.getAdVerFeature().put(tableName, avFeatureMap);
+                    break;
+                case "m":
+                    feature.getUserFeature().put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    break;
+                case "v":
+                    feature.getVideoFeature().put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    break;
+            }
+        }
+
+        return feature;
+    }
+
+    private FeatureKeyProto genWithCid(String table, String cid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkFormat, table, cid))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .build();
+    }
+
+    private FeatureKeyProto genWithAdVerId(String table, String adVerId) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(avUkFormat, table, adVerId))
+                .setTableName(table)
+                .putFieldValue("adverid", adVerId)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndRegion(String table, String cid, String region) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, region))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("region", region)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndAppType(String table, String cid, String appType) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, appType))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("apptype", appType)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndWeek(String table, String cid, String week) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, week))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("week", week)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndHour(String table, String cid, String hour) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, hour))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("hour", hour)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndBrand(String table, String cid, String brand) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, brand))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("brand", brand)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndWechatVersion(String table, String cid, String wechatVersion) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, wechatVersion))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("wechatversion", wechatVersion)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndVid(String table, String cid, String vid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, vid))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("vid", vid)
+                .build();
+    }
+
+    private FeatureKeyProto genWithVid(String table, String vid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(vidUkFormat, table, vid))
+                .setTableName(table)
+                .putFieldValue("vid", vid)
+                .build();
+    }
+
+    private FeatureKeyProto genWithMid(String table, String mid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(midUkFormat, table, mid))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .build();
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/impl/LogHubServiceImpl.java

@@ -40,10 +40,10 @@ public class LogHubServiceImpl implements LogHubService {
             @Override
             public void run() {
                 JSONObject logMap = new JSONObject();
-                logMap.put("pqtId", param.getPqtId());
+                logMap.put("pqtid", param.getPqtId());
                 logMap.put("mid", param.getMid());
-                logMap.put("videoId", param.getVideoId());
-                logMap.put("abCode", abCode);
+                logMap.put("vid", param.getVideoId());
+                logMap.put("expid", abCode);
                 // 获取AB实验列表
                 Set<String> abExpCode = new HashSet<>();
                 if (CollectionUtils.isNotEmpty(requestParam.getAdAbExpArr())) {
@@ -53,41 +53,42 @@ public class LogHubServiceImpl implements LogHubService {
                         }
                     }
                 }
-                logMap.put("abExpCode", abExpCode);
+                logMap.put("expids", abExpCode);
 
                 List<JSONObject> scoreResult = new ArrayList<>();
                 for (AdRankItem rankItem : rankItems) {
                     JSONObject json = new JSONObject();
-                    json.put("adId", rankItem.getAdId());
                     json.put("cid", rankItem.getAdId());
                     json.put("score", rankItem.getScore());
-                    json.put("ext", rankItem.getExt());
-                    json.put("weight", rankItem.getWeight());
+                    rankItem.getFeature().put("weight", rankItem.getWeight());
+                    json.put("feature", rankItem.getFeature());
                     scoreResult.add(json);
                 }
                 logMap.put("scoreResult", JSON.toJSONString(scoreResult));
 
                 AdRankItem top1 = rankItems.get(0);
-                logMap.put("top1_adId", top1.getAdId());
-                logMap.put("top1_cid", top1.getAdId());
-                logMap.put("top1_score", top1.getScore());
-                logMap.put("top1_ext", JSON.toJSONString(top1.getExt()));
-                logMap.put("top1_weight", top1.getWeight());
+                logMap.put("cid", top1.getAdId());
+                logMap.put("score", top1.getScore());
+                JSONObject feature = new JSONObject(top1.getFeature());
+                feature.put("weight", top1.getWeight());
+                logMap.put("feature", feature.toJSONString());
 
-                logMap.put("creativeList", JSON.toJSONString(adIdList));
-                logMap.put("adAbGroup", requestParam.getAdAbGroup());
-                logMap.put("scoreStrategy", scoreStrategy);
-                logMap.put("appType", requestParam.getAppType());
+                logMap.put("abcode", requestParam.getAdAbGroup());
+                logMap.put("scorestrategy", scoreStrategy);
+                logMap.put("apptype", requestParam.getAppType());
 
-                if (Objects.nonNull(requestParam.getStatisticsLog())) {
-                    logMap.put("earlyAdIds", requestParam.getStatisticsLog().getEarlyAdIds());
-                    logMap.put("earlyCidList", requestParam.getStatisticsLog().getEarlyCreativeIds());
-                    logMap.put("finalCidList", requestParam.getStatisticsLog().getFinalCreativeIds());
-                    logMap.put("commonFilterAfterAdIds", requestParam.getStatisticsLog().getCommonFilterAfterAdIds());
-                    logMap.put("commonFilterAfterCidList", requestParam.getStatisticsLog().getCommonFilterAfterCreativeIds());
-                    logMap.put("tacticsFilterAfterAdIds", requestParam.getStatisticsLog().getTacticsFilterAfterAdIds());
-                    logMap.put("tacticsFilterAfterCidList", requestParam.getStatisticsLog().getTacticsFilterAfterCreativeIds());
-                }
+                // logMap.put("creativeList", JSON.toJSONString(adIdList));
+                // if (Objects.nonNull(requestParam.getStatisticsLog())) {
+                //     JSONObject extInfo = new JSONObject();
+                //     extInfo.put("earlyAdIds", requestParam.getStatisticsLog().getEarlyAdIds());
+                //     extInfo.put("earlyCidList", requestParam.getStatisticsLog().getEarlyCreativeIds());
+                //     extInfo.put("finalCidList", requestParam.getStatisticsLog().getFinalCreativeIds());
+                //     extInfo.put("commonFilterAfterAdIds", requestParam.getStatisticsLog().getCommonFilterAfterAdIds());
+                //     extInfo.put("commonFilterAfterCidList", requestParam.getStatisticsLog().getCommonFilterAfterCreativeIds());
+                //     extInfo.put("tacticsFilterAfterAdIds", requestParam.getStatisticsLog().getTacticsFilterAfterAdIds());
+                //     extInfo.put("tacticsFilterAfterCidList", requestParam.getStatisticsLog().getTacticsFilterAfterCreativeIds());
+                //     logMap.put("extinfo", extInfo);
+                // }
 
                 aliyunLogManager.sendLog(project, logStore, "", logMap);
             }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/remote/FeatureV2RemoteService.java

@@ -0,0 +1,32 @@
+package com.tzld.piaoquan.ad.engine.service.remote;
+
+import com.tzld.piaoquan.recommend.feature.client.FeatureV2Client;
+import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * @author dyp
+ */
+@Component
+@Slf4j
+public class FeatureV2RemoteService {
+
+    @Autowired
+    private FeatureV2Client client;
+
+    public Map<String, String> getFeature(List<FeatureKeyProto> protos) {
+        if (CollectionUtils.isEmpty(protos)) {
+            return Collections.emptyMap();
+        }
+        return client.multiGetFeature(protos);
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService.java

@@ -13,12 +13,24 @@ import java.util.List;
 public interface RankService {
 
     AdRankItem adItemRank(RankRecommendRequestParam request);
-    AdRankItem adItemRankWithVideoAdThompson(RankRecommendRequestParam request);
-    AdPlatformCreativeDTO adBidRank(BidRankRecommendRequestParam request);
 
-    AdPlatformCreativeDTO adBidRankNewPid(BidRankRecommendRequestParam request);
+    default AdRankItem adItemRankWithVideoAdThompson(RankRecommendRequestParam request) {
+        return null;
+    }
 
-    List<AdRankItem> rank (ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems, String configFile);
+    default AdPlatformCreativeDTO adBidRank(BidRankRecommendRequestParam request) {
+        return null;
+    }
 
-    AdRankItem unionAdItemRank(UnionRankRecommendRequestParam request);
+    default AdPlatformCreativeDTO adBidRankNewPid(BidRankRecommendRequestParam request) {
+        return null;
+    }
+
+    default List<AdRankItem> rank(ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems, String configFile) {
+        return null;
+    }
+
+    default AdRankItem unionAdItemRank(UnionRankRecommendRequestParam request) {
+        return null;
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VideoAdThompsonScorerV2.java

@@ -43,52 +43,6 @@ public class VideoAdThompsonScorerV2 {
     Random random = new Random();
     Gson gson = new Gson();
 
-    // public List<AdRankItem> thompsonScorerByExp663(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
-    //     List<AdRankItem> result = new LinkedList<>();
-    //     String jsonStr;
-    //     CreativeStatistic statistic = null;
-    //     List<String> redisKey = new LinkedList<>();
-    //     adIdList.forEach(creativeDTO -> {
-    //         redisKey.add(redisCreativeStatisticsPrefix + creativeDTO.getCreativeId());
-    //     });
-    //     List<String> values = redisHelper.getValues(redisKey);
-    //     double score = 0d;
-    //     int i = 0;
-    //     for (AdPlatformCreativeDTO dto : adIdList) {
-    //         AdRankItem item = new AdRankItem();
-    //         item.setVideoId(param.getVideoId());
-    //         item.setCpa(dto.getCpa());
-    //         item.setAdId(dto.getCreativeId());
-    //         try {
-    //             jsonStr = values.get(i);
-    //             if (jsonStr == null) {
-    //                 score = exp663Param.getOrDefault("randomMin", 0.000001)
-    //                         + (random.nextDouble() *
-    //                         (exp663Param.getOrDefault("randomMax", 0.00001) - exp663Param.getOrDefault("randomMin", 0.000001)));
-    //             } else {
-    //                 statistic = gson.fromJson(jsonStr, CreativeStatistic.class);
-    //                 score = (exp663Param.getOrDefault("alpha", 1d) + Long.parseLong(statistic.getOrder()))
-    //                         / (exp663Param.getOrDefault("beta", 10000d) + Long.parseLong(statistic.getExp()));
-    //             }
-    //             double s1 = score;
-    //             score = score * dto.getCpa() * dto.getBid1() * dto.getBid2();
-    //
-    //             Map<String, Object> ext = this.extMap(statistic, "663", dto.getCpa(), null,
-    //                     exp663Param.getOrDefault("alpha", 1d), exp663Param.getOrDefault("beta", 10000d), null);
-    //             ext.put("s1", s1);
-    //             item.setExt(ext);
-    //             item.setScore(score);
-    //             item.setScore_type(663);
-    //         } catch (Exception e) {
-    //             log.error("svc=thompsonScorerByExp663 {}", gson.toJson(e.getStackTrace()));
-    //         }
-    //         result.add(item);
-    //         i++;
-    //     }
-    //     Collections.sort(result);
-    //     return result;
-    // }
-
     public List<AdRankItem> thompsonScorerByExp663(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
         List<AdRankItem> result = new LinkedList<>();
         Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(redisCreativeStatisticsPrefix, adIdList);
@@ -135,7 +89,7 @@ public class VideoAdThompsonScorerV2 {
             item.setCpa(cpa);
             item.setAdId(dto.getCreativeId());
             item.setScore(score);
-            item.setExt(ext);
+            item.setFeature(ext);
             item.setVideoId(param.getVideoId());
             item.setScore_type(663);
             item.setWeight(dto.getWeight());
@@ -184,7 +138,7 @@ public class VideoAdThompsonScorerV2 {
                 ext.put("s1", s1);
                 item.setCreativeCode(dto.getCreativeCode());
                 item.setWeight(dto.getWeight());
-                item.setExt(ext);
+                item.setFeature(ext);
                 item.setScore_type(664);
             } catch (Exception e) {
                 log.error("svc=thompsonScorerByExp664 {}", gson.toJson(e.getStackTrace()));
@@ -233,7 +187,7 @@ public class VideoAdThompsonScorerV2 {
                         exp665Param.getOrDefault("alpha", 1d), exp665Param.getOrDefault("beta", 10000d), null);
                 ext.put("s1", s1);
                 item.setCreativeCode(dto.getCreativeCode());
-                item.setExt(ext);
+                item.setFeature(ext);
             } catch (Exception e) {
                 log.error("svc=thompsonScorerByExp665 {}", gson.toJson(e.getStackTrace()));
             }
@@ -244,75 +198,6 @@ public class VideoAdThompsonScorerV2 {
         Collections.sort(result);
         return result;
     }
-    // public List<AdRankItem> thompsonScorerByExp666(ScoreParam param,List<AdPlatformCreativeDTO> adIdList){
-    //     List<AdRankItem> result=new LinkedList<>();
-    //     String jsonStr;
-    //     CreativeStatistic statistic = null;
-    //
-    //     List<String> redisKey=new LinkedList<>();
-    //     adIdList.forEach(creativeDTO -> {
-    //         redisKey.add(redisCreativeStatisticsPrefix+creativeDTO.getCreativeId());
-    //     });
-    //     List<String> values=redisHelper.getValues(redisKey);
-    //     List<String> combineKeys=new LinkedList<>();
-    //     adIdList.forEach(creativeDTO -> {
-    //         combineKeys.add(redisVideoCreativeStatisticsPrefix+param.getVideoId()+"_"+creativeDTO.getCreativeId());
-    //     });
-    //     List<String> combineValues=redisHelper.getValues(combineKeys);
-    //     int i=0;
-    //     double score=0d;
-    //     for(AdPlatformCreativeDTO dto:adIdList){
-    //         String statisticType = "vid+cid";
-    //         AdRankItem item=new AdRankItem();
-    //         item.setVideoId(param.getVideoId());
-    //         item.setCpa(dto.getCpa());
-    //         item.setAdId(dto.getCreativeId());
-    //         try {
-    //             jsonStr=combineValues.get(i);
-    //             if(jsonStr==null){
-    //                 jsonStr=values.get(i);
-    //                 if(jsonStr==null){
-    //                     statisticType = "";
-    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d),exp666Param.getOrDefault("beta",100000d));
-    //                 }else {
-    //                     statisticType = "cid";
-    //                     statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
-    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
-    //                             (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
-    //                 }
-    //             }else {
-    //                 statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
-    //                 if(Double.parseDouble(statistic.getExp())>exp666Param.getOrDefault("viewThreshold",5000d)){
-    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
-    //                             (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
-    //                 } else if( values.get(i)!=null) {
-    //                     statistic =gson.fromJson(values.get(i),CreativeStatistic.class);
-    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
-    //                             (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
-    //                 }else {
-    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d),exp666Param.getOrDefault("beta",100000d));
-    //                 }
-    //             }
-    //             double s1 = score;
-    //             score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
-    //             item.setScore(score);
-    //             item.setScore_type(666);
-    //
-    //             Map<String, Object> ext = this.extMap(statistic, "666", dto.getCpa(), exp666Param.getOrDefault("viewThreshold",5000d), exp663Param.getOrDefault("alpha", 1d),
-    //                     exp663Param.getOrDefault("beta", 10000d), exp666Param.getOrDefault("beta_k",9d));
-    //             ext.put("s1", s1);
-    //             ext.put("group_field", statisticType);
-    //             item.setExt(ext);
-    //
-    //         }catch (Exception e){
-    //             log.error("svc=thompsonScorerByExp666 {}",gson.toJson(e.getStackTrace()));
-    //         }
-    //         result.add(item);
-    //         i++;
-    //     }
-    //     Collections.sort(result);
-    //     return result;
-    // }
 
     public List<AdRankItem> thompsonScorerByExp666(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
         List<AdRankItem> result = new LinkedList<>();
@@ -370,7 +255,7 @@ public class VideoAdThompsonScorerV2 {
             item.setCpa(cpa);
             item.setAdId(dto.getCreativeId());
             item.setScore(score);
-            item.setExt(ext);
+            item.setFeature(ext);
             item.setVideoId(param.getVideoId());
             item.setScore_type(666);
             item.setWeight(dto.getWeight());
@@ -465,7 +350,7 @@ public class VideoAdThompsonScorerV2 {
             item.setAdId(dto.getCreativeId());
             item.setWeight(dto.getWeight());
             item.setScore(score);
-            item.setExt(ext);
+            item.setFeature(ext);
             item.setBid1(dto.getBid1());
             item.setBid2(dto.getBid2());
             item.setCreativeCode(dto.getCreativeCode());
@@ -528,7 +413,7 @@ public class VideoAdThompsonScorerV2 {
             item.setAdId(dto.getCreativeId());
             item.setWeight(dto.getWeight());
             item.setScore(score);
-            item.setExt(ext);
+            item.setFeature(ext);
             item.setBid1(dto.getBid1());
             item.setBid2(dto.getBid2());
             item.setCreativeCode(dto.getCreativeCode());
@@ -682,7 +567,7 @@ public class VideoAdThompsonScorerV2 {
             item.setCpa(cpa);
             item.setAdId(dto.getCreativeId());
             item.setScore(score);
-            item.setExt(ext);
+            item.setFeature(ext);
             item.setVideoId(param.getVideoId());
             item.setScore_type(scoreType);
             item.setWeight(dto.getWeight());

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogRovFMScorer.java

@@ -0,0 +1,160 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+
+import com.tzld.piaoquan.ad.engine.commons.score.BaseFMModelScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.FMModel;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+
+public class VlogRovFMScorer extends BaseFMModelScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogRovFMScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+
+    public VlogRovFMScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userAdFeature,
+                                    final List<AdRankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    public List<AdRankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                    final Map<String, String> userFeatureMap,
+                                    final List<AdRankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        FMModel model = (FMModel) this.getModel();
+
+        List<AdRankItem> result = rankByJava(sceneFeatureMap, userFeatureMap, rankItems);
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<AdRankItem> items) {
+        long startTime = System.currentTimeMillis();
+        FMModel model = (FMModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final FMModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex), ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        // 等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMap.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public double calcScore(final FMModel model,
+                            final AdRankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+
+
+        // Map<String, String> featureMap = new HashMap<>();
+        // if (MapUtils.isNotEmpty(item.getFeatureMap())) {
+        //     featureMap.putAll(item.getFeatureMap());
+        // }
+        // if (MapUtils.isNotEmpty(userFeatureMap)) {
+        //     featureMap.putAll(userFeatureMap);
+        // }
+        // if (MapUtils.isNotEmpty(sceneFeatureMap)) {
+        //     featureMap.putAll(sceneFeatureMap);
+        // }
+
+        double pro = 0.0;
+        // if (MapUtils.isNotEmpty(featureMap)) {
+        //     try {
+        //         pro = model.score(featureMap);
+        //         // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+        //     } catch (Exception e) {
+        //         LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+        //     }
+        // }
+        // item.setScoreRov(pro);
+        // item.getScoresMap().put("RovFMScore", pro);
+        // item.setAllFeatureMap(featureMap);
+        return pro;
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/convert/RequestConvert.java

@@ -17,10 +17,14 @@ public class RequestConvert {
         context.setMachineinfoModel(request.getMachineInfo().getModel());
         context.setMachineinfoSdkversion(request.getMachineInfo().getSdkVersion());
         context.setMachineinfoWchatversion(request.getMachineInfo().getWeChatVersion());
+
         LocalDateTime date=LocalDateTime.now();
-        context.setHour(date.getHour()+"");
+        context.setHour(String.valueOf(date.getHour()));
         context.setDay(date.format(DateTimeFormatter.ofPattern("yyyyMMdd")));
-        context.setWeek(date.getDayOfWeek().getValue()+"");
+        context.setWeek(String.valueOf(date.getDayOfWeek().getValue()));
+        context.setRegion(request.getRegion().replace("省", ""));
+        context.setCity(request.getCity().replace("市", ""));
+
         ScoreParam scoreParam=new ScoreParam();
         scoreParam.setRequestContext(context);
         scoreParam.setVideoId(request.getVideoId());

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/dto/AdPlatformCreativeDTO.java

@@ -14,6 +14,8 @@ public class AdPlatformCreativeDTO {
 
     private String creativeCode;
 
+    private String adVerId;
+
     private Integer bidType;
 
     private Double pctr;
@@ -32,6 +34,11 @@ public class AdPlatformCreativeDTO {
 
     private double weight;
 
+    /**
+     * 定向打分参数
+     */
+    private HumanDto humanScore;
+
     public static void main(String[] args) {
         System.out.println(JSON.toJSONString(AdPlatformCreativeDTO.builder()
                 .creativeId(3366L)

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/dto/HumanDto.java

@@ -0,0 +1,18 @@
+package com.tzld.piaoquan.ad.engine.service.score.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Map;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class HumanDto {
+
+    private Double exponent;
+
+    private Map<String, Double> detail;
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankService680.java

@@ -0,0 +1,411 @@
+package com.tzld.piaoquan.ad.engine.service.score.impl;
+
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.util.ExtractorUtils;
+import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
+import com.tzld.piaoquan.ad.engine.service.feature.Feature;
+import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
+import com.tzld.piaoquan.ad.engine.service.score.RankService;
+import com.tzld.piaoquan.ad.engine.service.score.convert.RequestConvert;
+import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
+import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Service
+public class RankService680 implements RankService {
+
+    @Autowired
+    private FeatureService featureService;
+
+    @Override
+    public AdRankItem adItemRank(RankRecommendRequestParam request) {
+
+        long ts = LocalDateTime.now().toEpochSecond(ZoneOffset.of("+8")) / 1000;
+
+        ScoreParam scoreParam = RequestConvert.requestConvert(request);
+        Feature feature = this.getFeature(scoreParam, request);
+        Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+        Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
+        Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
+        Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+
+        Map<String, String> userFeatureMap = new HashMap<>();
+        Map<String, String> c1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
+        List<TupleMapEntry<Tuple5>> midActionList = this.handleC1Feature(c1Feature, userFeatureMap);
+
+        Map<String, Double> midTimeDiffMap = this.parseC1FeatureListToTimeDiffMap(midActionList, ts);
+        Map<String, Double> actionStaticMap = this.parseC1FeatureListToActionStaticMap(midActionList);
+
+        Map<String, String> d1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
+        this.handleD1Feature(d1Feature, userFeatureMap);
+
+        Map<String, String> e1Feature = userFeature.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+
+        Map<String, String> d2Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf_rank", new HashMap<>());
+
+        for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+            String cidStr = dto.getCreativeId().toString();
+            Map<String, String> cidFeatureMap = new HashMap<>();
+            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+
+            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+
+            this.handleB1Feature(b1Feature, cidFeatureMap);
+
+            this.handleB2ToB5AndB8Feature(cidFeature, adVerFeature, cidFeatureMap);
+
+            this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+
+            this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+
+            this.handleD2Feature(d2Feature, cidFeatureMap, cidStr);
+
+            String title = b1Feature.getOrDefault("cidtitle", "");
+            this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
+        }
+
+
+        return null;
+    }
+
+    private Feature getFeature(ScoreParam param, RankRecommendRequestParam request) {
+        List<AdPlatformCreativeDTO> adIdList = request.getAdIdList();
+        List<String> cidList = adIdList.stream()
+                .map(AdPlatformCreativeDTO::getCreativeId)
+                .map(Object::toString)
+                .collect(Collectors.toList());
+
+        List<String> adVerIdList = adIdList.stream()
+                .map(AdPlatformCreativeDTO::getAdVerId)
+                .distinct()
+                .collect(Collectors.toList());
+        return featureService.getFeature(cidList, adVerIdList, param);
+    }
+
+    private void handleB1Feature(Map<String, String> b1Feature, Map<String, String> cidFeatureMap) {
+        if (StringUtils.isNotBlank(b1Feature.get("adid"))) {
+            String adId = b1Feature.get("adid");
+            cidFeatureMap.put("adid_" + adId, "1");
+        }
+        if (StringUtils.isNotBlank(b1Feature.get("adverid"))) {
+            String adVerId = b1Feature.get("adverid");
+            cidFeatureMap.put("adverid_" + adVerId, "1");
+        }
+        if (StringUtils.isNotBlank(b1Feature.get("targeting_conversion"))) {
+            String targetingConversion = b1Feature.get("targeting_conversion");
+            cidFeatureMap.put("targeting_conversion_" + targetingConversion, "1");
+        }
+        if (StringUtils.isNotBlank(b1Feature.get("cpa"))) {
+            String cpa = b1Feature.get("cpa");
+            cidFeatureMap.put("cpa", cpa);
+        }
+    }
+
+    private void handleB2ToB5AndB8Feature(Map<String, Map<String, String>> c1Feature, Map<String, Map<String, String>> adVerFeature, Map<String, String> cidFeatureMap) {
+        Map<String, String> b2Feature = adVerFeature.getOrDefault("alg_cid_feature_adver_action", new HashMap<>());
+        Map<String, String> b3Feature = c1Feature.getOrDefault("alg_cid_feature_cid_action", new HashMap<>());
+        Map<String, String> b4Feature = c1Feature.getOrDefault("alg_cid_feature_region_action", new HashMap<>());
+        Map<String, String> b5Feature = c1Feature.getOrDefault("alg_cid_feature_app_action", new HashMap<>());
+        Map<String, String> b8Feature = c1Feature.getOrDefault("alg_cid_feature_brand_action", new HashMap<>());
+
+        List<String> timeList = Arrays.asList("3h", "6h", "12h", "1d", "3d", "7d");
+        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
+                new Tuple2<>(b2Feature, "b2"),
+                new Tuple2<>(b3Feature, "b3"),
+                new Tuple2<>(b4Feature, "b4"),
+                new Tuple2<>(b5Feature, "b5"),
+                new Tuple2<>(b8Feature, "b8")
+        );
+        for (Tuple2<Map<String, String>, String> tuple2 : featureList) {
+            Map<String, String> feature = tuple2.f1;
+            String prefix = tuple2.f2;
+            for (String time : timeList) {
+                double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
+                double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
+                double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
+                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+                cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+            }
+        }
+
+    }
+
+    private void handleB6ToB7Feature(Map<String, Map<String, String>> c1Feature, Map<String, String> cidFeatureMap) {
+        Map<String, String> b6Feature = c1Feature.getOrDefault("alg_cid_feature_week_action", new HashMap<>());
+        Map<String, String> b7Feature = c1Feature.getOrDefault("alg_cid_feature_hour_action", new HashMap<>());
+
+        List<String> timeList = Arrays.asList("7d", "14d");
+        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
+                new Tuple2<>(b6Feature, "b6"),
+                new Tuple2<>(b7Feature, "b7")
+        );
+        for (Tuple2<Map<String, String>, String> tuple2 : featureList) {
+            Map<String, String> feature = tuple2.f1;
+            String prefix = tuple2.f2;
+            for (String time : timeList) {
+                double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
+                double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
+                double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
+                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+                cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+            }
+        }
+
+    }
+
+    private List<TupleMapEntry<Tuple5>> handleC1Feature(Map<String, String> c1Feature, Map<String, String> featureMap) {
+
+        // 用户特征
+        List<TupleMapEntry<Tuple5>> midActionList = new ArrayList<>();
+        if (c1Feature.containsKey("action")) {
+            String action = c1Feature.get("action");
+            midActionList = Arrays.stream(action.split(","))
+                    .map(r -> {
+                        String[] rList = r.split(":");
+                        Tuple5 tuple5 = new Tuple5(rList[1], rList[2], rList[3], rList[4], rList[5]);
+                        return new TupleMapEntry<>(rList[0], tuple5);
+                    })
+                    .sorted((a, b) -> Integer.compare(Integer.parseInt(b.value.f1), Integer.parseInt(a.value.f1)))
+                    .collect(Collectors.toList());
+        }
+
+        double viewAll = midActionList.size();
+        double clickAll = midActionList.stream().mapToInt(e -> Integer.parseInt(e.value.f2)).sum();
+        double converAll = midActionList.stream().mapToInt(e -> Integer.parseInt(e.value.f3)).sum();
+        double incomeAll = midActionList.stream().mapToInt(e -> Integer.parseInt(e.value.f4)).sum();
+        featureMap.put("viewAll", String.valueOf(viewAll));
+        featureMap.put("clickAll", String.valueOf(clickAll));
+        featureMap.put("converAll", String.valueOf(converAll));
+        featureMap.put("incomeAll", String.valueOf(incomeAll));
+        featureMap.put("ctr_all", String.valueOf(NumUtil.div(clickAll, viewAll)));
+        featureMap.put("ctcvr_all", String.valueOf(NumUtil.div(converAll, viewAll)));
+        featureMap.put("cvr_all", String.valueOf(NumUtil.div(clickAll, converAll)));
+        featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
+
+        return midActionList;
+    }
+
+    private void handleC1UIFeature(Map<String, Double> midTimeDiffMap, Map<String, Double> midActionStatic, Map<String, String> featureMap, String cid) {
+        if (midTimeDiffMap.containsKey("timediff_view_" + cid)) {
+            featureMap.put("timediff_view_" + cid, String.valueOf(midTimeDiffMap.getOrDefault("timediff_view_" + cid, 0.0)));
+        }
+        if (midTimeDiffMap.containsKey("timediff_click_" + cid)) {
+            featureMap.put("timediff_click_" + cid, String.valueOf(midTimeDiffMap.getOrDefault("timediff_click_" + cid, 0.0)));
+        }
+        if (midTimeDiffMap.containsKey("timediff_conver_" + cid)) {
+            featureMap.put("timediff_conver_" + cid, String.valueOf(midTimeDiffMap.getOrDefault("timediff_conver_" + cid, 0.0)));
+        }
+        if (midActionStatic.containsKey("actionstatic_view_" + cid)) {
+            featureMap.put("actionstatic_view_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)));
+        }
+        if (midActionStatic.containsKey("actionstatic_click_" + cid)) {
+            featureMap.put("actionstatic_click_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)));
+        }
+        if (midActionStatic.containsKey("actionstatic_conver_" + cid)) {
+            featureMap.put("actionstatic_conver_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0)));
+        }
+        if (midActionStatic.containsKey("actionstatic_income_" + cid)) {
+            featureMap.put("actionstatic_income_" + cid, String.valueOf(midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0)));
+        }
+        if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
+            double ctr = NumUtil.div(
+                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
+                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+            );
+            featureMap.put("actionstatic_ctr", String.valueOf(ctr));
+        }
+        if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("timediff_conver_" + cid)) {
+            double ctcvr = NumUtil.div(
+                    midActionStatic.getOrDefault("timediff_conver_" + cid, 0.0),
+                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+            );
+            featureMap.put("actionstatic_ctcvr", String.valueOf(ctcvr));
+        }
+        if (midActionStatic.containsKey("actionstatic_conver_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
+            double cvr = NumUtil.div(
+                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
+                    midActionStatic.getOrDefault("timediff_conver_" + cid, 0.0)
+            );
+            featureMap.put("actionstatic_cvr", String.valueOf(cvr));
+        }
+    }
+
+    private void handleD1Feature(Map<String, String> d1Feature, Map<String, String> featureMap) {
+        for (String prefix : Arrays.asList("3h", "6h", "12h", "1d", "3d", "7d")) {
+            double view = Double.parseDouble(d1Feature.getOrDefault("ad_view_" + prefix, "0"));
+            double click = Double.parseDouble(d1Feature.getOrDefault("ad_click_" + prefix, "0"));
+            double conver = Double.parseDouble(d1Feature.getOrDefault("ad_conversion_" + prefix, "0"));
+            double income = Double.parseDouble(d1Feature.getOrDefault("ad_income_" + prefix, "0"));
+            featureMap.put("d1_feature_" + prefix + "_ctr", String.valueOf(NumUtil.div(click, view)));
+            featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
+            featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+            featureMap.put("d1_feature_" + prefix + "_conver", String.valueOf(conver));
+            featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+        }
+    }
+
+    private void handleD2Feature(Map<String, String> d2Feature, Map<String, String> featureMap, String cid) {
+        if (MapUtils.isEmpty(d2Feature)) {
+            return;
+        }
+
+        Map<String, Map<String, Double>> vidRankMaps = new HashMap<>();
+        for (Map.Entry<String, String> entry : d2Feature.entrySet()) {
+            String key = entry.getKey();
+            String value = entry.getValue();
+            Map<String, Double> valueMap = Arrays.stream(value.split(","))
+                    .map(r -> r.split(":"))
+                    .collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
+            vidRankMaps.put(key, valueMap);
+        }
+
+        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
+        List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
+
+        for (String prefix1 : prefixes1) {
+            for (String prefix2 : prefixes2) {
+                String combinedKey = prefix1 + "_" + prefix2;
+                if (vidRankMaps.containsKey(combinedKey)) {
+                    Double rank = vidRankMaps.get(combinedKey).getOrDefault(cid, 0.0);
+                    if (rank >= 1.0) {
+                        featureMap.put("vid_rank_" + combinedKey, String.valueOf(NumUtil.div(1, rank)));
+                    }
+                }
+            }
+        }
+    }
+
+    private void handleE1AndE2Feature(Map<String, String> e1Feature, Map<String, String> e2Feature, String title, Map<String, String> featureMap) {
+        if (StringUtils.isEmpty(title)) {
+            return;
+        }
+
+        List<Tuple2<Map<String, String>, String>> tuple2List = Arrays.asList(
+                new Tuple2<>(e1Feature, "e1"),
+                new Tuple2<>(e2Feature, "e2")
+        );
+
+        List<String> tagsFieldList = Arrays.asList("tags_3d", "tags_7d", "tags_14d");
+        for (Tuple2<Map<String, String>, String> tuple2 : tuple2List) {
+            Map<String, String> feature = tuple2.f1;
+            String prefix = tuple2.f2;
+            if (MapUtils.isEmpty(feature)) {
+                continue;
+            }
+
+            for (String tagsField : tagsFieldList) {
+                if (StringUtils.isNotEmpty(feature.get(tagsField))) {
+                    String tags = feature.get(tagsField);
+                    Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    featureMap.put(prefix + "_" + tagsField + "_matchnum", String.valueOf(doubles[0]));
+                    featureMap.put(prefix + "_" + tagsField + "_maxscore", String.valueOf(doubles[1]));
+                    featureMap.put(prefix + "_" + tagsField + "_avgscore", String.valueOf(doubles[2]));
+                }
+            }
+        }
+    }
+
+    private Map<String, Double> parseC1FeatureListToTimeDiffMap(List<TupleMapEntry<Tuple5>> midActionList, long ts) {
+        Map<String, Double> midTimeDiffMap = new HashMap<>();
+        for (TupleMapEntry<Tuple5> entry : midActionList) {
+            String cid = entry.key;
+            long tsHistory = Long.parseLong(entry.value.f1);
+            long click = Long.parseLong(entry.value.f2);
+            long conver = Long.parseLong(entry.value.f3);
+            long d = (ts - tsHistory) / 3600 / 21;
+            if (!midTimeDiffMap.containsKey("timediff_view_" + cid)) {
+                midTimeDiffMap.put("timediff_view_" + cid, NumUtil.div(1, d));
+            }
+            if (!midTimeDiffMap.containsKey("timediff_click_" + cid) && click > 0) {
+                midTimeDiffMap.put("timediff_click_" + cid, NumUtil.div(1, d));
+            }
+            if (!midTimeDiffMap.containsKey("timediff_conver_" + cid) && conver > 0) {
+                midTimeDiffMap.put("timediff_conver_" + cid, NumUtil.div(1, d));
+            }
+        }
+        return midTimeDiffMap;
+    }
+
+    private Map<String, Double> parseC1FeatureListToActionStaticMap(List<TupleMapEntry<Tuple5>> midActionList) {
+        Map<String, Double> midActionStaticsMap = new HashMap<>();
+        for (TupleMapEntry<Tuple5> entry : midActionList) {
+            String cid = entry.key;
+            double click = Double.parseDouble(entry.value.f2);
+            double conver = Double.parseDouble(entry.value.f3);
+            double income = Double.parseDouble(entry.value.f4);
+
+            Double viewSum = midActionStaticsMap.getOrDefault("actionstatic_view_" + cid, 0.0);
+            midActionStaticsMap.put("actionstatic_view_" + cid, 1 + viewSum);
+
+            Double clickSum = midActionStaticsMap.getOrDefault("actionstatic_click_" + cid, 0.0);
+            midActionStaticsMap.put("actionstatic_click_" + cid, clickSum + click);
+
+            Double converSum = midActionStaticsMap.getOrDefault("actionstatic_conver_" + cid, 0.0);
+            midActionStaticsMap.put("actionstatic_conver_" + cid, converSum + conver);
+
+            Double incomSum = midActionStaticsMap.getOrDefault("actionstatic_income_" + cid, 0.0);
+            midActionStaticsMap.put("actionstatic_income_" + cid, incomSum + income);
+        }
+
+        return midActionStaticsMap;
+    }
+
+    public static class Tuple5 {
+        public String f1;
+        public String f2;
+        public String f3;
+        public String f4;
+        public String f5;
+
+        public Tuple5(String f1, String f2, String f3, String f4, String f5) {
+            this.f1 = f1;
+            this.f2 = f2;
+            this.f3 = f3;
+            this.f4 = f4;
+            this.f5 = f5;
+        }
+    }
+
+    public static class TupleMapEntry<T> {
+        public String key;
+        public T value;
+
+        public TupleMapEntry(String key, T value) {
+            this.key = key;
+            this.value = value;
+        }
+    }
+
+    public static class Tuple2<F1, F2> {
+        public F1 f1;
+
+        public F2 f2;
+
+        public Tuple2(F1 first, F2 name) {
+            this.f1 = first;
+            this.f2 = name;
+        }
+
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
+import com.tzld.piaoquan.ad.engine.commons.util.AbUtil;
 import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
 import com.tzld.piaoquan.ad.engine.service.predict.helper.NewExpInfoHelper;
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelParam;
@@ -22,13 +23,13 @@ import com.tzld.piaoquan.ad.engine.service.score.param.UnionRankRecommendRequest
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdItemFeature;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
@@ -39,19 +40,26 @@ import java.util.stream.Collectors;
 public class RankServiceImpl implements RankService {
 
     private final static Logger log = LoggerFactory.getLogger(RankServiceImpl.class);
-    DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd");
-    DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("yyyyMMdd HH:mm:ss");
+    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd");
+    private static final DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("yyyyMMdd HH:mm:ss");
+
+
     @Autowired
-    FeatureRemoteService featureRemoteService;
+    private FeatureRemoteService featureRemoteService;
     @Autowired
-    RankServiceThompsonImpl rankServiceThompson;
+    private RankServiceThompsonImpl rankServiceThompson;
     @Autowired
-    VideoAdThompsonScorerV2 videoAdThompsonScorerV2;
+    private VideoAdThompsonScorerV2 videoAdThompsonScorerV2;
     @Autowired
     private LogHubService logHubService;
+    @Autowired
+    private AdCreativeFeatureContainer adCreativeFeatureContainer;
 
     @Autowired
-    AdCreativeFeatureContainer adCreativeFeatureContainer;
+    private RankService680 fmRankService;
+    @Autowired
+    private TacticsAndFmModelScoreRankService tacticsAndFmModelScoreRankService;
+
     @Value("${ad.model.cpm.max:200}")
     Double cpmMax = 200d;
     @Value("${ad.model.cpm.min:30}")
@@ -61,16 +69,27 @@ public class RankServiceImpl implements RankService {
     @Value("${ad.cvr.adjusting.exp:652}")
     private String cvrAdjustingExpCode;
 
-    public AdRankItem adItemRank(RankRecommendRequestParam request) {
+    @Value("${fm.model.score.exp.code:680}")
+    private String fmModelScoreExpCode;
+
+    @Value("${tactics.fm.mode.score.exp.code:679}")
+    private String tacticsAndFmModelScoreExpCode;
+
+    public AdRankItem adItemRank(RankRecommendRequestParam requestParam) {
+
+        Set<String> expCodeSet = AbUtil.unfoldAllExpCode(requestParam.getAdAbExpArr());
+        Long appType = requestParam.getAppType();
+        Integer newExpGroup = requestParam.getNewExpGroup();
+        if (AbUtil.isInAbExp(expCodeSet, appType, newExpGroup, fmModelScoreExpCode)) {
+            return fmRankService.adItemRank(requestParam);
+        }else if (AbUtil.isInAbExp(expCodeSet, appType, newExpGroup, tacticsAndFmModelScoreExpCode)){
+            return tacticsAndFmModelScoreRankService.adItemRank(requestParam);
+        }
+        return adItemRankOld(requestParam);
+    }
+
+    public AdRankItem adItemRankOld(RankRecommendRequestParam request) {
         ScoreParam param = RequestConvert.requestConvert(request);
-        LocalDateTime currentTime = LocalDateTime.now();
-        int currentHour = currentTime.getHour();
-        int dayOfWeek = currentTime.getDayOfWeek().getValue();
-        param.getRequestContext().setHour(currentHour + "");
-        param.getRequestContext().setWeek(dayOfWeek + "");
-        param.getRequestContext().setRegion(request.getRegion().replace("省", ""));
-        param.getRequestContext().setCity(request.getCity().replace("市", ""));
-        param.getRequestContext().setDay(currentTime.format(dateFormatter));
 
         UserAdFeature userAdFeature = featureRemoteService.getUserAdFeature(request.getMid());
         if (userAdFeature == null) {
@@ -79,14 +98,14 @@ public class RankServiceImpl implements RankService {
         Map<Long, List<AdPlatformCreativeDTO>> groupMap = request
                 .getAdIdList()
                 .stream()
-                .collect(Collectors.groupingBy(creativeDTO -> creativeDTO.getCreativeId()));
+                .collect(Collectors.groupingBy(AdPlatformCreativeDTO::getCreativeId));
         Map<Long, AdRankItem> cache = adCreativeFeatureContainer.getAll(new ArrayList<>(groupMap.keySet()));
         List<AdRankItem> rankItems = Collections.emptyList();
         if (!cache.isEmpty()) {
             rankItems = new LinkedList<>(cache.values());
         }
         // 避免recommend-feature出问题
-        if (rankItems == null || rankItems.size() == 0) {
+        if (CollectionUtils.isEmpty(rankItems)) {
             rankItems = new LinkedList<>();
             for (Long adId : groupMap.keySet()) {
                 AdRankItem item = new AdRankItem();
@@ -97,15 +116,24 @@ public class RankServiceImpl implements RankService {
         }
         boolean inCpcPidExp = false;
         boolean inCvrAdjustingExp = false;
-        if (request.getAdAbExpArr() != null && request.getAdAbExpArr().size() != 0) {
-            for (Map<String, Object> map : request.getAdAbExpArr()) {
-                if (map.getOrDefault("abExpCode", "").equals(cpcPidExpCode)) {
-                    inCpcPidExp = true;
-                }
-                if (map.getOrDefault("abExpCode", "").equals(cvrAdjustingExpCode)) {
-                    inCvrAdjustingExp = true;
-                }
-            }
+        // if (request.getAdAbExpArr() != null && request.getAdAbExpArr().size() != 0) {
+        //     for (Map<String, Object> map : request.getAdAbExpArr()) {
+        //         if (map.getOrDefault("abExpCode", "").equals(cpcPidExpCode)) {
+        //             inCpcPidExp = true;
+        //         }
+        //         if (map.getOrDefault("abExpCode", "").equals(cvrAdjustingExpCode)) {
+        //             inCvrAdjustingExp = true;
+        //         }
+        //     }
+        // }
+        if (CollectionUtils.isNotEmpty(request.getAdAbExpArr())) {
+            Set<String> abExpCode = request.getAdAbExpArr().stream()
+                    .map(map -> map.get("abExpCode"))
+                    .filter(Objects::nonNull)
+                    .map(Object::toString)
+                    .collect(Collectors.toSet());
+            inCpcPidExp = abExpCode.contains(cpcPidExpCode);
+            inCvrAdjustingExp = abExpCode.contains(cvrAdjustingExpCode);
         }
         double lambda = -1d;
         if (inCpcPidExp) {
@@ -163,7 +191,7 @@ public class RankServiceImpl implements RankService {
             object.put("score", rankResult.get(0).getScore());
             object.put("pidLambda", rankResult.get(0).getPidLambda());
             object.put("lrsamples", rankResult.get(0).getLrSampleString());
-            object.put("dataTime", currentTime.format(timeFormatter));
+            object.put("dataTime", LocalDateTime.now().format(timeFormatter));
             object.put("creativeId", rankResult.get(0).getAdId());
             object.put("videoId", request.getVideoId());
             object.put("pqtId", request.getPqtId());
@@ -180,7 +208,7 @@ public class RankServiceImpl implements RankService {
             }
 
             for (AdRankItem adRankItem : rankResult) {
-                adRankItem.getExt().put("userAdFeature", JSON.toJSONString(userAdFeature));
+                adRankItem.getFeature().put("userAdFeature", JSON.toJSONString(userAdFeature));
             }
             // 日志上报
             logHubService.scoreLogUpload(param, request.getAdIdList(), rankResult, request,
@@ -205,12 +233,21 @@ public class RankServiceImpl implements RankService {
         param.getRequestContext().setDay(currentTime.format(dateFormatter));
 
         Set<String> expCodes = new HashSet<>();
-        if (request.getAdAbExpArr() != null && request.getAdAbExpArr().size() != 0) {
-            for (Map<String, Object> map : request.getAdAbExpArr()) {
-                String expCode = map.getOrDefault("abExpCode", "").toString();
-                expCodes.add(expCode);
-            }
+        // if (request.getAdAbExpArr() != null && request.getAdAbExpArr().size() != 0) {
+        //     for (Map<String, Object> map : request.getAdAbExpArr()) {
+        //         String expCode = map.getOrDefault("abExpCode", "").toString();
+        //         expCodes.add(expCode);
+        //     }
+        // }
+
+        if (CollectionUtils.isNotEmpty(request.getAdAbExpArr())) {
+            expCodes = request.getAdAbExpArr().stream()
+                    .map(m -> m.get("abExpCode"))
+                    .filter(Objects::nonNull)
+                    .map(Object::toString)
+                    .collect(Collectors.toSet());
         }
+
         ThresholdPredictModelParam modelParam = ThresholdPredictModelParam.builder()
                 .build();
         // 兜底方案

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/TacticsAndFmModelScoreRankService.java

@@ -0,0 +1,16 @@
+package com.tzld.piaoquan.ad.engine.service.score.impl;
+
+import com.tzld.piaoquan.ad.engine.service.score.RankService;
+import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+@Slf4j
+@Service
+public class TacticsAndFmModelScoreRankService implements RankService {
+    @Override
+    public AdRankItem adItemRank(RankRecommendRequestParam request) {
+        return null;
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/param/RankRecommendRequestParam.java

@@ -2,10 +2,12 @@ package com.tzld.piaoquan.ad.engine.service.score.param;
 
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 
 import java.util.List;
 
 @Data
+@EqualsAndHashCode(callSuper = true)
 public class RankRecommendRequestParam extends RecommendRequestParam{
      List<AdPlatformCreativeDTO> adIdList;
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/param/RecommendRequestParam.java

@@ -9,19 +9,19 @@ import java.util.Map;
 @Data
 @ToString
 public class RecommendRequestParam {
-    MachineInfoParam machineInfo;
+    private MachineInfoParam machineInfo;
 
-    Long videoId;
-    Long appType;
-    String mid;
-    //省-中文
-    String region = "-1";
-    //市-中文
-    String city = "-1";
-    Integer newExpGroup;
-    String pqtId;
-    List<Map> adAbExpArr ;
-    String adAbGroup;
+    private Long videoId;
+    private Long appType;
+    private String mid;
+    // 省-中文
+    private String region = "-1";
+    // 市-中文
+    private String city = "-1";
+    private Integer newExpGroup;
+    private String pqtId;
+    private List<Map<String, String>> adAbExpArr;
+    private String adAbGroup;
 
     private StatisticsLogParam statisticsLog;
 }