Merge branch 'feature/zhangbo_model' of algorithm/recommend-server into master

recommend-server-service/pom.xml

@@ -73,7 +73,29 @@
             <artifactId>fastutil</artifactId>
             <version>7.0.12</version>
         </dependency>
-
+        <dependency>
+            <groupId>org.xm</groupId>
+            <artifactId>similarity</artifactId>
+            <version>1.1</version>
+            <exclusions>
+                <!--                <exclusion>-->
+                <!--                    <artifactId>logback-access</artifactId>-->
+                <!--                    <groupId>ch.qos.logback</groupId>-->
+                <!--                </exclusion>-->
+                <!--                <exclusion>-->
+                <!--                    <artifactId>logback-classic</artifactId>-->
+                <!--                    <groupId>ch.qos.logback</groupId>-->
+                <!--                </exclusion>-->
+                <!--                <exclusion>-->
+                <!--                    <artifactId>logback-core</artifactId>-->
+                <!--                    <groupId>ch.qos.logback</groupId>-->
+                <!--                </exclusion>-->
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
 
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -158,7 +180,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.1.16</version>
+            <version>1.1.20</version>
         </dependency>
         <dependency>
             <groupId>com.google.protobuf</groupId>

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/Application.java

@@ -4,6 +4,7 @@ package com.tzld.piaoquan.recommend.server;
 
 import com.tzld.piaoquan.abtest.client.ABTestClient;
 import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
+import com.tzld.piaoquan.recommend.feature.client.FeatureV2Client;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
@@ -45,6 +46,11 @@ public class Application {
         return new FeatureClient();
     }
 
+    @Bean
+    public FeatureV2Client featureV2Client() {
+        return new FeatureV2Client();
+    }
+
     @Bean
     public ABTestClient abTestClient() {
         return new ABTestClient();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -17,6 +17,7 @@ public class RankItem implements Comparable<RankItem> {
 
     // featureMap中保存所有的特征
     public Map<String, String> featureMap = new HashMap<>();
+    public Map<String, Double> featureMapDouble = new HashMap<>();
     public String id;
     public Map<String, Double> scoresMap = new HashMap<>();
     public Map<String, String> itemBasicFeature = new HashMap<>();
@@ -27,6 +28,7 @@ public class RankItem implements Comparable<RankItem> {
     private Video video;
     private double scoreRos; // 记录ros的score
     private double scoreStr; // 记录str的score
+    private double scoreRov; // 记录str的score
 
     // 记录Item侧用到的特征
     private ItemFeature itemFeature;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/remote/FeatureV2RemoteService.java

@@ -0,0 +1,34 @@
+package com.tzld.piaoquan.recommend.server.remote;
+
+import com.tzld.piaoquan.recommend.feature.client.FeatureV2Client;
+import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * @author dyp
+ */
+@Component
+@Slf4j
+public class FeatureV2RemoteService {
+
+    @Autowired
+    private FeatureV2Client client;
+
+    public Map<String, String> getFeature(List<FeatureKeyProto> protos) {
+        if (CollectionUtils.isEmpty(protos)) {
+            return Collections.emptyMap();
+        }
+        Map<String, String> result = client.multiGetFeature(protos);
+        return result;
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/FeatureService.java

@@ -0,0 +1,165 @@
+package com.tzld.piaoquan.recommend.server.service;
+
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
+import com.tzld.piaoquan.recommend.server.remote.FeatureV2RemoteService;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+
+/**
+ * @author dyp
+ */
+@Service
+@Slf4j
+public class FeatureService {
+
+    @Autowired
+    private FeatureV2RemoteService remoteService;
+
+    /**
+     * @return k1:视频、k2:表、k3:特征、v:特征值
+     */
+    public Feature getFeature(String mid, List<String> vidList, String appType,
+                              String province, String headVid) {
+
+
+        List<FeatureKeyProto> protos = new ArrayList<>();
+
+        for (String vid : vidList) {
+            // TODO 补充其他特征
+            // vid
+            protos.add(genWithVid("alg_vid_feature_all_exp", vid));
+            protos.add(genWithVid("alg_vid_feature_all_share", vid));
+            protos.add(genWithVid("alg_vid_feature_all_return", vid));
+            protos.add(genWithVid("alg_vid_feature_exp2share", vid));
+            protos.add(genWithVid("alg_vid_feature_share2return", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_exp", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_root_share", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_root_return", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_exp", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_root_share", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_root_return", vid));
+
+            protos.add(genWithVid("alg_vid_feature_basic_info", vid));
+            // vid + apptype
+
+            // vid + province
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_exp", vid, province));
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_root_share", vid, province));
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_root_return", vid, province));
+
+            // vid + headvid
+            protos.add(genWithVidAndHeadVid("alg_recsys_feature_cf_i2i_new", vid, headVid));
+        }
+
+
+        // user
+        protos.add(genWithMid("alg_mid_feature_play", mid));
+        protos.add(genWithMid("alg_mid_feature_share_and_return", mid));
+        protos.add(genWithMid("alg_mid_feature_play_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_return_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_share_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_feed_exp_share_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_feed_exp_return_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_sharecf", mid));
+        protos.add(genWithMid("alg_mid_feature_returncf", mid));
+
+
+        Map<String, String> result = remoteService.getFeature(protos);
+
+        Feature feature = new Feature();
+
+        result.entrySet().forEach(e -> {
+
+            String[] uk = StringUtils.split(e.getKey(), ":");
+            String prefix = uk[0];
+            String table = uk[1];
+            Map<String, String> colMap = JSONUtils.fromJson(e.getValue(), new TypeToken<Map<String, String>>() {
+            }, Collections.emptyMap());
+            String featureStr = colMap.get("feature");
+
+            switch (prefix) {
+                case "v":
+                    String vid = uk[2];
+                    Map<String, Map<String, String>> tableFeatureMap = feature.getVideoFeature().getOrDefault(vid, new HashMap<>());
+                    tableFeatureMap.put(table, JSONUtils.fromJson(featureStr, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    feature.getVideoFeature().put(vid, tableFeatureMap);
+                    break;
+                case "u":
+                    feature.getUserFeature().put(table, JSONUtils.fromJson(featureStr, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    break;
+                default:
+                    break;
+            }
+
+        });
+
+
+        return feature;
+    }
+
+    private final String videoUkFormat = "v:%s:%s";
+    private final String userUkFormat = "u:%s";
+
+    private FeatureKeyProto genWithVid(String table, String vid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(videoUkFormat, table, vid))
+                .setTableName(table)
+                .putFieldValue("vid", vid)
+                .build();
+    }
+
+    private FeatureKeyProto genWithVidAndAppType(String table, String vid, String appType) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(videoUkFormat, table, vid))
+                .setTableName(table)
+                .putFieldValue("vid", vid)
+                .putFieldValue("apptype", appType)
+                .build();
+    }
+
+    private FeatureKeyProto genWithVidAndProvince(String table, String vid, String province) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(videoUkFormat, table, vid))
+                .setTableName(table)
+                .putFieldValue("vid", vid)
+                .putFieldValue("province", province)
+                .build();
+    }
+
+    private FeatureKeyProto genWithVidAndHeadVid(String table, String vid, String headVid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(videoUkFormat, table, vid))
+                .setTableName(table)
+                .putFieldValue("vid", vid)
+                .putFieldValue("headVid", headVid)
+                .build();
+    }
+
+    private FeatureKeyProto genWithMid(String table, String mid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(userUkFormat, table))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .build();
+    }
+
+    @Data
+    public static class Feature {
+        // k1:视频、k2:表、k3:特征、v:特征值
+        private Map<String, Map<String, Map<String, String>>> videoFeature = new HashMap<>();
+
+        // k1:mid、k2:特征、v:特征值
+        private Map<String, Map<String, String>> userFeature = new HashMap<>();
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -599,6 +599,7 @@ public class RecommendService {
         rankParam.setAbExpCodes(param.getAbExpCodes());
         rankParam.setExpIdMap(param.getExpIdMap());
         rankParam.setCategoryId(param.getCategoryId());
+        rankParam.setHeadVid(param.getVideoId());
         return rankParam;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/BlacklistContainer.java

@@ -155,7 +155,7 @@ public class BlacklistContainer {
     }
 
     public void refreshVideoTagCache() {
-        LOG.info("同步本地标签ID与视频列表的缓存任务开始");
+        // LOG.info("同步本地标签ID与视频列表的缓存任务开始");
         Map<Long, Set<Long>> tmpMap = new ConcurrentHashMap<>();
 
         if (MapUtils.isNotEmpty(tagFilterConfigMap)) {
@@ -179,13 +179,13 @@ public class BlacklistContainer {
             for (Long tagId : tagIdSet) {
                 List<WxVideoTagRel> wxVideoTagRels = wxVideoTagRelRepository.findAllByTagId(tagId);
                 Set<Long> videoIdSet = wxVideoTagRels.stream().map(WxVideoTagRel::getVideoId).collect(Collectors.toSet());
-                LOG.info("同步本地标签ID与视频列表缓存任务 -- tagId: {}, videoIdSize: {}", tagId, videoIdSet.size());
+                // LOG.info("同步本地标签ID与视频列表缓存任务 -- tagId: {}, videoIdSize: {}", tagId, videoIdSet.size());
                 tmpMap.put(tagId, videoIdSet);
             }
         }
         videoTagCache = tmpMap;
 
-        LOG.info("同步本地标签ID与视频列表的缓存任务结束");
+        // LOG.info("同步本地标签ID与视频列表的缓存任务结束");
     }
 
     public List<Long> filterUnsafeVideoByUser(List<Long> videoIds, String uid, Long hotSceneType, String cityCode, String clientIP, String mid, String usedScene, Integer appType) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankParam.java

@@ -25,6 +25,7 @@ public class RankParam {
     private String city;
     private MachineInfo machineInfo;
     private Set<String> abExpCodes;
+    private Long headVid=0L;
 
     // 层 - 实验
     private Map<String, String> expIdMap;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -48,7 +48,7 @@ public class RankRouter {
     @Autowired
     private RankStrategy4RegionMergeModelV567 rankStrategy4RegionMergeModelV567;
     @Autowired
-    private RankStrategy4RegionMergeModelV568 rankStrategy4RegionMergeModelV568;
+    private RankStrategy4RegionMergeModelV999 rankStrategy4RegionMergeModelV999;
     @Autowired
     private RankStrategy4RegionMergeModelV569 rankStrategy4RegionMergeModelV569;
     @Autowired
@@ -91,7 +91,7 @@ public class RankRouter {
             case "60117": // 567
                 return rankStrategy4RegionMergeModelV567.rank(param);
             case "60118": // 568
-                return rankStrategy4RegionMergeModelV568.rank(param);
+                return rankStrategy4RegionMergeModelV999.rank(param);
             case "60119": // 569
                 return rankStrategy4RegionMergeModelV569.rank(param);
             case "60120": // 576

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/ExtractorUtils.java

@@ -5,9 +5,69 @@ import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-
+import org.xm.Similarity;
 public class ExtractorUtils {
 
+    public static int findInsertPosition(double[] sortedArray, double target) {
+        int low = 0;
+        int high = sortedArray.length - 1;
+
+        while (low <= high) {
+            int mid = low + (high - low) / 2;
+            double midValue = sortedArray[mid];
+
+            if (midValue < target) {
+                low = mid + 1;
+            } else if (midValue > target) {
+                high = mid - 1;
+            } else {
+                // 找到相等的值，尝试在右侧寻找插入点
+                while (mid < sortedArray.length - 1 && sortedArray[mid + 1] == target) {
+                    mid++;
+                }
+                return mid + 1; // 返回当前mid的下一个位置作为插入点
+            }
+        }
+
+        return low; // 返回low作为插入点
+    }
+    public static Double[] funcC34567ForTags(String tags, String title) {
+        String[] tagsList = tags.split(",");
+        int d1 = 0;
+        List<String> d2 = new ArrayList<>();
+        double d3 = 0.0;
+        double d4 = 0.0;
+
+        for (String tag : tagsList) {
+            if (title.contains(tag)) {
+                d1++;
+                d2.add(tag);
+            }
+            double score = Similarity.conceptSimilarity(tag, title);
+            if (score > d3) {
+                d3 = score;
+            }
+            d4 += score;
+        }
+
+        d4 = (tagsList.length > 0) ? d4 / tagsList.length : d4;
+
+        // 使用数组来返回多个值
+        Double[] result = {(double) d1, d3, d4};
+        return result;
+    }
+    public static Double calDiv(double a, double b){
+        if (a == 0 || b == 0){
+            return 0D;
+        }
+        return a / b;
+    }
+    public static Double calLog(double a){
+        if (a <= 0){
+            return 0D;
+        }
+        return Math.log(a + 1.0);
+    }
     public static Double division(String s1, String s2, Map<String, String> maps){
         double rate = 0.0;
         if (maps.containsKey(s1) && maps.containsKey(s2)){

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV999.java

@@ -0,0 +1,398 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeModelBasic {
+    @ApolloJsonValue("${rank.score.merge.weightv568:}")
+    private Map<String, Double> mergeWeight;
+
+    @Autowired
+    private FeatureService featureService;
+
+    Map<String, double[]> bucketsMap = new HashMap<>();
+    Map<String, Double> bucketsLen = new HashMap<>();
+
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        List<Video> oldRovs = new ArrayList<>();
+        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        removeDuplicate(oldRovs);
+        int sizeReturn = param.getSize();
+        List<Video> v0 = oldRovs.size() <= sizeReturn
+                ? oldRovs
+                : oldRovs.subList(0, sizeReturn);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v0);
+
+
+        //-------------------相关性召回 融合+去重-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+        //-------------------流量池直接送 融合+去重-------------------
+        List<Video> v9 = extractAndSort(param, FlowPoolLastDayTopRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v9);
+        //-------------------地域相关召回 融合+去重-------------------
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v1);
+        //-------------------节日扶持召回 融合+去重-------------------
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
+        List<Video> rovRecallRank = new ArrayList<>();
+        rovRecallRank.addAll(v0);
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 5.0).intValue(), v9.size())));
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+        // k1:视频、k2:表、k3:特征、v:特征值
+        String provinceCn = param.getProvince().replaceAll("省$", "");
+        String headVid = String.valueOf(param.getHeadVid());
+        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
+                String.valueOf(param.getAppType()), provinceCn, headVid);
+        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
+        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
+
+
+
+
+        // TODO 2 特征处理
+        Map<String, Double> userFeatureMapDouble = new HashMap<>();
+        String mid = param.getMid();
+        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
+        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
+        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
+        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags", new HashMap<>());
+        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
+        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
+
+        if (!c1.isEmpty()){
+            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
+            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
+            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
+            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
+        }
+        if (!c2.isEmpty()){
+            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
+            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
+            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
+            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
+            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
+            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
+            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
+            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
+        }
+
+        Map<String, String> c34567Map = new HashMap<>(15);
+        List<Tuple2> tmpList0 = Arrays.asList(
+                new Tuple2(c3, "c3_feature"),
+                new Tuple2(c4, "c4_feature"),
+                new Tuple2(c5, "c5_feature"),
+                new Tuple2(c6, "c6_feature"),
+                new Tuple2(c7, "c7_feature")
+        );
+        for (Tuple2 tuple2: tmpList0){
+            for (String key_time: Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                String tags = tuple2.first.getOrDefault(key_time, "");
+                if (!tags.isEmpty()){
+                    c34567Map.put(tuple2.name + "_" + key_time, tags);
+                }
+            }
+        }
+
+        Map<String, Map<String, String []>> c89Map = new HashMap<>(4);
+        List<Tuple2> tmpList1 = Arrays.asList(
+                new Tuple2(c8, "c8_feature"),
+                new Tuple2(c9, "c9_feature")
+        );
+        for (Tuple2 tuple2: tmpList1){
+            for (String key_action: Arrays.asList("share", "return")){
+                String cfListStr = tuple2.first.getOrDefault(key_action, "");
+                if (!cfListStr.isEmpty()){
+                    Map<String, String []> cfMap = new HashMap<>();
+                    String[] entries = cfListStr.split(",");
+                    for (String entry : entries) {
+                        String[] rList = entry.split(":");
+                        if (rList.length >= 4) { // 确保分割后有四个元素
+                            String key = rList[0];
+                            String value1 = rList[1];
+                            String value2 = rList[2];
+                            String value3 = rList[3];
+                            String [] strs = {value1, value2, value3};
+                            cfMap.put(key, strs);
+                        }
+                    }
+                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
+                }
+            }
+        }
+
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        for (RankItem item : rankItems) {
+            Map<String, Double> featureMap = new HashMap<>();
+            String vid = item.getVideoId() + "";
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp", new HashMap<>());
+            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
+            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share", new HashMap<>());
+            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
+
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return", new HashMap<>());
+
+            List<Tuple4> originData = Arrays.asList(
+                    new Tuple4(b1, b2, b3, "b123"),
+                    new Tuple4(b1, b6, b7, "b167"),
+                    new Tuple4(b8, b9, b10, "b8910"),
+                    new Tuple4(b11, b12, b13, "b111213"),
+                    new Tuple4(b17, b18, b19, "b171819")
+            );
+
+            for (Tuple4 tuple4 : originData) {
+                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
+                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault( "exp_pv_" + prefix2, "0.0"));
+                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault( "share_pv_" + prefix2, "0.0"));
+                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault( "return_uv_" + prefix2, "0.0"));
+
+                    double f1 = ExtractorUtils.calDiv(share, exp);
+                    double f2 = ExtractorUtils.calLog(share);
+                    double f3 = ExtractorUtils.calDiv(returns, exp);
+                    double f4 = ExtractorUtils.calLog(returns);
+                    double f5 = f3 * f4;
+
+                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
+                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
+                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
+                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
+                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
+
+                    featureMap.put(key1, f1);
+                    featureMap.put(key2, f2);
+                    featureMap.put(key3, f3);
+                    featureMap.put(key4, f4);
+                    featureMap.put(key5, f5);
+                }
+            }
+
+            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
+            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
+
+            String title = videoInfo.getOrDefault("title", "");
+            if (!title.isEmpty()){
+                for (String name: Arrays.asList("c3_feature","c4_feature","c5_feature","c6_feature","c7_feature")){
+                    for (String key_time: Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                        if (!tags.isEmpty()){
+                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
+                        }
+                    }
+                }
+            }
+
+            if (!vid.isEmpty()){
+                for (String key_feature: Arrays.asList("c8_feature", "c9_feature")){
+                    for (String key_action: Arrays.asList("share", "return")){
+                        Map<String, String []> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
+                        if (cfMap.containsKey(vid)){
+                            String [] scores = cfMap.get(vid);
+                            Double score1 = Double.parseDouble(scores[0]);
+                            Double score2 = Double.parseDouble(scores[1]);
+                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
+                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
+                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
+                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
+                        }
+                    }
+                }
+            }
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new", new HashMap<>());
+            if (!d1.isEmpty()){
+                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
+                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
+                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
+            }
+            item.featureMapDouble = featureMap;
+        }
+
+        // 3 连续值特征分桶
+        readBucketFile();
+        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
+        for (Map.Entry<String, Double> entry: userFeatureMapDouble.entrySet()){
+            String name = entry.getKey();
+            Double score = entry.getValue();
+            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)){
+                Double bucketNum = this.bucketsLen.get(name);
+                double [] buckets = this.bucketsMap.get(name);
+                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                userFeatureMap.put(name, String.valueOf(scoreNew));
+            }
+        }
+
+        for (RankItem item : rankItems){
+            Map<String, String> featureMap = new HashMap<>();
+            Map<String, Double> featureMapDouble = item.featureMapDouble;
+
+            for (Map.Entry<String, Double> entry: featureMapDouble.entrySet()){
+                String name = entry.getKey();
+                Double score = entry.getValue();
+                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)){
+                    Double bucketNum = this.bucketsLen.get(name);
+                    double [] buckets = this.bucketsMap.get(name);
+                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                    featureMap.put(name, String.valueOf(scoreNew));
+                }
+            }
+            item.featureMap = featureMap;
+        }
+
+        // TODO 3 排序
+        Map<String, String> sceneFeatureMap = new HashMap<>(0);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240609.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : items) {
+            double score = item.getScoreRov();
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
+
+        return result;
+    }
+
+    private Map<String, Map<String, String>> extractVideoFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
+    }
+
+    private Map<String, String> extractSceneFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
+    }
+
+    private Map<String, String> extractUserFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
+    }
+
+    private void readBucketFile(){
+        InputStream resourceStream = RankStrategy4RegionMergeModelV999.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        if (resourceStream != null) {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                Map<String, double[]> bucketsMap = new HashMap<>();
+                Map<String, Double> bucketsLen = new HashMap<>();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    // 替换空格和换行符，过滤空行
+                    line = line.replace(" ", "").replaceAll("\n", "");
+                    if (!line.isEmpty()) {
+                        String[] rList = line.split("\t");
+                        if (rList.length == 3) {
+                            String key = rList[0];
+                            double value1 = Double.parseDouble(rList[1]);
+                            bucketsLen.put(key, value1);
+                            double[] value2 = Arrays.stream(rList[2].split(","))
+                                    .mapToDouble(Double::valueOf)
+                                    .toArray();
+                            bucketsMap.put(key, value2);
+                        }
+                    }
+                }
+                this.bucketsMap = bucketsMap;
+                this.bucketsLen = bucketsLen;
+            } catch (IOException e) {
+                log.error("something is wrong in parse bucket file:" + e);
+            }
+        } else {
+            log.error("no bucket file");
+        }
+
+    }
+
+    static class Tuple4 {
+        public Map<String, String> first;
+        public Map<String, String> second;
+        public Map<String, String> third;
+
+        public String name;
+
+        public Tuple4(Map<String, String> first, Map<String, String> second, Map<String, String> third, String name){
+            this.first = first;
+            this.second = second;
+            this.third = third;
+            this.name = name;
+        }
+
+    }
+
+    static class Tuple2 {
+        public Map<String, String> first;
+
+        public String name;
+
+        public Tuple2(Map<String, String> first, String name){
+            this.first = first;
+            this.name = name;
+        }
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -160,9 +160,12 @@ public class RecallService implements ApplicationContextAware {
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
                 break;
             case "60117": // 567
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
+                strategies.addAll(getRegionRecallStrategy(param));
             case "60118": // 568
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                 strategies.addAll(getRegionRecallStrategy(param));
+                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
             case "60119": // 569
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                 strategies.add(strategyMap.get(TitleTagRecallStrategyV1.class.getSimpleName()));
@@ -287,10 +290,11 @@ public class RecallService implements ApplicationContextAware {
             case "60114": // 564
             case "60115": // 565
             case "60117": // 567
-            case "60118": // 568
             case "60104": // 去掉sim的对比实验
                 strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                 break;
+            case "60118": // 568
+                break;
             case "60110": // 新内容的召回（流量池的Top内容）
                 strategies.add(strategyMap.get(TopGoodPerformanceVideoRecallStrategy.class.getSimpleName()));
                 if (!hitUserBlacklist || !isInBlacklist) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/BaseFMModelScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+import com.tzld.piaoquan.recommend.server.service.score.model.FMModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseFMModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseFMModelScorer.class);
+
+    public BaseFMModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(FMModel.class);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/BaseLRV2ModelScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+import com.tzld.piaoquan.recommend.server.service.score.model.LRV2Model;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseLRV2ModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRV2ModelScorer.class);
+
+    public BaseLRV2ModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(LRV2Model.class);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -34,6 +34,7 @@ public final class ScorerUtils {
         ScorerUtils.init(FLOWPOOL_CONF);
         ScorerUtils.init(VIDEO_SCORE_CONF_FOR_AD);
         ScorerUtils.init("feeds_score_config_20240228.conf");
+        ScorerUtils.init("feeds_score_config_20240609.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v1.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v2.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v3.conf");

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogRovFMScorer.java

@@ -0,0 +1,159 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.FMModel;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+
+public class VlogRovFMScorer extends BaseFMModelScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogRovFMScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+
+    public VlogRovFMScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                  final Map<String, String> userFeatureMap,
+                                  final List<RankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        FMModel model = (FMModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(
+                sceneFeatureMap, userFeatureMap, rankItems
+        );
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<RankItem> items) {
+        long startTime = System.currentTimeMillis();
+        FMModel model = (FMModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final FMModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMap.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public double calcScore(final FMModel model,
+                            final RankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+
+
+        Map<String, String> featureMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(item.getFeatureMap())) {
+            featureMap.putAll(item.getFeatureMap());
+        }
+        if (MapUtils.isNotEmpty(userFeatureMap)) {
+            featureMap.putAll(userFeatureMap);
+        }
+        if (MapUtils.isNotEmpty(sceneFeatureMap)) {
+            featureMap.putAll(sceneFeatureMap);
+        }
+
+        double pro = 0.0;
+        if (MapUtils.isNotEmpty(featureMap)) {
+            try {
+                pro = model.score(featureMap);
+                LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.setScoreRov(pro);
+        return pro;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogRovLRScorer.java

@@ -0,0 +1,159 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.LRV2Model;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+
+public class VlogRovLRScorer extends BaseLRV2ModelScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogRovLRScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+
+    public VlogRovLRScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                  final Map<String, String> userFeatureMap,
+                                  final List<RankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        LRV2Model model = (LRV2Model) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(
+                sceneFeatureMap, userFeatureMap, rankItems
+        );
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<RankItem> items) {
+        long startTime = System.currentTimeMillis();
+        LRV2Model model = (LRV2Model) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final LRV2Model model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMap.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public double calcScore(final LRV2Model lrModel,
+                            final RankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+
+
+        Map<String, String> featureMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(item.getFeatureMap())) {
+            featureMap.putAll(item.getFeatureMap());
+        }
+        if (MapUtils.isNotEmpty(userFeatureMap)) {
+            featureMap.putAll(userFeatureMap);
+        }
+        if (MapUtils.isNotEmpty(sceneFeatureMap)) {
+            featureMap.putAll(sceneFeatureMap);
+        }
+
+        double pro = 0.0;
+        if (MapUtils.isNotEmpty(featureMap)) {
+            try {
+                pro = lrModel.score(featureMap);
+                // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.setScoreRov(pro);
+        return pro;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/FMModel.java

@@ -0,0 +1,144 @@
+package com.tzld.piaoquan.recommend.server.service.score.model;
+
+
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class FMModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(FMModel.class);
+    private Map<String, List<Float>> model;
+
+    public void putFeature(Map<String, List<Float>> model, String[] items) {
+
+        String featureKey = items[0];
+        List<Float> weights = new ArrayList<>();
+        for (int i = 1; i < items.length; i++) {
+            weights.add(Float.valueOf(items[i]));
+        }
+
+        model.put(featureKey, weights);
+    }
+
+    public float getWeight(Map<String, List<Float>> model, String featureKey, int index) {
+        if (!model.containsKey(featureKey)) {
+            return 0.0f;
+        }
+
+        return model.get(featureKey).get(index);
+
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.model == null)
+            return 0;
+        return model.size();
+    }
+
+    public void cleanModel() {
+        this.model = null;
+    }
+
+    public Float score(Map<String, String> featureMap) {
+        float sum = 0.0f;
+
+        if (MapUtils.isNotEmpty(featureMap)) {
+            // 计算 sum w*x
+            float sum0 = 0.0f;
+            for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                float x = NumberUtils.toFloat(e.getValue(), 0.0f);
+                float w = getWeight(this.model, e.getKey(), 0);
+                sum0 += w * x;
+            }
+            sum += sum0;
+
+            // 计算 sum v*v*x*X
+            float sum1 = 0.0f;
+            for (int i = 1; i < 9; i++) {
+                float sum10 = 0.0f;
+                float sum11 = 0.0f;
+                for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                    float x = NumberUtils.toFloat(e.getValue(), 0.0f);
+                    float v = getWeight(this.model, e.getKey(), i);
+                    float d = v * x;
+                    sum10 += d;
+                    sum11 += d * d;
+                }
+                sum1 += sum10 * sum10 - sum11;
+            }
+            sum1 = 0.5f * sum1;
+            float biasW = model.get("bias").get(0);
+            sum = biasW + sum0 + sum1;
+        }
+
+        return (float) (1.0f / (1 + Math.exp(-sum)));
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        Map<String, List<Float>> model = new HashMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", model.size(), curTime);
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 9) {
+                if (items[0].equals("bias")) {
+                    putFeature(model, items);
+                }
+                continue;
+            }
+
+            putFeature(model, items);
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.model = model;
+
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", model.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 9) {
+                continue;
+            }
+            putFeature(model, items);
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", model.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/LRV2Model.java

@@ -0,0 +1,111 @@
+package com.tzld.piaoquan.recommend.server.service.score.model;
+
+
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class LRV2Model extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(LRV2Model.class);
+    private Map<String, Float> lrModel;
+
+    public void putFeature(Map<String, Float> model, String featureKey, float weight) {
+        model.put(featureKey, weight);
+    }
+
+    public float getWeight(Map<String, Float> model, String featureKey) {
+        return model.getOrDefault(featureKey, 0.0f);
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.lrModel == null)
+            return 0;
+        return lrModel.size();
+    }
+
+    public void cleanModel() {
+        this.lrModel = null;
+    }
+
+    public Float score(Map<String, String> featureMap) {
+        float sum = 0.0f;
+
+        if (MapUtils.isNotEmpty(featureMap)) {
+            for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                float w = getWeight(this.lrModel, e.getKey());
+                sum += w * NumberUtils.toFloat(e.getValue(), 0.0f);
+            }
+
+            float biasW = lrModel.get("bias");
+            sum += biasW;
+        }
+
+
+        return (float) (1.0f / (1 + Math.exp(-sum)));
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        Map<String, Float> model = new HashMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+
+            putFeature(model, items[0], Float.valueOf(items[1].trim()).floatValue());
+            if (cnt++ < 10) {
+                LOGGER.debug("fea: " + items[0] + ", weight: " + items[1]);
+            }
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.lrModel = model;
+
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+            putFeature(model, items[0], Float.valueOf(items[1]).floatValue());
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/FlowPoolScorer.java

@@ -54,5 +54,10 @@ public class FlowPoolScorer extends AbstractScorer4Recall {
         return id2BetaScore.subList(0, Math.min(60, id2BetaScore.size()));
     }
 
+    public static void main(String[] args) {
+        BetaDistribution betaSample;
+        betaSample = new BetaDistribution(0.0, 0.0);
+        System.out.println(betaSample.sample());
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/web/RecommendController.java

@@ -1,12 +1,15 @@
 package com.tzld.piaoquan.recommend.server.web;
 
 import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.tzld.piaoquan.recommend.server.client.ProtobufUtils;
 import com.tzld.piaoquan.recommend.server.client.RecommendHttpRequest;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendResponse;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.VideoRecommendService;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
@@ -26,6 +29,15 @@ public class RecommendController {
     @Autowired
     private VideoRecommendService videoRecommendService;
 
+    @Autowired
+    private FeatureService featureService;
+
+    @RequestMapping("/demo")
+    public String demo() {
+//        return JSONUtils.toJson(featureService.getFeature(Lists.newArrayList("10054148", "20637161"), "0", "1"));
+        return "";
+    }
+
     @RequestMapping("/homepage/recommend")
     public String homepageRecommend(@RequestBody RecommendHttpRequest httpRequest) {
         MDC.put("appType", String.valueOf(httpRequest.getAppType()));

recommend-server-service/src/main/resources/feeds_score_config_20240609.conf

@@ -0,0 +1,7 @@
+scorer-config = {
+  rov-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogRovFMScorer"
+    scorer-priority = 96
+    model-path = "zhangbo/model_aka8.txt"
+  }
+}