Merge branch 'vlog_merge_refactor_smz' of algorithm/recommend-server into master

召回架构重构

recommend-server-client/src/main/java/com/tzld/piaoquan/recommend/server/client/ModelClient.java

@@ -3,7 +3,6 @@ package com.tzld.piaoquan.recommend.server.client;
 import com.tzld.piaoquan.recommend.server.gen.model.ModelServiceGrpc;
 import com.tzld.piaoquan.recommend.server.gen.model.ScoreRequest;
 import com.tzld.piaoquan.recommend.server.gen.model.ScoreResponse;
-import lombok.extern.slf4j.Slf4j;
 import net.devh.boot.grpc.client.inject.GrpcClient;
 import org.springframework.stereotype.Component;
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/Application.java

@@ -21,6 +21,8 @@ import org.springframework.http.converter.protobuf.ProtobufHttpMessageConverter;
 })
 @ComponentScan({
         "com.tzld.piaoquan.recommend.server.service",
+        "com.tzld.piaoquan.recommend.server.implement",
+        "com.tzld.piaoquan.recommend.server.framework.utils",
         "com.tzld.piaoquan.recommend.server.grpcservice",
         "com.tzld.piaoquan.recommend.server.remote",
         "com.tzld.piaoquan.recommend.server.config",

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -1,10 +1,14 @@
 package com.tzld.piaoquan.recommend.server.common.base;
 
+import com.google.common.collect.Maps;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.CandidateInfo;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import lombok.Data;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 
@@ -13,6 +17,7 @@ public class RankItem implements Comparable<RankItem> {
 
     // featureMap中保存所有的特征
     public Map<String, String> featureMap = new HashMap<>();
+    public String id;
     public Map<String, Double> scoresMap = new HashMap<>();
     public Map<String, String> itemBasicFeature = new HashMap<>();
     public Map<String, Map<String, Double>> itemRealTimeFeature = new HashMap<>();
@@ -25,7 +30,34 @@ public class RankItem implements Comparable<RankItem> {
     // 记录Item侧用到的特征
     private ItemFeature itemFeature;
 
+    // 记录召回信息
+    private String queue;
+    private CandidateInfo candidateInfo;
+    private List<CandidateInfo> candidateInfoList = new ArrayList<>(); // 兼容多个召回key命中
+
+    // merge信息
+    private List<String> mergeQueuePath = new ArrayList<>();
+    private List<String> mergeDecisionLabels = new ArrayList<>();
+
+    // 多样性过滤因子
+    // 排序因子
+    private Map<String, Double> rankItemCategories = Maps.newHashMap();
+    private Map<String, Double> rankItemSubCategories = Maps.newHashMap();
+    private Map<String, Double> rankItemUserTags = Maps.newHashMap();
+    private Map<String, Double> rankItemTitleTags = Maps.newHashMap();
+
+
+    // 排序侧信息
+    private Map<String, Double> rankerScore = Maps.newHashMap();
+    private Map<String, Integer> rankerIndex = Maps.newHashMap();
+    public RankItem(){
+
+    }
+
+
+
     public RankItem(Video video) {
+        this.id = String.valueOf(video.getVideoId());
         this.videoId = video.getVideoId();
         this.score = 0.0;
         this.scoreRos = 0.0;
@@ -34,12 +66,37 @@ public class RankItem implements Comparable<RankItem> {
     }
 
     public RankItem(Long videoId) {
+        this.id = videoId.toString();
         this.videoId = videoId;
         this.score = 0.0;
     }
 
-    private Map<String, Double> rankerScore = new HashMap<>();
-    private Map<String, Integer> rankerIndex = new HashMap<>();
+    public RankItem(RankItem other) {
+        this.id = other.id;
+        this.videoId = other.videoId;
+        this.score = other.score;
+        this.scoreRos = other.scoreRos;
+        this.scoreStr = other.scoreStr;
+        this.video = other.video;
+
+        // merge queue
+        this.queue = other.queue;
+        if (other.getMergeQueuePath() != null)
+            this.mergeQueuePath.addAll(other.getMergeQueuePath());
+        if (other.getMergeDecisionLabels() != null)
+            this.mergeDecisionLabels.addAll(other.getMergeDecisionLabels());
+
+
+        // candidateinfo
+        this.candidateInfo = other.getCandidateInfo() != null ? other.getCandidateInfo().deepcopy() : null;
+        if (other.getCandidateInfoList() != null) {
+            for (CandidateInfo tmpCandidateInfo : other.getCandidateInfoList()) {
+                this.candidateInfoList.add(tmpCandidateInfo.deepcopy());
+            }
+        }
+
+
+    }
 
 
     @Override
@@ -56,5 +113,39 @@ public class RankItem implements Comparable<RankItem> {
         }
     }
 
+    public RankItem deepcopy() {
+        return new RankItem(this);
+    }
+
+
+    public void addToCandidateInfoList(CandidateInfo candidateInfo) {
+        this.candidateInfoList.add(candidateInfo);
+    }
+
+
+    public void addMergeQueuePath(String mergeQueueName) {
+        this.mergeQueuePath.add(mergeQueueName);
+    }
+
+    public void addMergeDecisionLabel(String label) {
+        this.mergeDecisionLabels.add(label);
+    }
+
+    public void putRankerScore(String rankerName, Double rankerScore) {
+        this.rankerScore.put(rankerName, rankerScore);
+    }
+
+    public String getLastMergeQueueName() {
+        int pathSize = this.mergeQueuePath.size();
+        if (pathSize > 0) {
+            return this.mergeQueuePath.get(pathSize - 1);
+        } else {
+            return getCandidateInfo().getCandidateQueueName();
+        }
+    }
+
+
+
+
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Candidate.java

@@ -0,0 +1,40 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import com.google.gson.Gson;
+import lombok.Data;
+
+
+/**
+ * 记录召回下一路队列索引信息
+ */
+@Data
+public class Candidate {
+    private String candidateKey;
+    private int candidateNum;
+    private QueueName candidateQueueName;
+    // 上层待合并的queue name
+    private String mergeQueueName;
+    private int mergeQueueNum;
+
+    public Candidate() {
+    }
+
+    public Candidate(Candidate other) {
+        this.candidateKey = other.getCandidateKey();
+        this.candidateNum = other.getCandidateNum();
+        this.candidateQueueName = other.getCandidateQueueName();
+        this.mergeQueueName = other.getMergeQueueName();
+        this.mergeQueueNum = other.getMergeQueueNum();
+    }
+
+    public Candidate deepcopy() {
+        return new Candidate(this);
+    }
+
+
+    @Override
+    public String toString() {
+        return new Gson().toJson(this);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/CandidateInfo.java

@@ -0,0 +1,31 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import com.google.gson.Gson;
+import lombok.Data;
+
+@Data
+public class CandidateInfo {
+    private String candidateQueueName;     // 队列名称
+    private int position;               // 在召回中的位置
+    private Candidate candidate;        // 队列配置的信息
+
+    public CandidateInfo() {
+    }
+
+    public CandidateInfo(CandidateInfo other) {
+        this.candidateQueueName = other.getCandidateQueueName();
+        this.position = other.getPosition();
+        this.candidate = other.candidate != null ? new Candidate(other.candidate) : null;
+    }
+
+    public CandidateInfo deepcopy() {
+        return new CandidateInfo(this);
+    }
+
+
+    @Override
+    public String toString() {
+        return new Gson().toJson(this);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Entry.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * 一个队列召回的内容实体
+ * @param <T>
+ */
+public class Entry<T> {
+    public final String id;
+    public final T item;
+    public final Map<String, Double> scores;
+    public final Map<String, String> explanations;
+
+
+    public Entry(T item, String id) {
+        this.item = item;
+        this.id = id;
+        this.scores = new HashMap<String, Double>();
+        this.explanations = new HashMap<String, String>();
+    }
+
+    public Entry(Entry<T> other) {
+        this.item = other.item;
+        this.id = other.id;
+        this.scores = other.scores;
+        this.explanations = other.explanations;
+    }
+
+    public void addScore(String name, double score) {
+        scores.put(name, score);
+    }
+
+    public void addExplanation(String name, String explanation) {
+        explanations.put(name, explanation);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Queue.java

@@ -0,0 +1,56 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import java.util.*;
+
+/**
+ * An abstract data type represents a queue of certain item.
+ *   一个队列内召回Items完整list
+ */
+public class Queue<T> implements Iterable<Entry<T>> {
+    private final List<Entry<T>> entries;
+    private final String name;
+    private final String explain;
+
+    public Queue(String name) {
+        entries = new ArrayList<Entry<T>>();
+        this.name = name;
+        this.explain = null;
+    }
+
+    public Queue(String name, String explain) {
+        this.name = name;
+        this.explain = explain;
+        this.entries = new ArrayList<Entry<T>>();
+    }
+
+    public void reverse() {
+        Collections.reverse(this.entries);
+    }
+
+    public void add(Entry<T> entry) {
+        entries.add(entry);
+    }
+
+    public void addAll(Collection<Entry<T>> paramEntries) {
+        entries.addAll(paramEntries);
+    }
+
+    public List<Entry<T>> get() {
+        return entries;
+    }
+
+    public Iterator<Entry<T>> iterator() {
+        return entries.iterator();
+    }
+
+    public int size() {
+        return entries.size();
+    }
+
+    public void limit(int num) {
+        if (!entries.isEmpty() && num >= 0 && num < entries.size()) {
+            entries.subList(num, entries.size()).clear();
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/QueueName.java

@@ -0,0 +1,114 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.FluentIterable;
+import lombok.Data;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Queue names.召回队列命名处理
+ */
+@Data
+public class QueueName implements Serializable, Comparable<QueueName> {
+    private static final Function<Pair<String, String>, String> MATCH_PATTERN =
+            new Function<Pair<String, String>, String>() {
+                @Override
+                public String apply(Pair<String, String> input) {
+                    return input.getLeft() + "=" + input.getRight();
+                }
+            };
+    public static final long DEFAULT_LOCAL_CACHE_TTL = 15 * 60 * 1000;
+    private final List<Pair<String, String>> matches = new ArrayList<Pair<String, String>>();
+    private final String itemType;
+    private final String ordering;
+    private final long ttl;
+
+    private String metaChannel;  // meta 渠道
+
+    public long getTTL(){
+        return this.ttl;
+    }
+
+    public QueueName(String itemType, String ordering) {
+        this(itemType, ordering, DEFAULT_LOCAL_CACHE_TTL);
+    }
+
+    public QueueName(String itemType, String ordering, long ttl) {
+        this.itemType = itemType;
+        this.ordering = ordering;
+        this.ttl = ttl;
+    }
+
+    public static QueueName fromString(String string, long ttl) {
+        String[] parts = string.split(":");
+        String itemType = parts[0];
+        String ordering = parts[parts.length - 1].split("=")[1];
+
+        QueueName result = new QueueName(itemType, ordering, ttl);
+        for (int i = 2; i < parts.length - 1; ++i) {
+            String[] sides = parts[i].split("=");
+            result.addMatch(sides[0], sides[1]);
+        }
+        return result;
+    }
+
+    public static QueueName fromString(String string) {
+        return QueueName.fromString(string, DEFAULT_LOCAL_CACHE_TTL);
+    }
+
+    public static Pair<String, String> parseMatchPair(String match) {
+        String[] arr = match.split("=");
+        if (arr.length == 2) {
+            return Pair.of(arr[0], arr[1]);
+        }
+        return Pair.of("", "");
+    }
+
+    public QueueName addMatch(String key, String value) {
+        if (value == null || value.equals("")) {
+            value = "_";
+        } else {
+            value = value.replace("=", "_");
+            value = value.replace(":", "_");
+        }
+        if (key.equals("channel")) {
+            this.setMetaChannel(value);
+        }
+        matches.add(Pair.of(key, value));
+        return this;
+    }
+
+    public Iterable<String> getMatches() {
+        return FluentIterable.from(matches).transform(MATCH_PATTERN);
+    }
+
+
+    @Override
+    public String toString() {
+        Iterable<String> matchesString = FluentIterable.from(matches)
+                .transform(MATCH_PATTERN);
+
+        return itemType + ":queue:" + Joiner.on(":").join(matchesString) + ":ordering=" + ordering;
+    }
+
+    @Override
+    public int hashCode() {
+        return toString().hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return other instanceof QueueName && ((QueueName) other).toString().equals(toString());
+    }
+
+    @Override
+    public int compareTo(QueueName other) {
+        return other.toString().compareTo(ordering.toString());
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/User.java

@@ -0,0 +1,52 @@
+package com.tzld.piaoquan.recommend.server.framework.common;
+
+import lombok.Data;
+
+@Data
+public class User {
+
+    private String id;
+    private String uid;
+    private String mid;
+    private UserAttention userAttention;
+
+    // user profile 基础画像信息
+    private String machine_model;
+    private String machine_brand;
+    private String sdk;
+    private String gender;
+    private String ifCreater;
+    private String userType;
+    private String region;
+    private String city;
+
+    // user-content info 用户内容tag统计数据
+    private UserAction last1monthVideoTags;
+    private UserAction last7dayVideoTags;
+    private UserAction last1dayVideoTags;
+    private UserAction lastSessionVideoTags;
+
+    private UserAction last1monthTitleTags;
+    private UserAction last7dayTitleTags;
+    private UserAction last1dayTitleTags;
+    private UserAction lastSessionTitleTags;
+
+    // 用户关注up主信息
+    private UserAction last1monthPublishers;
+
+    // user-action info 用户
+    private UserAction last1monthUserAction;
+    private UserAction last7dayUserAction;
+    private UserAction last1dayUserAction;
+    private UserAction last1hourUserAction;
+    private UserAction lastSessionUserAction;
+
+    // user-group info
+    private String userGroup;
+
+    // user-vector
+    private String userVector;
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/UserAction.java

@@ -0,0 +1,21 @@
+package com.tzld.piaoquan.recommend.server.framework.common;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class UserAction {
+    //记录用户行为中 itemids，tags等信息
+
+    private List<String> followed;
+    private List<String> follow_and_realplay;
+    private List<String> played;
+    private List<String> shared;
+
+    private List<String> feedsPlayed;
+    private List<String> realPlayed;
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/UserAttention.java

@@ -0,0 +1,26 @@
+package com.tzld.piaoquan.recommend.server.framework.common;
+
+public class UserAttention {
+    /**
+     分维度的用户特征数据, 包含：
+     1、分不同时间维度的ItemList
+     30天、7天、1天、1H、实时，可以提供给I2I类召回策略用
+
+     2、分不同时间维度的categoryList
+     30天、7天、1天、1H、实时，可以提供给CB类召回策略用
+
+     3、用户group信息
+     天级别更新数据，可以提供给usergroup -> Item召回策略用。
+
+     4、分时间维度的发布者List
+     30天、7天、1天、1H、实时，可以提供 发布者 -> Item类召回用
+
+     */
+
+
+
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/IndexCandidateQueue.java

@@ -0,0 +1,88 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * 定义召回下子队列的Queue 融合信息
+ */
+public abstract class IndexCandidateQueue extends StrategyQueue {
+
+    protected Map<String, Candidate> myCandidates = new HashMap<String, Candidate>();
+
+    public IndexCandidateQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    public abstract int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId);
+
+
+    @Override
+    public int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        // TODO: sublist by every queue
+        return MergeUtils.simpleMergeWithProtection(items, rankerItemsListMap, recNum, expId);
+    }
+
+    @Override
+    public int candidate(final Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        myCandidates.clear();
+        int n = addCandidateKey(candidateMap, recallNum, user, requestData, requestIndex, expId);
+        candidateMap.putAll(myCandidates);
+
+        return n;
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, String key, int num, String queue) {
+        if (candidates.containsKey(key)) {
+            return 0;
+        } else {
+            Candidate candidate = new Candidate();
+            candidate.setCandidateKey(key);
+            candidate.setCandidateNum(num);
+            candidate.setMergeQueueName(queue);
+            candidates.put(key, candidate);
+            return num;
+        }
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue) {
+        return this.addCandidateKey(candidates, queueName, num, queue, true, true, true);
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue, boolean needScore, boolean needPornRank) {
+        return this.addCandidateKey(candidates, queueName, num, queue, needScore, needPornRank, true);
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue, boolean needScore, boolean needPornRank, boolean needExposeHistoryFilter) {
+        if (candidates.containsKey(queueName.toString())) {
+            return 0;
+        } else {
+            Candidate candidate = new Candidate();
+            candidate.setCandidateKey(queueName.toString());
+            candidate.setCandidateNum(num);
+            candidate.setMergeQueueName(queue);
+            candidates.put(queueName.toString(), candidate);
+            return num;
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "IndexCandidateQueue{" +
+                "myCandidates=" + myCandidates +
+                ", rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/MergeRule.java

@@ -0,0 +1,45 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * Merge 规则配置
+ */
+@Data
+public class MergeRule {
+    public String queueName;
+
+    public Set<Integer> enableExpIdSet = null;
+    public Set<Integer> disableExpIdSet = null;
+
+    public double recallPercentage = 0.1; // 召回百分比, 默认占 10%
+    public int minMergeNum = 0; // 合并保护规则: 最小合并条数
+    public int maxMergeNum = 100; // 合并保护规则: 最大合并条数
+
+    private Map<String, String> properties;
+
+    public MergeRule() {
+        this.properties = new HashMap<String, String>();
+    }
+
+    public boolean isDisabled(final int expId) {
+        if (enableExpIdSet != null && !enableExpIdSet.contains(expId)) {
+            return true;
+        }
+        if (disableExpIdSet != null && disableExpIdSet.contains(expId)) {
+            return true;
+        }
+        return false;
+    }
+
+    public void putProperty(String key, String value) {
+        this.properties.put(key, value);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/MergeUtils.java

@@ -0,0 +1,322 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.google.common.base.Function;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.CandidateInfo;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.*;
+
+
+public class MergeUtils {
+    public static final Logger LOGGER = LoggerFactory.getLogger(MergeUtils.class);
+
+    // mergeConfFile_topQueueName 对应的 StrategyQueue 缓存
+    private static final Map<String, Config> queueConfigMap = new HashMap<>();
+
+    public static StrategyQueue createTopQueue(String mergeConfFile, String topQueueName) {
+        String key = mergeConfFile + "_" + topQueueName;
+        Config mergeQueueConf;
+        if (queueConfigMap.containsKey(key)) {
+            mergeQueueConf = queueConfigMap.get(key);
+        } else {
+            mergeQueueConf = ConfigFactory.parseResources(mergeConfFile);
+            queueConfigMap.put(key, mergeQueueConf);
+        }
+        StrategyQueueConfig strategyQueueConfig = new StrategyQueueConfig();
+        if (strategyQueueConfig.load(mergeQueueConf)) {
+            LOGGER.debug("Merger config init succ");
+        } else {
+            LOGGER.error("Merge config init failed: init MergeBiz using queue config {}", mergeConfFile);
+        }
+
+        StrategyQueue topQueue = null;
+        try {
+            topQueue = MergeUtils.constructStrategyQueue(topQueueName, strategyQueueConfig);
+        } catch (InstantiationException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (IllegalAccessException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (ClassNotFoundException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (NoSuchMethodException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (InvocationTargetException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        }
+        return topQueue;
+    }
+
+    public static StrategyQueue constructStrategyQueue(String queueName, StrategyQueueConfig strategyQueueConfig) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
+        if (!strategyQueueConfig.getStrategyQueueInfoMap().containsKey(queueName)) {
+            return null;
+        }
+
+        StrategyQueueInfo strategyQueueInfo = strategyQueueConfig.getStrategyQueueInfoMap().get(queueName);
+        StrategyQueue strategyQueue = (StrategyQueue) Class.forName(strategyQueueInfo.getQueueClass())
+                .getConstructor(StrategyQueueInfo.class, strategyQueueConfig.getClass())
+                .newInstance(strategyQueueInfo, strategyQueueConfig);
+
+        return strategyQueue;
+    }
+
+
+    /**
+     * 分发items到策略树中
+     * 分发的过程中不要破坏相对顺序
+     *
+     * @param strategyQueue
+     * @param items
+     */
+    public static void distributeItemsToMultiQueues(StrategyQueue strategyQueue, List<RankItem> items) {
+        List<StrategyQueue> strategyQueueList = strategyQueue.getAllQueues();
+
+        Multimap<String, RankItem> mergeQueuesItems = ArrayListMultimap.create();
+        for (RankItem item : items) {
+            for (CandidateInfo candidateInfo : item.getCandidateInfoList()) {
+                String mergeQueue = candidateInfo.getCandidate().getMergeQueueName();
+                RankItem currentItem = item.deepcopy();
+
+                // set candidate info
+                currentItem.setQueue(mergeQueue);
+                currentItem.setCandidateInfo(candidateInfo);
+
+                mergeQueuesItems.put(mergeQueue, currentItem);
+            }
+        }
+//         debug log
+//        for (Map.Entry<String, RankItem> entry : mergeQueuesItems.entries()) {
+//            LOGGER.info("after distribute item queue info: entry: [{}]", entry);
+//        }
+        for (StrategyQueue queue : strategyQueueList) {
+            String mergeQueueName = queue.getStrategyQueueInfo().getQueueName();
+            if (mergeQueuesItems.containsKey(mergeQueueName)) {
+                List<RankItem> currMergeQueueItems = new ArrayList<RankItem>(mergeQueuesItems.get(mergeQueueName));
+                queue.setItems(currMergeQueueItems);
+            }
+        }
+    }
+
+
+
+
+    /**
+     * 基于 score 字段融合: score 较大的 Item 优先
+     *
+     * @param resultRankerItems
+     * @param rankerItemsListMap
+     * @param freeRecNum
+     * @param expId
+     */
+    public static void simpleMergeByScore(List<RankItem> resultRankerItems,
+                                          final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap,
+                                          int freeRecNum,
+                                          int expId) {
+        // 先定义按score排序优选的独立额
+        PriorityQueue<Pair<String, Integer>> mergePriorityQueue = new PriorityQueue<Pair<String, Integer>>(freeRecNum, new Comparator<Pair<String, Integer>>() {
+            @Override
+            public int compare(Pair<String, Integer> o1, Pair<String, Integer> o2) {
+                return rankerItemsListMap.get(o1.getLeft()).getRight().get(o1.getRight()).compareTo(rankerItemsListMap.get(o2.getLeft()).getRight().get(o2.getRight()));
+            }
+        });
+
+        // 大于最小mergenum 同时小于maxMergenum，在队列中add
+        for (Pair<MergeRule, List<RankItem>> entry : rankerItemsListMap.values()) {
+            if (entry.getLeft().isDisabled(expId) || entry.getRight() == null || entry.getRight().isEmpty()) {
+                continue;
+            }
+
+            MergeRule myRule = entry.getLeft();
+            if (myRule.minMergeNum < myRule.maxMergeNum && entry.getRight().size() > myRule.minMergeNum) {
+                Collections.sort(entry.getRight().subList(myRule.minMergeNum, entry.getRight().size()));
+                mergePriorityQueue.add(Pair.of(myRule.queueName, myRule.minMergeNum));
+            }
+        }
+        //
+        while (freeRecNum > 0 && !mergePriorityQueue.isEmpty()) {
+            Pair<String, Integer> item = mergePriorityQueue.poll();
+            String myName = item.getLeft();
+            int myIndex = item.getRight();
+
+            resultRankerItems.add(rankerItemsListMap.get(myName).getRight().get(myIndex));
+            resultRankerItems.get(resultRankerItems.size() - 1).addMergeDecisionLabel("score:" + myName);
+            freeRecNum -= 1;
+            myIndex += 1;
+
+            if (freeRecNum > 0 && rankerItemsListMap.get(myName).getRight().size() > myIndex && myIndex < rankerItemsListMap.get(myName).getLeft().maxMergeNum) {
+                mergePriorityQueue.add(Pair.of(myName, myIndex));
+            }
+        }
+    }
+
+    /**
+     * 队列融合:
+     * 1. 各队列基于排序不变;
+     * 2. 各队列保证最小合并数量 (minMergeNum) 条目;
+     * 3. 各队列限制最大合并数量 (maxMergeNum) 条目;
+     * 4. 从各队列取 Score 最高条目.
+     *
+     * @param rankerItemsListMap
+     * @param recNum
+     * @param expId
+     * @return
+     */
+    public static int simpleMergeWithProtection(List<RankItem> resultRankerItems,
+                                                final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap,
+                                                int recNum,
+                                                int expId) {
+        if (resultRankerItems == null) {
+            resultRankerItems = new LinkedList<RankItem>();
+        }
+        // 保证各队列最小合并条目后剩余可自由调配条目数量
+        int freeRecNum = recNum;
+        for (Pair<MergeRule, List<RankItem>> entry : rankerItemsListMap.values()) {
+
+            if (entry.getLeft().isDisabled(expId) || entry.getRight() == null || entry.getRight().isEmpty()) {
+                continue;
+            }
+
+            MergeRule myRule = entry.getLeft();
+            int mySize = entry.getRight().size();
+            int myMinMergeNum = Math.min(myRule.minMergeNum, mySize);
+            freeRecNum -= myMinMergeNum;
+
+            int prevSize = resultRankerItems.size();
+            // 先保证最小长度添加
+            resultRankerItems.addAll(entry.getRight().subList(0, myMinMergeNum));
+
+            int afterSize = resultRankerItems.size();
+            for (int ix = prevSize; ix < afterSize; ++ix) {
+                resultRankerItems.get(ix).addMergeDecisionLabel("min_protect:" + myRule.queueName);
+            }
+        }
+
+        if (freeRecNum > 0) {
+            //保证最小长度后，其他按score融合
+            simpleMergeByScore(resultRankerItems, rankerItemsListMap, freeRecNum, expId);
+        }
+
+        return resultRankerItems.size();
+    }
+
+    /**
+     * 按优先级队列融合:
+     * 1. 各队列基于 priority 依次融合: 各队列保证最小合并数量 (minMergeNum) 条目;
+     * 2. 若没有凑足 recNum, 各队列限制最大合并数量 (maxMergeNum) 条目条件下按 score 优先选择;
+     *
+     * @param rankerItemsListMap
+     * @param recNum
+     * @param user
+     * @param requestData
+     * @param requestIndex
+     * @param expId
+     * @return
+     */
+    public static int simpleMergeByPriority(List<RankItem> resultRankerItems, final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        if (resultRankerItems == null) {
+            resultRankerItems = new LinkedList<RankItem>();
+        }
+
+        final List<Map.Entry<String, Pair<MergeRule, List<RankItem>>>> rankerItemsList = new LinkedList<Map.Entry<String, Pair<MergeRule, List<RankItem>>>>(rankerItemsListMap.entrySet());
+
+        // 保证各队列最小合并条目后剩余可自由调配条目数量
+        Collections.sort(rankerItemsList, new Comparator<Map.Entry<String, Pair<MergeRule, List<RankItem>>>>() {
+            private final int DEFAULT_PRIORITY = 0;
+
+            @Override
+            public int compare(Map.Entry<String, Pair<MergeRule, List<RankItem>>> o1, Map.Entry<String, Pair<MergeRule, List<RankItem>>> o2) {
+                int p1 = getPriority(o1.getValue().getLeft()), p2 = getPriority(o2.getValue().getLeft());
+                return p1 - p2;
+            }
+
+            private int getPriority(MergeRule rule) {
+                if (rule.getProperties().containsKey("priority")) {
+                    return Integer.valueOf(rule.getProperties().get("priority"));
+                }
+                return DEFAULT_PRIORITY;
+            }
+        });
+
+        int freeRecNum = recNum;
+        for (int i = 0; i < rankerItemsList.size(); ++i) {
+            if (rankerItemsList.get(i).getValue().getLeft().isDisabled(expId) || rankerItemsList.get(i).getValue().getRight() == null || rankerItemsList.get(i).getValue().getRight().isEmpty()) {
+                continue;
+            }
+
+            MergeRule myRule = rankerItemsList.get(i).getValue().getLeft();
+            int mySize = rankerItemsList.get(i).getValue().getRight().size();
+            int myMinMergeNum = Math.min(myRule.minMergeNum, mySize);
+            freeRecNum -= myMinMergeNum;
+
+            int prevSize = resultRankerItems.size();
+            resultRankerItems.addAll(rankerItemsList.get(i).getValue().getRight().subList(0, myMinMergeNum));
+            if (mySize > myMinMergeNum) {
+                Collections.sort(rankerItemsList.get(i).getValue().getRight().subList(myMinMergeNum, mySize));
+            }
+            int afterSize = resultRankerItems.size();
+            for (int ix = prevSize; ix < afterSize; ++ix) {
+                resultRankerItems.get(ix).addMergeDecisionLabel("min_protect:" + myRule.queueName);
+            }
+        }
+
+        if (freeRecNum > 0) {
+            simpleMergeByScore(resultRankerItems, rankerItemsListMap, freeRecNum, expId);
+        }
+
+        return resultRankerItems.size();
+    }
+
+    /**
+     * 基于 tag 和 category 重排序, 保证具有相同的分类或标签的 Item 在 windowSize 窗口内只有 similarLimit 条
+     *
+     * @param items
+     * @param expectedRecommendItemsCount
+     * @param windowSize
+     * @param similarLimit
+     */
+    public static void diversityRerank(List<RankItem> items, Function<Pair<RankItem, RankItem>, Boolean> similarFunc, int expectedRecommendItemsCount, final int windowSize, final int similarLimit) {
+        List<RankItem> rerankedItems = new LinkedList<RankItem>();
+        expectedRecommendItemsCount = Math.min(expectedRecommendItemsCount, items.size());
+        for (int ix = 0; ix < expectedRecommendItemsCount; ++ix) {
+            int windowStartPosition = Math.max(ix - windowSize + 1, 0);
+            int nextId = 0;
+            while (nextId < items.size()) {
+                int similarCount = 0;
+                for (int jx = windowStartPosition; jx < ix && similarCount < similarLimit; ++jx) {
+                    if (similarFunc.apply(Pair.of(items.get(nextId), rerankedItems.get(jx)))) {
+                        similarCount += 1;
+                    }
+                }
+                if (similarCount < similarLimit) {
+                    break;
+                }
+                nextId += 1;
+            }
+            if (nextId >= items.size()) {
+                nextId = 0;
+            }
+
+            RankItem nextItem = items.get(nextId);
+            nextItem.addMergeDecisionLabel("sim_rerank:" + windowStartPosition + "_" + windowSize);
+            items.remove(nextId);
+            rerankedItems.add(nextItem);
+        }
+
+        if (items.size() > 0) {
+            rerankedItems.addAll(items);
+        }
+
+        items.clear();
+        items.addAll(rerankedItems);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/SimilarityUtils.java

@@ -0,0 +1,116 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.google.common.collect.Sets;
+import com.google.common.base.Function;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import org.apache.commons.lang3.tuple.Pair;
+
+import javax.annotation.Nullable;
+
+
+public class SimilarityUtils {
+
+    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameUserTagOrCategoryFunc() {
+        return new Function<Pair<RankItem, RankItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
+                return isSameCategory(itemPair.getLeft(), itemPair.getRight()) || isSameUserTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameTitleTagFunc() {
+        return new Function<Pair<RankItem, RankItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
+                return isSameTitleTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameUserTagFunc() {
+        return new Function<Pair<RankItem, RankItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
+                return isSameUserTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameCategoryFunc() {
+        return new Function<Pair<RankItem, RankItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
+                return isSameCategory(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankItem, RankItem>, Boolean> getIsUserTagFunc() {
+        return new Function<Pair<RankItem, RankItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
+                return isSameUserTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static boolean isSameCategory(RankItem rankerItem1, RankItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getRankItemCategories() == null ||
+                rankerItem2.getRankItemCategories() == null) {
+            return false;
+        }
+
+        return !Sets.intersection(rankerItem1.getRankItemCategories().keySet(),
+                        rankerItem2.getRankItemCategories().keySet())
+                .isEmpty();
+    }
+
+    public static boolean isSameTitleTag(RankItem rankerItem1, RankItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getRankItemTitleTags() == null ||
+                rankerItem2.getRankItemTitleTags() == null) {
+            return false;
+        }
+        return !Sets.intersection(rankerItem1.getRankItemTitleTags().keySet(),
+                        rankerItem2.getRankItemTitleTags().keySet())
+                .isEmpty();
+    }
+
+    public static boolean isSameUserTag(RankItem rankerItem1, RankItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getRankItemUserTags() == null ||
+                rankerItem2.getRankItemUserTags() == null) {
+            return false;
+        }
+        return !Sets.intersection(rankerItem1.getRankItemUserTags().keySet(),
+                        rankerItem2.getRankItemUserTags().keySet())
+                .isEmpty();
+    }
+
+    public static boolean isSameQueue(RankItem rankerItem1, RankItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getQueue() == null) {
+            return false;
+        }
+        return rankerItem1.getQueue().equals(rankerItem2.getQueue());
+    }
+
+    public static boolean isSameSubCategory(RankItem item1, RankItem item2) {
+        if (item1 == null || item2 == null || item1.getRankItemSubCategories() == null
+                || item2.getRankItemSubCategories() == null) {
+            return false;
+        }
+
+        return !Sets.intersection(item1.getRankItemSubCategories().keySet(),
+                        item2.getRankItemSubCategories().keySet())
+                .isEmpty();
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/SimpleMergeQueue.java

@@ -0,0 +1,36 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.*;
+
+
+public class SimpleMergeQueue extends StrategyQueue {
+    public SimpleMergeQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        return MergeUtils.simpleMergeWithProtection(items, rankerItemsListMap, recNum, expId);
+    }
+
+    @Override
+    public int candidate(Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        return simpleCandidateByPercentage(candidateMap, recallNum, user, requestData, requestIndex, expId);
+    }
+
+    @Override
+    public String toString() {
+        return "SimpleMergeQueue{" +
+                "rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueue.java

@@ -0,0 +1,202 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.google.common.collect.Maps;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import lombok.Data;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.*;
+
+@Data
+public abstract class StrategyQueue {
+    // 得等召回结果，定义为RankItem
+    protected final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsList = new HashMap<>();
+    private final Logger LOGGER = LoggerFactory.getLogger(StrategyQueue.class);
+    protected Map<String, StrategyQueue> children = Maps.newHashMap();
+
+    protected List<RankItem> items = new ArrayList<>();
+    private StrategyQueueInfo strategyQueueInfo;
+    private List<StrategyQueue> allQueues = null;
+
+    public StrategyQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        this.strategyQueueInfo = strategyQueueInfo;
+
+        // 自顶向下构建child queue 初始化
+        for (String childName : strategyQueueInfo.getChildren()) {
+            if (!strategyQueueConfig.getStrategyQueueInfoMap().containsKey(childName)) {
+                LOGGER.error("No config for Child queue [{}]", childName);
+                continue;
+            }
+            StrategyQueueInfo childQueueInfo = strategyQueueConfig.getStrategyQueueInfoMap().get(childName);
+
+            StrategyQueue childQueue = null;
+            String childQueueClass = childQueueInfo.getQueueClass();
+            try {
+                childQueue = (StrategyQueue) Class.forName(childQueueClass)
+                        .getConstructor(childQueueInfo.getClass(), strategyQueueConfig.getClass())
+                        .newInstance(childQueueInfo, strategyQueueConfig);
+            } catch (InstantiationException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (IllegalAccessException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (ClassNotFoundException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (NoSuchMethodException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (InvocationTargetException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            }
+            if (childQueue != null) {
+                putChild(childName, childQueue);
+            } else {
+                LOGGER.error("construct child queue [{}] failed", childName);
+            }
+        }
+
+    }
+
+    public List<StrategyQueue> getAllQueues() {
+        if (allQueues == null) {
+            allQueues = new LinkedList<>();
+
+            if (!strategyQueueInfo.isLeaf()) {
+                for (Map.Entry<String, StrategyQueue> child : children.entrySet()) {
+                    allQueues.addAll(child.getValue().getAllQueues());
+                }
+            }
+
+            allQueues.add(this);
+        }
+        return allQueues;
+    }
+
+
+    private void putChild(String queueName, StrategyQueue child) {
+        children.put(queueName, child);
+    }
+
+
+    public void clearItems() {
+        items.clear();
+    }
+
+    public void addItem(RankItem item) {
+        this.items.add(item);
+    }
+
+    public abstract int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId);
+
+    // 定义索引Key 生成方式
+    public abstract int candidate(Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId);
+
+    public final int merge(final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        this.beforeMerge(recNum, user, requestData, requestIndex, expId);
+        this.doMerge(rankerItemsList, recNum, user, requestData, requestIndex, expId);
+        this.doDeDup();
+        return this.afterMerge();
+    }
+
+    // Merge 前从每个Queue中获取Item
+    // 放入RankerItem中
+    public void beforeMerge(final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        if (strategyQueueInfo.isLeaf()) {
+            LOGGER.debug("leaf queue [{}] with [{}] items to merge", getStrategyQueueInfo().getQueueName(), items.size());
+        } else {
+            rankerItemsList.clear();
+            clearItems();
+            int queueCount = 0, itemCount = 0;
+            for (MergeRule rule : strategyQueueInfo.getRulesList()) {
+                if (rule.isDisabled(expId)) {
+                    continue;
+                }
+                int myRecNum = Math.min(recNum, rule.maxMergeNum);
+                //递归的执行子节点的merge逻辑
+                int actualRecNum = children.get(rule.queueName).merge(myRecNum, user, requestData, requestIndex, expId);
+                queueCount += 1;
+                itemCount += actualRecNum;
+                rankerItemsList.put(rule.queueName, Pair.of(rule, children.get(rule.queueName).getItems()));
+            }
+            LOGGER.debug(getStrategyQueueInfo().getQueueName() + " merge info: [{}] queues with [{}] items to merge", queueCount, itemCount);
+        }
+    }
+
+    /**
+     * id dedup 去重操作
+     *
+     * @return
+     */
+    public final int doDeDup() {
+        List<RankItem> pureItems = new ArrayList<RankItem>();
+        Set<String> deDupItems = new HashSet<String>();
+        int deDupCount = 0;
+        for (RankItem item : items) {
+            if (deDupItems.contains(item.getVideoId())) {
+                if (LOGGER.isDebugEnabled()) {
+                    LOGGER.debug("merge queue dedup item, queue [{}] itemid [{}]",
+                            strategyQueueInfo.getQueueName(), item.getVideoId());
+                }
+                deDupCount++;
+                continue;
+            }
+            deDupItems.add(item.getId());
+            pureItems.add(item);
+        }
+        items = pureItems;
+        LOGGER.debug("dedup items, queue [{}], dedup number [{}], current number [{}]",
+                new Object[]{getStrategyQueueInfo().getQueueName(), deDupCount, items.size()});
+        return items.size();
+    }
+
+    // 给item增加融合队列名称
+    public int afterMerge() {
+        int n = items.size();
+        for (RankItem item : items) {
+            item.addMergeQueuePath(getStrategyQueueInfo().getQueueName());
+        }
+        LOGGER.debug("number of items after [{}] merge: [{}]", getStrategyQueueInfo().getQueueName(), n);
+        return n;
+    }
+
+    // 读取子队列的merge规则，按队列生成candidate
+    protected int simpleCandidateByPercentage(Map<String, Candidate> candidateMap, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int n = 0;
+
+        for (MergeRule rule : strategyQueueInfo.getRulesList()) {
+            if (rule.isDisabled(expId)) {
+                continue;
+            }
+
+            int myRecallNum = (int) (recallNum * rule.recallPercentage);
+            if (!children.containsKey(rule.queueName)) {
+                LOGGER.error("Rule for not exist child queue [{}]", rule.queueName);
+            }
+
+            Map<String, Candidate> myCandidates = new HashMap<>();
+            StrategyQueue strategyQueue = children.get(rule.queueName);
+            if (strategyQueue != null) {
+                n += strategyQueue.candidate(myCandidates, myRecallNum, user, requestData, requestIndex, expId);
+            }
+            candidateMap.putAll(myCandidates);
+        }
+
+        return n;
+    }
+
+    @Override
+    public String toString() {
+        return "StrategyQueue{" +
+                "rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                ", strategyQueueInfo=" + strategyQueueInfo +
+                ", allQueues=" + allQueues +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueueConfig.java

@@ -0,0 +1,232 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import lombok.Data;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.*;
+
+@Data
+public class StrategyQueueConfig {
+    private final Logger LOGGER = LoggerFactory.getLogger(StrategyQueueConfig.class);
+
+    private static final String CHILDREN_STRING = "children";
+    private static final String STRATEGY_QUEUE_CLASS_CONF_STRING = "class";
+
+    public static final String DEFAULT_STRATEGY_CLASS_NAME = "com.tzld.piaoquan.recommend.server.framework.merger.SimpleMergeQueue";
+
+    private static final String MERGE_RULE_STRING = "merge-rule";
+    private static final String QUEUE_NAME_STRING = "queue-name";
+    private static final String RECALL_PERCENTAGE_STRING = "recall-percentage";
+    private static final String ENABLE_EXPERIMENT_LIST_STRING = "enable-exp";
+    private static final String DISABLE_EXPERIMENT_LIST_STRING = "disable-exp";
+    private static final String MIN_MERGE_NUM_STRING = "min-merge-num";
+    private static final String MAX_MERGE_NUM_STRING = "max-merge-num";
+
+    private static final Set<String> RESERVED_PROPERTIES = new HashSet<String>(Arrays.asList(QUEUE_NAME_STRING, RECALL_PERCENTAGE_STRING, ENABLE_EXPERIMENT_LIST_STRING, DISABLE_EXPERIMENT_LIST_STRING, MIN_MERGE_NUM_STRING, MAX_MERGE_NUM_STRING));
+
+    Map<String, StrategyQueueInfo> strategyQueueInfoMap;
+
+
+    public boolean load(Config config) {
+        config.checkValid(config, "queue-config");
+        Config queueConf = config.getConfig("queue-config");
+        if (loadMergeQueues(queueConf)) {
+            LOGGER.debug("Load queues config success");
+        } else {
+            LOGGER.error("Load queues config failed");
+            return false;
+        }
+
+        config.checkValid(config, "rule-config");
+        Config rulesConf = config.getConfig("rule-config");
+        if (loadMergeRules(rulesConf)) {
+            LOGGER.debug("Load rules config success");
+        } else {
+            LOGGER.error("Load rules config failed");
+            return false;
+        }
+        // TODO: validate queue structure & rules
+        for (String queueName : strategyQueueInfoMap.keySet()) {
+            StrategyQueue queue = null;
+            try {
+                queue = MergeUtils.constructStrategyQueue(queueName, this);
+            } catch (ClassNotFoundException e) {
+                e.printStackTrace();
+            } catch (NoSuchMethodException e) {
+                e.printStackTrace();
+            } catch (IllegalAccessException e) {
+                e.printStackTrace();
+            } catch (InvocationTargetException e) {
+                e.printStackTrace();
+            } catch (InstantiationException e) {
+                e.printStackTrace();
+            }
+
+            if (queue == null) {
+                LOGGER.error("Queue config info & rules error");
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    public boolean load(String configFile) {
+        Config queueConfig = ConfigFactory.parseResources(configFile);
+        return load(queueConfig);
+    }
+
+    private boolean loadMergeQueues(Config queueConf) {
+        strategyQueueInfoMap = new HashMap<String, StrategyQueueInfo>();
+
+        ConfigObject confObj = queueConf.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            String myName = it.getKey();
+            Config myConf = ((ConfigObject) it.getValue()).toConfig();
+            if (loadMergeQueueInfo(myName, myConf)) {
+                LOGGER.debug("Load queue [{}] info success", myName);
+            } else {
+                LOGGER.error("Load queue [{}] info failed", myName);
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean loadMergeQueueInfo(String name, Config conf) {
+        LOGGER.debug("Start load queue [{}]", name);
+        if (!strategyQueueInfoMap.containsKey(name)) {
+            strategyQueueInfoMap.put(name, new StrategyQueueInfo(name));
+        }
+        StrategyQueueInfo mergeQueueInfo = strategyQueueInfoMap.get(name);
+
+        if (conf.hasPath(STRATEGY_QUEUE_CLASS_CONF_STRING)) {
+            mergeQueueInfo.setQueueClass(conf.getString(STRATEGY_QUEUE_CLASS_CONF_STRING));
+        } else {
+            LOGGER.info("No Class config was found for queue [{}], use default class [{}]", name, DEFAULT_STRATEGY_CLASS_NAME);
+            mergeQueueInfo.setQueueClass(DEFAULT_STRATEGY_CLASS_NAME);
+        }
+
+        // 递归的把child节点放在queue中
+        if (conf.hasPath(CHILDREN_STRING)) {
+            ConfigObject childrenConf = conf.getObject(CHILDREN_STRING);
+            for (ConfigObject.Entry<String, ConfigValue> it : childrenConf.entrySet()) {
+                String key = it.getKey();
+                Config childConf = ((ConfigObject) it.getValue()).toConfig();
+                String childName = key;
+
+                boolean isLoadSuccess = loadMergeQueueInfo(childName, childConf);
+                if (isLoadSuccess) {
+                    LOGGER.debug("Load queue [{}] SUCCESS", childName);
+                } else {
+                    LOGGER.error("Load queue [{}] FAILED: [{}]", childName, it.getValue());
+                    return false;
+                }
+
+                mergeQueueInfo.addChild(childName);
+            }
+        } else {
+            mergeQueueInfo.setLeaf(true);
+        }
+
+        LOGGER.debug("Load queue [{}] success", name);
+        return true;
+    }
+
+    private boolean loadMergeRules(Config conf) {
+        ConfigObject confObj = conf.root();
+
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            String myName = it.getKey();
+            Config subTreeConf = ((ConfigObject) it.getValue()).toConfig();
+            boolean isParseSuccess = parseMergeRule(myName, subTreeConf);
+            if (isParseSuccess) {
+                LOGGER.debug("Parse rules for queue [{}] SUCCESS", myName);
+            } else {
+                LOGGER.error("Parse rules for queue [{}] FAILED: [{}]", myName, it.getValue());
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private boolean parseMergeRule(String name, Config ruleConf) {
+        if (!strategyQueueInfoMap.containsKey(name)) {
+            LOGGER.error("skip rules for not exist queue [{}]", name);
+            return true;
+        } else {
+            LOGGER.debug("parse rules for queue [{}]", name);
+        }
+
+        if (!ruleConf.hasPath(MERGE_RULE_STRING)) {
+            LOGGER.debug("no merge rules for queue [{}]", name);
+            return true;
+        }
+
+        ConfigObject ruleConfObj = ruleConf.getObject(MERGE_RULE_STRING);
+
+        for (ConfigObject.Entry<String, ConfigValue> it : ruleConfObj.entrySet()) {
+            String ruleName = it.getKey();
+            MergeRule rule = new MergeRule();
+
+            Config myConf = ((ConfigObject) it.getValue()).toConfig();
+
+            rule.recallPercentage = getOrDefault(myConf, RECALL_PERCENTAGE_STRING, 0.1);
+            rule.minMergeNum = getOrDefault(myConf, MIN_MERGE_NUM_STRING, 0);
+            rule.maxMergeNum = getOrDefault(myConf, MAX_MERGE_NUM_STRING, 500);
+
+            // 子队列名规则: 用 queue-name 显式指定, 默认为对应规则名
+            rule.queueName = getOrDefault(myConf, QUEUE_NAME_STRING, ruleName);
+            if (myConf.hasPath(ENABLE_EXPERIMENT_LIST_STRING)) {
+                List<Integer> expIds = myConf.getIntList(ENABLE_EXPERIMENT_LIST_STRING);
+                rule.enableExpIdSet = new HashSet<Integer>(expIds);
+            }
+            if (myConf.hasPath(DISABLE_EXPERIMENT_LIST_STRING)) {
+                List<Integer> expIds = myConf.getIntList(DISABLE_EXPERIMENT_LIST_STRING);
+                rule.disableExpIdSet = new HashSet<Integer>(expIds);
+            }
+
+            for (ConfigObject.Entry<String, ConfigValue> properties : myConf.entrySet()) {
+                if (!RESERVED_PROPERTIES.contains(properties.getKey())) {
+                    rule.putProperty(properties.getKey(), myConf.getString(properties.getKey()));
+                }
+            }
+
+            if (strategyQueueInfoMap.get(name).getChildren().contains(rule.queueName)) {
+                strategyQueueInfoMap.get(name).addRule(rule);
+            } else {
+                LOGGER.info("Skip rule={} for not exist queue={}", ruleName, rule.queueName);
+            }
+        }
+        return true;
+    }
+
+    private int getOrDefault(Config conf, String path, int defaultValue) {
+        if (conf.hasPath(path)) {
+            return conf.getInt(path);
+        }
+        return defaultValue;
+    }
+
+    private String getOrDefault(Config conf, String path, String defaultValue) {
+        if (conf.hasPath(path)) {
+            return conf.getString(path);
+        }
+        return defaultValue;
+    }
+
+    private double getOrDefault(Config conf, String path, double defaultValue) {
+        if (conf.hasPath(path)) {
+            return conf.getDouble(path);
+        }
+        return defaultValue;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueueInfo.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import lombok.Data;
+
+import java.util.*;
+
+
+/**
+ * 每一个召回 or 队列的融合配置信息
+ */
+
+@Data
+public class StrategyQueueInfo {
+    private String queueName;
+
+    private boolean isLeaf = false;
+
+    private String queueClass; // Full path of StrategyQueue Class
+
+    private List<MergeRule> rulesList;
+
+    private Set<String> children;
+
+    public StrategyQueueInfo(String name) {
+        this.queueName = name;
+        rulesList = new LinkedList<MergeRule>();
+        children = new HashSet<String>();
+    }
+
+    public void addRule(MergeRule rule) {
+        this.rulesList.add(rule);
+    }
+
+    public void addChild(String childName) {
+        this.children.add(childName);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/GBDTModel.java

@@ -0,0 +1,196 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.*;
+
+
+
+public class GBDTModel extends Model {
+    private static final Logger LOGGER = LoggerFactory.getLogger(GBDTModel.class);
+    private static final float LINEAR_TRANSFORM_LOWER_BOUND = 0f;
+    private static final float LINEAR_TRANSFORM_SLOPE = 1f;
+    private int featureCount = 0;
+    private Map<String, Integer> featureIdMap = null;
+    private List<HashMap<Integer, Node>> boosterModel = null;
+
+    private float transform(final float score) {
+        return (score - LINEAR_TRANSFORM_LOWER_BOUND) * LINEAR_TRANSFORM_SLOPE;
+    }
+
+    private int getFeatureId(final String feature) {
+        return featureIdMap.containsKey(feature) ? featureIdMap.get(feature) : -1;
+    }
+
+    @Override
+    public int getModelSize() {
+        return boosterModel.size();
+    }
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws Exception {
+        List<HashMap<Integer, Node>> model = new ArrayList<HashMap<Integer, Node>>();
+        HashMap<Integer, Node> tree = null;
+
+        featureIdMap = new HashMap<String, Integer>();
+        featureCount = 0;
+
+        BufferedReader input = new BufferedReader(in);
+        String line;
+        while ((line = input.readLine()) != null) {
+            String[] tokens = line.trim().split(":");
+            if (tokens.length == 1) {
+                if (tokens[0].startsWith("booster")) {
+                    if (tree != null && tree.size() > 0) {
+                        model.add(tree);
+                    }
+
+                    tree = new HashMap<Integer, Node>();
+                }
+            } else if (tokens.length == 2) {
+                Node node = new Node();
+
+                Integer id = Integer.parseInt(tokens[0]);
+
+                String[] items = tokens[1].split(" ");
+                if (items.length == 1) {
+                    node.isLeaf = true;
+
+                    String[] dt = items[0].split("=");
+                    node.value = Float.parseFloat(dt[1]);
+                } else if (items.length == 2) {
+                    node.isLeaf = false;
+                    String[] fieldDescriptions = null;
+                    node.compareType = NodeCompareType.LT;
+                    if (items[0].substring(1, items[0].length() - 1).contains("<=")) {
+                        fieldDescriptions = items[0].substring(1, items[0].length() - 1).split("<=");
+                        node.compareType = NodeCompareType.LE;
+                    } else {
+                        fieldDescriptions = items[0].substring(1, items[0].length() - 1).split("<");
+                    }
+                    // feature to id
+                    int featureId = getFeatureId(fieldDescriptions[0]);
+                    if (featureId < 0) {
+                        featureIdMap.put(fieldDescriptions[0], featureCount);
+                        featureId = featureCount;
+                        featureCount += 1;
+                    }
+                    node.splitFeatureId = featureId;
+
+                    if (fieldDescriptions.length == 1) {
+                        node.fieldType = NodeFieldType.BINARY;
+                    } else {
+                        node.fieldType = NodeFieldType.QUANTITATIVE;
+                        node.splitCondition = Float.parseFloat(fieldDescriptions[1]);
+                    }
+
+                    String[] childrenDescriptions = items[1].split(",");
+                    for (String childDescription : childrenDescriptions) {
+                        String[] descs = childDescription.split("=");
+                        if (descs[0].equals("yes")) {
+                            node.yes = Integer.parseInt(descs[1]);
+                        } else if (descs[0].equals("no")) {
+                            node.no = Integer.parseInt(descs[1]);
+                        } else if (descs[0].equals("missing")) {
+                            node.missing = Integer.parseInt(descs[1]);
+                        }
+                    }
+                }
+                tree.put(id, node);
+            }
+        }
+        if (tree != null && !tree.isEmpty()) {
+            model.add(tree);
+        }
+
+        LOGGER.info("Boosted tree model load over and tree number is " + model.size());
+        input.close();
+        in.close();
+        if (model != null && model.size() > 0) {
+            boosterModel = model;
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private float score(final Map<Integer, Node> tree, final boolean[] featureIsExists, final Float[] featureValues) {
+        int id = 0;
+        while (true) {
+            Node currentNode = tree.get(id);
+            if (currentNode.isLeaf) {
+                return currentNode.value;
+            }
+
+            if (currentNode.fieldType == NodeFieldType.BINARY) {
+                if (!featureIsExists[currentNode.splitFeatureId]) {
+                    id = currentNode.no;
+                } else {
+                    id = currentNode.yes;
+                }
+            } else if (currentNode.fieldType == NodeFieldType.QUANTITATIVE) {
+                if (!featureIsExists[currentNode.splitFeatureId]) {
+                    id = currentNode.missing;
+                } else {
+                    Float value = featureValues[currentNode.splitFeatureId];
+
+                    if ((currentNode.compareType == NodeCompareType.LT && value < currentNode.splitCondition)
+                            || (currentNode.compareType == NodeCompareType.LE && value <= currentNode.splitCondition)) {
+                        id = currentNode.yes;
+                    } else {
+                        id = currentNode.no;
+                    }
+                }
+            } else {
+                LOGGER.error("Reach undefined condition: {}", id);
+            }
+        }
+    }
+
+    public Float score(final Map<String, Double> features, Map<String, Double> featuresScore) {
+        Float result = 0f;
+
+        boolean[] featureIsExists = new boolean[featureCount];
+        Float[] featureValues = new Float[featureCount];
+        for (Map.Entry<String, Double> entry : features.entrySet()) {
+            int featureId = getFeatureId(entry.getKey());
+            if (featureId >= 0) {
+                featureIsExists[featureId] = true;
+                featureValues[featureId] = entry.getValue().floatValue();
+            }
+        }
+
+        for (Map<Integer, Node> tree : boosterModel) {
+            result += score(tree, featureIsExists, featureValues);
+        }
+
+        Float transformedResult = transform(result);
+
+        LOGGER.debug("[calc_dwelltime]features: " + Arrays.toString(features.entrySet().toArray()) + ", prediction:" + result + " , transformed:" + transformedResult);
+
+        return transformedResult;
+    }
+
+    private enum NodeFieldType {
+        BINARY, QUANTITATIVE
+    }
+
+    private enum NodeCompareType {
+        LT, LE
+    }
+
+    private class Node {
+
+        boolean isLeaf;
+        float value;
+        NodeFieldType fieldType;
+        NodeCompareType compareType;
+        int splitFeatureId;
+        Float splitCondition;
+        Integer yes, no, missing;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/LRModel.java

@@ -0,0 +1,169 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+
+import com.tzld.piaoquan.recommend.feature.model.sample.BaseFeature;
+import com.tzld.piaoquan.recommend.feature.model.sample.GroupedFeature;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
+import it.unimi.dsi.fastutil.longs.Long2FloatMap;
+import it.unimi.dsi.fastutil.longs.Long2FloatOpenHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+
+public class LRModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(LRModel.class);
+    private final int bucketBits = 10;  // power(2, 10) => 1024 个槽位
+    private List<Long2FloatMap> lrModel;
+
+    public LRModel() {
+        //配置不同环境的hdfs conf
+        this.lrModel = constructModel();
+    }
+
+    public List<Long2FloatMap> getLrModel() {
+        return lrModel;
+    }
+
+    public List<Long2FloatMap> constructModel() {
+        List<Long2FloatMap> initModel = new ArrayList<Long2FloatMap>();
+        int buckets = (int) Math.pow(2, bucketBits);
+        for (int i = 0; i < buckets; i++) {
+            Long2FloatMap internalModel = new Long2FloatOpenHashMap();
+            internalModel.defaultReturnValue(0.0f);
+            initModel.add(internalModel);
+        }
+        return initModel;
+    }
+
+    public int getBucket(long featureHash) {
+        return (int) (((featureHash >> bucketBits) << bucketBits) ^ featureHash);
+    }
+
+    public void putFeature(List<Long2FloatMap> model, long featureHash, float weight) {
+        model.get(getBucket(featureHash)).put(featureHash, weight);
+    }
+
+    public float getWeight(List<Long2FloatMap> model, long featureHash) {
+        return model.get(getBucket(featureHash)).get(featureHash);
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.lrModel == null)
+            return 0;
+        int sum = 0;
+        for (Map<Long, Float> model : this.lrModel) {
+            sum += model.size();
+        }
+        return sum;
+    }
+
+    public void cleanModel() {
+        this.lrModel = null;
+    }
+
+    public Float score(LRSamples lrSamples) {
+        float sum = 0.0f;
+        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
+            GroupedFeature groupedFeature = lrSamples.getFeatures(i);
+            if (groupedFeature != null && groupedFeature.getFeaturesCount() != 0) {
+                for (int j = 0; j < groupedFeature.getFeaturesCount(); j++) {
+                    BaseFeature baseFeature = groupedFeature.getFeatures(j);
+                    if (baseFeature != null) {
+                        float weight = getWeight(this.lrModel, baseFeature.getIdentifier());
+                        baseFeature.toBuilder().setWeight(weight);
+                        sum += weight;
+                    }
+                }
+            }
+        }
+
+        float pro = (float) (1.0f / (1 + Math.exp(-sum)));
+        lrSamples.toBuilder().setPredictCtr(pro);
+        return pro;
+    }
+
+    public Float getWeights(LRSamples lrSamples) {
+        float sum = 0.0f;
+
+        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
+            GroupedFeature gf = lrSamples.getFeatures(i);
+            if (gf != null && gf.getFeatures(i) != null) {
+                for (int j = 0; j < gf.getFeaturesCount(); j++) {
+                    BaseFeature fea = gf.getFeatures(j);
+                    if (fea != null) {
+                        float tmp = getWeight(this.lrModel, fea.getIdentifier());
+                        fea.toBuilder().setWeight(tmp);
+                        sum += tmp;
+                    }
+                }
+            }
+        }
+        lrSamples.toBuilder().setWeight(sum);
+        return sum;
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        List<Long2FloatMap> model = constructModel();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+
+            putFeature(model, new BigInteger(items[0].trim()).longValue(), Float.valueOf(items[1].trim()).floatValue());
+            if (cnt++ < 10) {
+                LOGGER.debug("fea: " + items[0] + ", weight: " + items[1]);
+            }
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.lrModel = model;
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+            putFeature(model, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/Model.java

@@ -0,0 +1,11 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+
+import java.io.InputStreamReader;
+
+abstract public class Model {
+    public abstract int getModelSize();
+
+    public abstract boolean loadFromStream(InputStreamReader in) throws Exception;
+}
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ModelManager.java

@@ -0,0 +1,247 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.OSSClientBuilder;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.auth.DefaultCredentialProvider;
+import com.aliyun.oss.model.OSSObject;
+import com.ctrip.framework.apollo.Config;
+import com.ctrip.framework.apollo.ConfigService;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+
+@Slf4j
+public class ModelManager {
+    private static final int SCHEDULE_PERIOD = 10;
+    private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+    private static ModelManager instance;
+    Map<String, ModelLoadTask> loadTasks = new HashMap<>();
+    Map<String, String> modelPathMap = new HashMap<>();
+    private OSS client;
+    private String bucketName;
+
+    private final String modelOssEndpoint = "model.oss.internal.endpoint";
+    private final String modelOssAccessKeyId = "model.oss.accessKeyId";
+    private final String modelOssAccessKeySecret = "model.oss.accessKetSecret";
+    private final String modelOssBucketName = "model.oss.bucketName";
+
+    private ModelManager() {
+        // config load
+        Config config = ConfigService.getAppConfig();
+        String endpoint = config.getProperty(modelOssEndpoint, "");
+        String accessKeyId = config.getProperty(modelOssAccessKeyId, "");
+        String accessKetSecret = config.getProperty(modelOssAccessKeySecret, "");
+        // oss client
+        CredentialsProvider credentialsProvider = new DefaultCredentialProvider(accessKeyId, accessKetSecret);
+        this.client = new OSSClientBuilder().build(endpoint, credentialsProvider);
+        this.bucketName = config.getProperty(modelOssBucketName, "");
+
+        config.addChangeListener(changeEvent -> {
+            if (changeEvent.isChanged(modelOssEndpoint)
+                    || changeEvent.isChanged(modelOssAccessKeyId)
+                    || changeEvent.isChanged(modelOssAccessKeySecret)) {
+                String endpointNew = config.getProperty(modelOssEndpoint, "");
+                String accessKeyIdNew = config.getProperty(modelOssAccessKeyId, "");
+                String accessKetSecretNew = config.getProperty(modelOssAccessKeySecret, "");
+                CredentialsProvider credentialsProviderNew = new DefaultCredentialProvider(accessKeyIdNew,
+                        accessKetSecretNew);
+                this.client = new OSSClientBuilder().build(endpointNew, credentialsProviderNew);
+            }
+            if (changeEvent.isChanged(modelOssBucketName)) {
+                this.bucketName = config.getProperty(modelOssBucketName, "");
+            }
+        });
+
+
+        start(SCHEDULE_PERIOD);
+    }
+
+    public static ModelManager getInstance() {
+        if (instance == null) {
+            synchronized (ModelManager.class) {
+                if (instance == null) {
+                    instance = new ModelManager();
+                }
+            }
+        }
+        return instance;
+    }
+
+    /**
+     * 添加一个加载任务到管理器
+     *
+     * @param modelName  Model的名字, 注册到ModelManager的不同model需要不同的名字
+     * @param path       Model在OSS上的全路径
+     * @param modelClass Model的子类型
+     */
+    public void registerModel(String modelName, String path, Class<? extends Model> modelClass) throws ModelRegisterException, IOException {
+        if (modelPathMap.containsKey(modelName)) {
+            // fail fast
+            // throw new RuntimeException(modelName + " already exists");
+            // hard code  广告需要视频模型打分数据，配置要分开
+            return;
+
+//            String oldPath = modelPathMap.get(modelName);
+//            if (path.equals(oldPath)) {
+//                //如果模型的path没有发生改变, 不做任何操作
+//                log.info("Model [{}] and Path [{}] has exist", modelName, path);
+//                return;
+//            } else {
+//                //如果模型的path发生改变, 需要注销掉原有的任务
+//                unRegisterModel(modelName);
+//            }
+        }
+
+        modelPathMap.put(modelName, path);
+        if (loadTasks.containsKey(path)) {
+            ModelLoadTask loadTask = loadTasks.get(path);
+            loadTask.refCount++;
+        } else {
+            ModelLoadTask task = new ModelLoadTask(path, modelClass);
+            task.refCount++;
+            loadTasks.put(path, task);
+            loadModel(task, false, true);
+        }
+    }
+
+    /**
+     * 删除一个加载任务
+     *
+     * @param modelName Model的名字, 需要和registerModel的名字一致
+     */
+    private void unRegisterModel(String modelName) {
+        if (modelPathMap.containsKey(modelName)) {
+            String path = modelPathMap.get(modelName);
+            if (loadTasks.containsKey(path)) {
+                ModelLoadTask task = loadTasks.get(path);
+                task.refCount--;
+                if (task.refCount == 0) {
+                    loadTasks.remove(path);
+                }
+            }
+            modelPathMap.remove(modelName);
+        }
+    }
+
+    /**
+     * @param modelName
+     * @return
+     */
+    public Model getModel(String modelName) {
+        if (modelPathMap.containsKey(modelName) && loadTasks.containsKey(modelPathMap.get(modelName))) {
+            return loadTasks.get(modelPathMap.get(modelName)).model;
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * 开始调度
+     *
+     * @param period
+     */
+    protected void start(long period) {
+        final Runnable task = new Runnable() {
+            public void run() {
+                // 模型更新开关
+                // boolean modelUpdateSwitch = Configuration.getBoolean("recommend-service-framework.model-update-switch", true);
+                boolean modelUpdateSwitch = true;
+                log.info("model update switch [{}]", modelUpdateSwitch);
+                if (modelUpdateSwitch) {
+                    updateModels(false);
+                }
+            }
+        };
+        scheduler.scheduleAtFixedRate(task, 10, period, TimeUnit.MINUTES); // 10分钟
+    }
+
+    /**
+     * 更新模型
+     */
+    public void updateModels(final boolean isForceLoads) {
+        log.info("begin to update: [{}]", loadTasks.keySet().size());
+        for (String modelPath : loadTasks.keySet()) {
+            log.debug("load task model path [{}]", modelPath);
+            ModelLoadTask task = loadTasks.get(modelPath);
+            loadModel(task, isForceLoads, false);
+        }
+    }
+
+    /**
+     * 检查并加载模型
+     * <p>
+     * 从oss加载：
+     * https://help.aliyun.com/zh/oss/developer-reference/streaming-download-7?spm=a2c4g.11186623.0.0.4b527c7dm8LejC
+     *
+     * @param loadTask
+     */
+    private void loadModel(final ModelLoadTask loadTask, final boolean isForceLoads, final boolean isRegister) {
+        if (loadTask.isLoading) {
+            return;
+        }
+        loadTask.isLoading = true;
+        OSSObject ossObj = null;
+        try {
+            ossObj = client.getObject(bucketName, loadTask.path);
+            long timeStamp = ossObj.getObjectMetadata().getLastModified().getTime();
+            if (loadTask.lastModifyTime < timeStamp || isForceLoads) {
+                log.info("model file changed, ready to update, last modify: [{}], current model time: [{}]",
+                        loadTask.lastModifyTime, timeStamp);
+
+                Model model = loadTask.modelClass.newInstance();
+                if (model.loadFromStream(new InputStreamReader(ossObj.getObjectContent()))) {
+                    loadTask.model = model;
+                    loadTask.lastModifyTime = timeStamp;
+                }
+            }
+            ossObj.close();
+        } catch (Exception e) {
+            log.error("update model fail", e);
+        } finally {
+            loadTask.isLoading = false;
+            if (ossObj != null) {
+                try {
+                    ossObj.close();
+                } catch (IOException e) {
+                    log.error("close ossObj fail", e);
+                }
+            }
+        }
+    }
+
+    public class ModelRegisterException extends Exception {
+
+        public ModelRegisterException(String s) {
+            super(s);
+        }
+    }
+
+    /**
+     * 调度的任务单元
+     */
+    private class ModelLoadTask {
+
+        private int refCount;
+        private final String path;
+        private long lastModifyTime;
+        private boolean isLoading;
+        private final Class<? extends Model> modelClass;
+        private Model model;
+
+        ModelLoadTask(String path, Class<? extends Model> modelClass) {
+            this.refCount = 0;
+            this.path = path;
+            this.lastModifyTime = 0;
+            this.modelClass = modelClass;
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/OssConfig.java

@@ -0,0 +1,15 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+import lombok.Data;
+
+/**
+ * @author dyp
+ */
+@Data
+public class OssConfig {
+
+    private String accessKeyId;
+    private String accessKeySecret;
+    private String endpoint;
+    private String bucketName;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ThompsonSamplingModel.java

@@ -0,0 +1,90 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.base.VideoActionFeature;
+import org.apache.commons.math3.distribution.BetaDistribution;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class ThompsonSamplingModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25;  // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(ThompsonSamplingModel.class);
+
+    // key = videoid, value = < push, exp, play, realplay, share, retures >
+    private Map<Long, VideoActionFeature> thompsonSamplingModel;
+
+    private static final int alpha = 20;
+    private static final int beta_returns = 100;
+
+    public ThompsonSamplingModel() {
+        //配置不同环境的hdfs conf
+        this.thompsonSamplingModel = new HashMap<>();
+    }
+
+    public Map<Long, VideoActionFeature> getThompsonSamplingModel() {
+        return this.thompsonSamplingModel;
+    }
+
+
+
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+        Map<Long, VideoActionFeature> initModel = new HashMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 3) {
+                continue;
+            }
+            Long videoId = new BigInteger(items[0].trim()).longValue();
+            VideoActionFeature videoFeature = new VideoActionFeature();
+            videoFeature.setView(Double.valueOf(items[1].trim()));
+            videoFeature.setPlay(Double.valueOf(items[2].trim()));
+            videoFeature.setShare(Double.valueOf(items[3].trim()));
+            videoFeature.setReturns(Double.valueOf(items[5].trim()));
+            initModel.put(videoId, videoFeature);
+        }
+
+        this.thompsonSamplingModel = initModel;
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.thompsonSamplingModel == null)
+            return 0;
+        int sum = this.thompsonSamplingModel.size();
+        return sum;
+    }
+
+    public double score(RankItem rankItem) {
+        double score = 0.0f;
+        VideoActionFeature videoActionFeature = this.thompsonSamplingModel.getOrDefault(rankItem.getVideoId(), new VideoActionFeature());
+
+        int alpha = (int) videoActionFeature.getReturns() + ThompsonSamplingModel.alpha;
+        int beta = beta_returns + (int) videoActionFeature.getView();
+        score = this.betaSampler(alpha, beta);
+        return score;
+    }
+
+    public double betaSampler(double alpha, double beta) {
+        BetaDistribution betaSample = new BetaDistribution(alpha, beta);
+        return betaSample.sample();
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/AbstractFilter.java

@@ -0,0 +1,31 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public abstract class AbstractFilter<T> {
+    protected final static Logger LOGGER = LoggerFactory.getLogger(AbstractFilter.class);
+    protected final FilterConfigInfo filterConfigInfo;
+    protected final RecommendRequest requestContext;
+    protected final User user;
+
+    public AbstractFilter(FilterConfigInfo filterConfigInfo,
+                          RecommendRequest requestContext,
+                          User user) {
+
+        this.filterConfigInfo = filterConfigInfo;
+        this.requestContext = requestContext;
+        this.user = user;
+    }
+
+    public FilterConfigInfo getFilterConfigInfo() {
+        return filterConfigInfo;
+    }
+
+    public abstract void doFilter(Candidate candidate, List<T> t);
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/BaseRecaller.java

@@ -0,0 +1,298 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.*;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.recaller.provider.QueueProvider;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+import java.util.stream.Collectors;
+
+
+public class BaseRecaller<Video> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(BaseRecaller.class);
+    private static final long DEFAULT_QUEUE_LOAD_TIMEOUT = 150; // ms
+    private static final long DEFAULT_PARALLEL_FILTER_TIMEOUT = 200; // ms
+
+    private static final String FILTER_CONF = "filter_config.conf"; // ms
+    private static final ExecutorService filterExecutorService = new ThreadPoolExecutor(128, 128,
+            0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1000), r -> new Thread(r, "filterExecutorService"),
+            new ThreadPoolExecutor.CallerRunsPolicy());
+
+    private static final ExecutorService fetchQueueExecutorService = new ThreadPoolExecutor(128, 128,
+            0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1000), r -> new Thread(r, "fetchQueueExecutorService"),
+            new ThreadPoolExecutor.CallerRunsPolicy());
+    private final QueueProvider<Video> queueProvider;
+
+    private static final FilterConfig filterConfig;
+
+    private final long QUEUE_LOAD_TIMEOUT;
+
+    static {
+        filterConfig = new FilterConfig(FILTER_CONF);
+    }
+
+    public BaseRecaller(QueueProvider<Video> queueProvider) {
+        this(queueProvider, DEFAULT_QUEUE_LOAD_TIMEOUT);
+    }
+
+    public BaseRecaller(QueueProvider<Video> queueProvider,
+                        long queueLoadTimeout) {
+        this.queueProvider = queueProvider;
+        this.QUEUE_LOAD_TIMEOUT = queueLoadTimeout;
+    }
+
+
+    public BaseRecaller(QueueProvider<Video> queueProvider,
+                        FilterConfig filterConfig,
+                        long queueLoadTimeout) {
+        this.queueProvider = queueProvider;
+        this.QUEUE_LOAD_TIMEOUT = queueLoadTimeout;
+    }
+
+
+    public String extractItemId(Entry<Video> entry) {
+        return entry.id;
+    }
+
+
+    public boolean isValidItem(Video item) {
+        return item != null;
+    }
+
+    /**
+     * 把queue中的entry, 补充candidiate信息，放入RankItem中
+     *
+     * @param entries
+     * @param candidate
+     * @return
+     */
+    private List<RankItem> toHits(final Iterable<Entry<Video>> entries, final Candidate candidate) {
+
+        List<RankItem> result = new ArrayList<RankItem>();
+        for (Entry entry : entries) {
+            RankItem item = new RankItem();
+            item.setId(extractItemId(entry));
+            item.putRankerScore("L3Score", (Double) entry.scores.get("ordering"));
+            item.setQueue(candidate.getMergeQueueName()); // merge queue
+
+            CandidateInfo candidateInfo = new CandidateInfo();
+            candidateInfo.setCandidateQueueName(candidate.getCandidateKey());
+            candidateInfo.setCandidate(candidate);
+
+            item.setCandidateInfo(candidateInfo);
+            item.addToCandidateInfoList(candidateInfo);
+            result.add(item);
+        }
+        return result;
+    }
+
+
+    // 读取redis中的数据放入queue中
+    public Map<Candidate, Queue<Video>> loadQueues(List<Candidate> candidates) {
+        // update queueName
+        Iterable<Candidate> updateCandidates = FluentIterable.from(candidates).transform(candidate -> {
+            try {
+                long ttl = QueueName.DEFAULT_LOCAL_CACHE_TTL;
+                candidate.setCandidateQueueName(QueueName.fromString(candidate.getCandidateKey(), ttl));
+            } catch (Exception e) {
+                candidate.setCandidateQueueName(null);
+                LOGGER.error("error parse QueueName [{}]", candidate.getCandidateKey());
+            }
+            return candidate;
+        });
+
+        // parse queues
+        Iterable<QueueName> queueNames = FluentIterable.from(updateCandidates).transform(candidate -> candidate.getCandidateQueueName());
+
+        // parallel load queues, redis 或者缓存获取index
+        Map<QueueName, Queue<Video>> queues = Maps.newConcurrentMap();
+        try {
+            // 格式
+            queues = queueProvider.loads(Lists.newArrayList(queueNames), QUEUE_LOAD_TIMEOUT, TimeUnit.MILLISECONDS);
+            LOGGER.info("load queue success [{}]", queues.size());
+        } catch (Exception e) {
+            LOGGER.error("load queue occur error [{}]", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        // parse candidate map
+        Map<Candidate, Queue<Video>> candidateQueueMap = Maps.newConcurrentMap();
+        for (Candidate candidate : updateCandidates) {
+            QueueName name = candidate.getCandidateQueueName();
+            Queue<Video> entries = queues.get(name);
+            if (queues.containsKey(name) && entries != null) {
+                candidateQueueMap.put(candidate, entries);
+            }
+        }
+        return candidateQueueMap;
+    }
+
+
+    /**
+     * recall
+     *
+     * @param requestData
+     * @param user
+     * @param recallCandidates
+     * @return
+     */
+    public List<RankItem> recalling(final RecommendRequest requestData, final User user, List<Candidate> recallCandidates) {
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        stopwatch.reset().start();
+        // load from redis
+        List<Callable<Map<Candidate, Queue<Video>>>> fetchQueueCalls = Lists.newArrayList();
+        fetchQueueCalls.add(() -> {
+            boolean isFromRedis = true;
+            return obtainQueue(recallCandidates, requestData, user, isFromRedis);
+        });
+
+        // 多线程执行load
+        List<Future<Map<Candidate, Queue<Video>>>> fetchQueueFutures = null;
+        try {
+            fetchQueueFutures = fetchQueueExecutorService.invokeAll(fetchQueueCalls, DEFAULT_QUEUE_LOAD_TIMEOUT,
+                    TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("[fetch queue error] inter fail: {}", ExceptionUtils.getStackTrace(e));
+        } catch (Exception e) {
+            LOGGER.error("[fetch queue error] ex", ExceptionUtils.getStackTrace(e));
+        }
+
+        Map<Candidate, Queue<Video>> candidateQueueMap = Maps.newHashMap();
+        if (CollectionUtils.isNotEmpty(fetchQueueFutures)) {
+            for (Future<Map<Candidate, Queue<Video>>> future : fetchQueueFutures) {
+                if (future.isDone() && !future.isCancelled()) {
+                    Map<Candidate, Queue<Video>> result = null;
+                    try {
+                        result = future.get();
+                    } catch (InterruptedException e) {
+                        LOGGER.error("[fetch queue error] InterruptedException {}", ExceptionUtils.getStackTrace(e));
+                    } catch (ExecutionException e) {
+                        LOGGER.error("[fetch queue error] ex {}", ExceptionUtils.getStackTrace(e));
+                    }
+                    if (result != null) {
+//                        LOGGER.info("fetch queue success [{}]", JSONObject.toJSONString(result));
+                        candidateQueueMap.putAll(result);
+                    }
+                }
+            }
+        }
+
+        LOGGER.info("traceId: [{}], cost = {}, recall candidateQueueMap: [{}]", requestData.getRequestId(),
+                stopwatch.elapsed().toMillis(), JSONObject.toJSONString(candidateQueueMap));
+
+        stopwatch.reset().start();
+        List<RankItem> result = convertToRankItem(candidateQueueMap, requestData, user);
+
+        LOGGER.info("traceId: [{}], cost = {}, convertToRankItem result: [{}]", requestData.getRequestId(),
+                stopwatch.elapsed().toMillis(), JSONObject.toJSONString(result));
+        return result;
+    }
+
+
+    // 转成RankItem
+    // 同时给Filter预留处理
+    private List<RankItem> convertToRankItem(Map<Candidate, Queue<Video>> candidateQueueMap,
+                                             RecommendRequest requestData,
+                                             User user) {
+
+
+        final FilterPipeline<Video> recallFilter = new FilterPipeline<>(filterConfig, requestData, user);
+
+        LOGGER.info("traceId: [{}], recallFilter: [{}]", requestData.getRequestId(),
+                JSONObject.toJSONString(recallFilter));
+
+        final List<Callable<List<RankItem>>> callables = new ArrayList<>();
+        for (final Map.Entry<Candidate, Queue<Video>> entry : candidateQueueMap.entrySet()) {
+            callables.add(() -> {
+                List<RankItem> candidateHits = new ArrayList<>();
+                final Candidate candidate = entry.getKey();
+                List<Entry<Video>> entriesValue = CommonCollectionUtils.iterableToList(entry.getValue());
+                LOGGER.info("traceId: [{}], entry.getValue: [{}], entriesValue: [{}]", requestData.getRequestId(),
+                        JSONObject.toJSONString(entry.getValue()), JSONObject.toJSONString(entriesValue));
+                try {
+                    final Map<Video, Entry<Video>> entryMap = new HashMap<>();
+                    List<Video> entriesList = entriesValue.stream().map(e -> {
+                        entryMap.put(e.item, e);
+                        return e.item;
+                    }).collect(Collectors.toList());
+                    List<Video> entriesAfterFilter = recallFilter.doFilter(candidate, entriesList);
+                    LOGGER.info("traceId: [{}] entriesAfterFilter: [{}]",
+                            requestData.getRequestId(), JSONObject.toJSONString(entriesAfterFilter));
+                    // 1. filter  TODO 待后续增加自定义filter
+                    if (CollectionUtils.isNotEmpty(entriesAfterFilter)) {
+                        List<Entry<Video>> entries = entriesAfterFilter.stream()
+                                .limit(candidate.getCandidateNum())
+                                .map(entryMap::get)
+                                .collect(Collectors.toList());
+                        // 2. toHits
+                        candidateHits.addAll(toHits(entries, candidate));
+                    }
+
+                    // debug log for tracing
+                    LOGGER.info("recalled candidate [{}], queue length [{}], expected [{}], hit [{}]",
+                            candidate.getCandidateKey(), entry.getValue().size(), candidate.getCandidateNum(), candidateHits.size());
+                } catch (Exception e) {
+                    LOGGER.error("recall filter queue occur error, queue [{}], error: [{}]", candidate.toString(), ExceptionUtils.getFullStackTrace(e));
+                }
+                return candidateHits;
+            });
+        }
+
+        Map<String, RankItem> hits = new HashMap<>();
+        try {
+            List<Future<List<RankItem>>> futures = filterExecutorService.invokeAll(callables, DEFAULT_PARALLEL_FILTER_TIMEOUT, TimeUnit.MILLISECONDS);
+            for (Future<List<RankItem>> future : futures) {
+                try {
+                    if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                        List<RankItem> part = future.get();
+                        // add to result
+                        for (RankItem item : part) {
+                            // merge candidate Info
+                            if (hits.containsKey(item.getId())) {
+                                hits.get(item.getId()).addToCandidateInfoList(item.getCandidateInfo());
+                            } else {
+                                hits.put(item.getId(), item);
+                            }
+                        }
+                        LOGGER.debug("collect items from filter task, userid: [{}], item size [{}]", "", part.size());
+                    } else {
+                        LOGGER.error("parallel recall filter Canceled {} ", "");
+                    }
+                } catch (Exception e) {
+                    LOGGER.error("parallel recall filter occur error, uid: [{}], Exception [{}]",
+                            "", ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        } catch (Exception e) {
+            LOGGER.error("parallel recall filter occur error, uid: [{}], Exception [{}]",
+                    "", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        List<RankItem> result = new ArrayList<>(hits.values());
+
+        return result;
+    }
+
+
+    private Map<Candidate, Queue<Video>> obtainQueue(List<Candidate> refactorCandidates, RecommendRequest requestData, User user, boolean isFromRedis) {
+        return loadQueues(refactorCandidates);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterConfig.java

@@ -0,0 +1,99 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+
+ */
+
+public class FilterConfig {
+    private static Logger LOGGER = LoggerFactory.getLogger(FilterConfig.class);
+    private List<FilterConfigInfo> filterConfigInfoList = new ArrayList<FilterConfigInfo>();
+
+    public FilterConfig(Config config) {
+        this.load(config);
+    }
+
+
+    public FilterConfig(String configFile) {
+        this.load(configFile);
+    }
+
+    public FilterConfig() {
+    }
+
+    public boolean load(String configFile) {
+        Config filterConfig = ConfigFactory.parseResources(configFile);
+        return load(filterConfig);
+    }
+
+    public boolean load(Config config) {
+        Config recallConfig = config.getConfig("recall-config");
+
+        Config filterConf = recallConfig.getConfig("filter-config");
+        try {
+            loadFilters(filterConf);
+            int pos = 0;
+            for (FilterConfigInfo filterConfigInfo : filterConfigInfoList) {
+                LOGGER.info("filter at position [{}], priority [{}] filter name [{}] ",
+                        new Object[]{pos++, filterConfigInfo.getFilterPriority(), filterConfigInfo.getConfigName()});
+            }
+            LOGGER.debug("Load filter config success");
+        } catch (Exception e) {
+            LOGGER.error("Load filter config failed, [{}]", ExceptionUtils.getFullStackTrace(e));
+            return false;
+        }
+
+        return true;
+    }
+
+    public List<FilterConfigInfo> getFilterConfigInfoList() {
+        return filterConfigInfoList;
+    }
+
+    private void loadFilters(Config config) throws Exception {
+        ConfigObject confObj = config.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            Config conf = ((ConfigObject) it.getValue()).toConfig();
+
+            // parse config
+            String configName = it.getKey();
+            String filterName = conf.getString("filter-name");
+            int filterPriority = 0;
+            if (conf.hasPath("filter-priority"))
+                filterPriority = conf.getInt("filter-priority");
+
+            List<Integer> disableExpIds = new ArrayList<Integer>();
+            if (conf.hasPath("disable-expids")) {
+                disableExpIds = conf.getIntList("disable-expids");
+            }
+            FilterConfigInfo filterConfigInfo = new FilterConfigInfo(configName,
+                    filterName, filterPriority, disableExpIds);
+            LOGGER.debug("parse filter config info [{}]", filterConfigInfo);
+
+            addConfigByPriority(filterConfigInfoList, filterConfigInfo);
+        }
+    }
+
+    private void addConfigByPriority(List<FilterConfigInfo> configInfoList, FilterConfigInfo addConfigInfo) {
+
+        int pos = 0;
+        for (; pos < configInfoList.size(); pos++) {
+            if (configInfoList.get(pos).getFilterPriority() <= addConfigInfo.getFilterPriority()) {
+                break;
+            }
+        }
+        configInfoList.add(pos, addConfigInfo);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterConfigInfo.java

@@ -0,0 +1,59 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller;
+
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.FluentIterable;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+
+public class FilterConfigInfo {
+    private Set<Integer> disableExpIds;
+    private String filterName;
+    private Integer filterPriority;
+    private String configName;
+
+    public FilterConfigInfo(String configName,
+                            String filterName,
+                            Integer filterPriority,
+                            List<Integer> expIds) {
+        this.configName = configName;
+        this.filterName = filterName;
+        this.filterPriority = filterPriority;
+        this.disableExpIds = new HashSet<Integer>();
+        this.disableExpIds.addAll(expIds);
+    }
+
+    public Integer getFilterPriority() {
+        return filterPriority;
+    }
+
+    public String getFilterName() {
+        return filterName;
+    }
+
+    public Set<Integer> getDisableExpIds() {
+        return disableExpIds;
+    }
+
+    public String getConfigName() {
+        return configName;
+    }
+
+    @Override
+    public String toString() {
+        return configName + ":" + filterPriority + ":" + filterName + ":" +
+                Joiner.on(",").join(FluentIterable.from(disableExpIds).transform(new Function<Integer, String>() {
+                    @Nullable
+                    @Override
+                    public String apply(Integer integer) {
+                        return integer.toString();
+                    }
+                }));
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterPipeline.java

@@ -0,0 +1,64 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class FilterPipeline<T> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(FilterPipeline.class);
+    public int filterNum;
+    private final FilterConfig config;
+    private final List<AbstractFilter<T>> filters;
+    private final RecommendRequest requestContext;
+    private final User user;
+
+    public FilterPipeline(FilterConfig config,
+                          RecommendRequest requestContext,
+                          User user) {
+
+        this.config = config;
+        this.requestContext = requestContext;
+        this.user = user;
+        this.filters = new ArrayList<>();
+        this.constructFilters(config);
+    }
+
+    public List<AbstractFilter<T>> getFilters() {
+        return filters;
+    }
+
+    public void constructFilters(FilterConfig config) {
+        // add filter
+        if (config != null) {
+            for (FilterConfigInfo filterConfigInfo : config.getFilterConfigInfoList()) {
+                try {
+                    AbstractFilter<T> filter = (AbstractFilter) Class.forName(filterConfigInfo.getFilterName())
+                            .getConstructor(FilterConfigInfo.class, RecommendRequest.class, User.class)
+                            .newInstance(filterConfigInfo, this.requestContext, this.user);
+
+                    this.filters.add(filter);
+                } catch (Exception e) {
+                    LOGGER.error("Filter config info construct error, [{}] [{}]",
+                            filterConfigInfo, ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public List<T> doFilter(Candidate candidate, List<T> list) {
+        LOGGER.info("FilterPipeline doFilter start traceId:{}, before list size:{}", requestContext.getRequestId(), list.size());
+        for (AbstractFilter filter : filters) {
+            filter.doFilter(candidate, list);
+        }
+        LOGGER.info("FilterPipeline doFilter end traceId:{}, after list size:{}", requestContext.getRequestId(), list.size());
+        return list;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/CacheEntry.java

@@ -0,0 +1,24 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+
+import com.google.common.base.Optional;
+import lombok.Data;
+
+@Data
+public class CacheEntry<T> {
+    private String itemId; // itemId
+    private Optional<T> item;
+    private Long expireTime; // 静态特征失效时间
+    private Long dynamicFeatureExpireTime; // 动态特征失效时间
+
+    public CacheEntry(String itemId,
+                      Long expireTime,
+                      Long dynamicFeatureExpireTime,
+                      Optional<T> item) {
+        this.itemId = itemId;
+        this.expireTime = expireTime;
+        this.dynamicFeatureExpireTime = dynamicFeatureExpireTime;
+        this.item = item;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/Index.java

@@ -0,0 +1,44 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.Data;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+@Data
+public class Index {
+    private String indexName = null;
+    private List<IndexEntry> indexEntryList = new ArrayList<>();
+
+
+    // TODO
+    // redis中一路召回队列key 的结果 转化到Index中
+    public Index(String indexName, String indexValue) {
+        if (StringUtils.isBlank(indexName) || StringUtils.isBlank(indexValue)) {
+            return;
+        }
+        this.indexName = indexName;
+        JSONObject jsonObject = JSONObject.parseObject(indexValue);
+        String valueList = jsonObject.getString("value_list");
+        valueList = "[" + valueList + "]";
+        List<List<String>> videoScores = JSONUtils.fromJson(valueList, new TypeToken<List<List<String>>>() {
+        }, Collections.emptyList());
+
+        videoScores.stream().forEach(itemAndScore -> {
+                    IndexEntry indexEntry = new IndexEntry();
+                    if (itemAndScore.size() >= 2) {
+                        indexEntry.setItemId(itemAndScore.get(0));
+                        indexEntry.setScore(Double.valueOf(itemAndScore.get(1)));
+                        this.indexEntryList.add(indexEntry);
+                    }
+                }
+        );
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/IndexEntry.java

@@ -0,0 +1,10 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+import lombok.Data;
+
+@Data
+public class IndexEntry {
+    private String itemId;
+    private Double score;
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/QueueProvider.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+
+
+
+public interface QueueProvider<T> {
+
+    Queue<T> load(QueueName name) throws Exception;
+
+    Index get(QueueName name) throws Exception;
+
+    Map<QueueName, Queue<T>> loads(List<QueueName> names) throws Exception;
+
+    Map<QueueName, Queue<T>> loads(List<QueueName> names, long timeout, TimeUnit timeUnit) throws Exception;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/RedisBackedQueue.java

@@ -0,0 +1,199 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Iterables;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.utils.FixedThreadPoolHelper;
+import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+
+public class RedisBackedQueue implements QueueProvider<Video> {
+    private static final Logger logger = LoggerFactory.getLogger(RedisBackedQueue.class);
+    private static final int DEFAULT_TTL = 7 * 24 * 60 * 60;
+    private static final long CACHE_MAXIMUMSIZE = 50 * 10000; // default 50w
+    private static final long CACHE_TIMEOUT_MS = 5 * 60 * 1000L;
+
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(64);
+
+    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+    private final ExecutorService queueRefreshExecutorService = new FixedThreadPoolHelper(4, "queueRefresh").getThreadPoolExecutor();
+
+    // 不能为static
+    private final BlockingQueue<QueueName> asyncRefreshQueue = new LinkedBlockingQueue<QueueName>(50000);
+    private final RedisSmartClient client;
+    private final LoadingCache<QueueName, Pair<Long, Queue<Video>>> cache;
+
+    /**
+     * 实例化 RedisBackedQueue, 并提供基于Guava Cache的本地缓存管理工作
+     *
+     * @param client         redis-cluster 连接池
+     * @param cacheTimeOutMs 缓存有效期, 开启缓存的情况下,默认缓存5min,以写时间为基准
+     */
+    public RedisBackedQueue(RedisSmartClient client, final long cacheTimeOutMs) {
+        this.client = client;
+
+        cache = CacheBuilder.newBuilder()
+                .expireAfterWrite((long) (cacheTimeOutMs * 1.3), TimeUnit.MILLISECONDS)
+                .maximumSize(CACHE_MAXIMUMSIZE)
+                .build(new RedisBackedQueueWithoutMetaCacheLoader(client));
+
+        // protect key
+        for (int i = 0; i < 4; i++) {
+            queueRefreshExecutorService.execute(new CacheRefreshRunnable());
+        }
+    }
+
+    public RedisBackedQueue(RedisSmartClient client) {
+        this(client, CACHE_TIMEOUT_MS);
+    }
+
+    public RedisSmartClient getClient() {
+        return client;
+    }
+
+
+    // redis读取队列内内容
+    public Index get(QueueName name) throws Exception {
+        String valueJson = client.get(name.toString());
+        logger.info("RedisBackedQueue get QueueName: [{}], value: [{}]", name, valueJson);
+        Index index = new Index(name.toString(), valueJson);
+        return index;
+    }
+
+
+
+    /**
+     * 加载指定索引, 并指定索引本地缓存的TTL
+     * @param name QueueName
+     * @return
+     * @throws ExecutionException
+     */
+    public Queue<Video> load(QueueName name) throws ExecutionException {
+
+        Pair<Long, Queue<Video>> cachedQueue = cache.get(name);
+//        logger.info("RedisBackedQueue load QueueName: [{}], cachedQueue size: [{}]", name,
+//                cachedQueue.getValue().size());
+        if (cachedQueue == null) {
+            // 清理本地缓存中当前key的数据
+            cache.invalidate(name);
+            logger.debug("invalidate queue [{}]", name);
+            return new Queue<Video>(name.toString());
+        }
+        // update key refresh time
+        if (cachedQueue.getKey() == Long.MIN_VALUE) {
+            cache.put(name, Pair.of(System.currentTimeMillis() + name.getTTL(), cachedQueue.getValue()));
+        }
+
+        if (name.getTTL() == 0) {
+            cache.invalidate(name);
+            logger.debug("invalidate queue [{}]", name);
+        }
+
+        // check and refresh
+        if (name.getTTL() > 0 && System.currentTimeMillis() > cachedQueue.getKey()) {
+            boolean isok = asyncRefreshQueue.offer(name);
+            if (logger.isDebugEnabled()) {
+                logger.debug("offer async queue name [{}], result [{}]", name, isok);
+            }
+            if (asyncRefreshQueue.size() > CACHE_MAXIMUMSIZE) {
+                logger.warn("index backgroud blocking queue length [{}], it is danger", asyncRefreshQueue.size());
+            }
+            if (!isok) {
+                logger.error("index backgroud offer blocking queue error, queuename [{}], with queue lenght [{}]", name, asyncRefreshQueue.size());
+            }
+        }
+
+        return cachedQueue.getValue();
+    }
+
+    public Map<QueueName, Queue<Video>> loads(List<QueueName> names) throws ExecutionException {
+        return this.loads(names, 200, TimeUnit.MILLISECONDS);
+    }
+
+    public Map<QueueName, Queue<Video>> loads(List<QueueName> names, long timeout, TimeUnit timeUnit) throws ExecutionException {
+
+        final Iterable<List<QueueName>> namesIterable = Iterables.partition(names, 1);
+        final List<Callable<Map<QueueName, Queue<Video>>>> callables =
+                new ArrayList<Callable<Map<QueueName, Queue<Video>>>>();
+        for (final List<QueueName> queueNames : namesIterable) {
+            callables.add(() -> {
+                Map<QueueName, Queue<Video>> result = new HashMap<>();
+                for (final QueueName name : queueNames) {
+                    try {
+                        Queue queue = load(name);
+                        result.put(name, queue);
+                    } catch (Exception e) {
+                        logger.error("load queue error [{}] [{}]", name, ExceptionUtils.getFullStackTrace(e));
+                    }
+                }
+                return result;
+            });
+        }
+
+        Map<QueueName, Queue<Video>> loadQueue = new ConcurrentHashMap<QueueName, Queue<Video>>();
+        try {
+            List<Future<Map<QueueName, Queue<Video>>>> futures = executorService.invokeAll(callables);
+            for (Future<Map<QueueName, Queue<Video>>> future : futures) {
+                if (future.isDone() && !future.isCancelled()) {
+                    try {
+                        Map<QueueName, Queue<Video>> ret = future.get();
+                        loadQueue.putAll(ret);
+                    } catch (ExecutionException ee) {
+                        logger.error("Failed to execute load a queue partition for {}", ExceptionUtils.getFullStackTrace(ee));
+                    }
+                }
+            }
+        } catch (InterruptedException ie) {
+            logger.error("Interrupted when waiting for parallel queue load finish for {}", ExceptionUtils.getFullStackTrace(ie));
+        }
+        return loadQueue;
+    }
+
+
+
+    // 使用Guava Cache 的异步刷新接口, 异步刷新到期的key
+    public class CacheRefreshRunnable implements Runnable {
+
+        @Override
+        public void run() {
+            while (true) {
+                QueueName queueName = null;
+                try {
+                    queueName = asyncRefreshQueue.take(); // take async item, blocked when no item
+                    final long startTime = System.currentTimeMillis();
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("async refresh thread get queuename [{}]", queueName); // get itemid
+                    }
+
+                    Pair<Long, Queue<Video>> entry = cache.get(queueName);
+                    // check refresh
+                    if (startTime > entry.getKey()) {
+                        cache.refresh(queueName);
+                    }
+
+                    long endTime = System.currentTimeMillis();
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("parallel refresh queue [{}] spenttime [{}]", queueName, endTime - startTime);
+                    }
+                } catch (Exception e) {
+                    logger.error("schedule refresh queue error [{}], exception [{}]", queueName, ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/RedisBackedQueueWithoutMetaCacheLoader.java

@@ -0,0 +1,56 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+import com.google.common.base.Function;
+import com.google.common.cache.CacheLoader;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Entry;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+
+public class RedisBackedQueueWithoutMetaCacheLoader extends CacheLoader<QueueName, Pair<Long, Queue<Video>>> {
+
+    private static final Logger logger = LoggerFactory.getLogger(RedisBackedQueueWithoutMetaCacheLoader.class);
+    private final RedisSmartClient client;
+
+    public RedisBackedQueueWithoutMetaCacheLoader(RedisSmartClient client) {
+        this.client = client;
+    }
+
+    @Override
+    public Pair<Long, Queue<Video>> load(QueueName name) throws Exception {
+        String valueJson = client.get(name.toString());
+        Index index = new Index(name.toString(), valueJson);
+
+        if (null == index || CollectionUtils.isEmpty(index.getIndexEntryList())) {
+//            logger.error("empty_index_fetch");
+        }
+
+        //IndexEntry 转化到Entry中 video 格式
+        Queue<Video> queue = new Queue<Video>(name.toString());
+        Iterable<Entry<Video>> iterable = Iterables.transform(index.getIndexEntryList(), new Function<IndexEntry, Entry<Video>>() {
+            @Nullable
+            @Override
+            public Entry<Video> apply(IndexEntry indexEntry) {
+                Video recallData = new Video();
+                recallData.setVideoId(Long.valueOf(indexEntry.getItemId()));
+                recallData.setPushFrom(name.toString());
+                Entry<Video> entry = new Entry<Video>(recallData, indexEntry.getItemId());
+                entry.addScore("ordering", indexEntry.getScore());
+                return entry;
+            }
+        });
+        queue.addAll(Lists.newArrayList(iterable));
+
+        return Pair.of(Long.MIN_VALUE, queue);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/AbstractScorer.java

@@ -0,0 +1,71 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.Model;
+import com.tzld.piaoquan.recommend.server.service.score.model.ModelManager;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+
+public abstract class AbstractScorer {
+    public static Logger LOGGER = LoggerFactory.getLogger(AbstractScorer.class);
+    protected ScorerConfigInfo scorerConfigInfo;
+    protected ModelManager modelManager = ModelManager.getInstance();
+
+    public AbstractScorer(ScorerConfigInfo scorerConfigInfo) {
+        this.scorerConfigInfo = scorerConfigInfo;
+    }
+
+    public void loadModel() {
+    }
+
+    public boolean isEnable() {
+        return !getScorerConfigInfo().getDisableSwitch();
+    }
+
+
+    public void doLoadModel(Class<? extends Model> modelClass) {
+
+        String modelPath = scorerConfigInfo.getModelPath();
+        if (StringUtils.isNotBlank(modelPath)) {
+            try {
+                // 使用 modelPath 作为 modelName 注册
+                modelManager.registerModel(modelPath, modelPath, modelClass);
+                LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
+            } catch (ModelManager.ModelRegisterException e) {
+                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+            } catch (IOException e) {
+                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+            }
+        } else {
+            LOGGER.error("modelpath is null, for model class [{}]", modelClass);
+        }
+    }
+
+    public Model getModel() {
+        if (StringUtils.isBlank(scorerConfigInfo.getModelPath())) {
+            return null;
+        }
+        return modelManager.getModel(scorerConfigInfo.getModelPath());
+    }
+
+    public ScorerConfigInfo getScorerConfigInfo() {
+        return scorerConfigInfo;
+    }
+
+    public abstract List<RankItem> scoring(final ScoreParam param,
+                                           final UserFeature userFeature,
+                                           final List<RankItem> rankItems);
+
+    public abstract List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                           final Map<String, String> userFeatureMap,
+                                           final List<RankItem> rankItems);
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseGBDTModelScorer.java

@@ -0,0 +1,19 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.tzld.piaoquan.recommend.server.service.score.model.GBDTModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class BaseGBDTModelScorer extends AbstractScorer {
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseGBDTModelScorer.class);
+
+    public BaseGBDTModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public synchronized void loadModel() {
+        doLoadModel(GBDTModel.class);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseLRModelScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseLRModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRModelScorer.class);
+
+    public BaseLRModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(LRModel.class);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseThompsonSamplingScorer.java

@@ -0,0 +1,32 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.ThompsonSamplingModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+
+public abstract class BaseThompsonSamplingScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseThompsonSamplingScorer.class);
+
+    public BaseThompsonSamplingScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(ThompsonSamplingModel.class);
+    }
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                           final Map<String, String> userFeatureMap,
+                                           final List<RankItem> rankItems){
+
+        return rankItems;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScoreParam.java

@@ -0,0 +1,13 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
+import lombok.Data;
+
+/**
+ * @author dyp
+ */
+@Data
+public class ScoreParam {
+    private String mid;
+    private RequestContext requestContext;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerConfig.java

@@ -0,0 +1,140 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+public class ScorerConfig {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
+    private final List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
+
+    public ScorerConfig(Config config) {
+        this.load(config);
+    }
+
+    public ScorerConfig(String configFile) {
+        this.load(configFile);
+    }
+
+    public ScorerConfig() {
+    }
+
+    public static Config mergeConfig(Config baselineConfig, List<Config> expConfigs) {
+        if (expConfigs == null || expConfigs.size() == 0) {
+            return baselineConfig;
+        }
+        Config resultConfig = expConfigs.get(0);
+        for (Config config : expConfigs) {
+            resultConfig = resultConfig.withFallback(config);
+        }
+        resultConfig = resultConfig.withFallback(baselineConfig);
+        return resultConfig;
+    }
+
+    public boolean load(String configFile) {
+        Config config = ConfigFactory.parseResources(configFile);
+        return load(config);
+    }
+
+    public boolean load(Config baselineConfig, List<Config> expConfigs) {
+        Config config = mergeConfig(baselineConfig, expConfigs);
+        return load(config);
+    }
+
+    public boolean load(Config config) {
+        Config scorerConfig = config.getConfig("scorer-config");
+
+        try {
+            loadScorers(scorerConfig);
+            int pos = 0;
+            for (ScorerConfigInfo scorerConfigInfo : configInfoList) {
+                LOGGER.debug("scorer at position [{}], priority [{}], scorer name [{}]",
+                        pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName());
+            }
+            LOGGER.debug("Load scorer config success");
+        } catch (Exception e) {
+            LOGGER.error("Load scorer config failed, [{}]", ExceptionUtils.getFullStackTrace(e));
+            return false;
+        }
+
+        return true;
+    }
+
+    public List<ScorerConfigInfo> getConfigInfoList() {
+        return configInfoList;
+    }
+
+
+    public String loadOptionStringConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getString(path) : null;
+    }
+
+    private Config loadOptionConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
+    }
+
+    private void loadScorers(Config config) throws Exception {
+
+        ConfigObject confObj = config.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            Config conf = ((ConfigObject) it.getValue()).toConfig();
+            // parse config
+            String configName = it.getKey();
+            String scorerName = conf.getString("scorer-name");
+            int scorerPriority = 0;
+            if (conf.hasPath("scorer-priority"))
+                scorerPriority = conf.getInt("scorer-priority");
+            Boolean disableSwitch = false;
+            if (conf.hasPath("disable-switch")) {
+                disableSwitch = conf.getBoolean("disable-switch");
+            }
+            Config paramConfig = loadOptionConfig(conf, "param-config");
+            // model path
+            String modelPath = loadOptionStringConfig(conf, "model-path");
+            if (modelPath == null) {
+                modelPath = loadOptionStringConfig(conf, "default-model-path");
+                LOGGER.debug("model-path is not exists in config file, use default-model-path instead, modelPath={}", modelPath);
+            }
+            // enable queues
+            Set<String> enableQueues = new HashSet<String>();
+            if (conf.hasPath("enable-queues")) {
+                enableQueues.addAll(conf.getStringList("enable-queues"));
+            }
+            ScorerConfigInfo configInfo = new ScorerConfigInfo(configName,
+                    scorerName,
+                    scorerPriority,
+                    disableSwitch,
+                    enableQueues,
+                    modelPath,
+                    paramConfig
+            );
+            LOGGER.debug("parse scorer config info [{}]", configInfo);
+            // add to ConfigInfoList
+            addConfigByPriority(configInfoList, configInfo);
+        }
+    }
+
+    private void addConfigByPriority(List<ScorerConfigInfo> configInfoList, ScorerConfigInfo addConfigInfo) {
+
+        int pos = 0;
+        for (; pos < configInfoList.size(); pos++) {
+            if (configInfoList.get(pos).getScorerPriority() <= addConfigInfo.getScorerPriority()) {
+                break;
+            }
+        }
+
+        configInfoList.add(pos, addConfigInfo);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerConfigInfo.java

@@ -0,0 +1,78 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.google.gson.Gson;
+import com.typesafe.config.Config;
+
+import java.util.Set;
+
+
+public class ScorerConfigInfo {
+
+    private String configName;
+    private Integer scorerPriority;
+    private Boolean disableSwitch;
+    private String scorerName;
+    private Set<String> enableQueues;
+    private String modelPath;
+    private Config paramConfig; // param config
+
+    public ScorerConfigInfo(String configName,
+                            String scorerName,
+                            Integer scorerPriority,
+                            Boolean disableSwitch,
+                            Set<String> enableQueues,
+                            String modelPath,
+                            Config paramConfig) {
+
+        this.configName = configName;
+        this.scorerName = scorerName;
+        this.scorerPriority = scorerPriority;
+        this.disableSwitch = disableSwitch;
+        this.enableQueues = enableQueues;
+        this.modelPath = modelPath;
+        this.paramConfig = paramConfig;
+    }
+
+    public Config getParamConfig() {
+        return paramConfig;
+    }
+
+    public Set<String> getEnableQueues() {
+        return enableQueues;
+    }
+
+    public void setEnableQueues(Set<String> enableQueues) {
+        this.enableQueues = enableQueues;
+    }
+
+    public Integer getScorerPriority() {
+        return scorerPriority;
+    }
+
+    public String getScorerName() {
+        return scorerName;
+    }
+
+    public String getConfigName() {
+        return configName;
+    }
+
+    public boolean isQueueEnable(String queueName) {
+        return this.enableQueues == null ||
+                this.enableQueues.isEmpty() ||
+                this.enableQueues.contains(queueName);
+    }
+
+    public String getModelPath() {
+        return modelPath;
+    }
+
+    public Boolean getDisableSwitch() {
+        return disableSwitch;
+    }
+
+    @Override
+    public String toString() {
+        return new Gson().toJson(this);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerPipeline.java

@@ -0,0 +1,187 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+
+@Slf4j
+public class ScorerPipeline {
+    public static final int corePoolSize = 128;
+    public static final int SCORE_TIME_OUT = 400;
+    public static final Logger LOGGER = LoggerFactory.getLogger(ScorerPipeline.class);
+    public static final ExecutorService executorService = Executors.newFixedThreadPool(corePoolSize);
+
+    public List<AbstractScorer> scorers;
+
+    public ScorerPipeline(List<AbstractScorer> scorers) {
+        this.scorers = scorers;
+    }
+
+    /**
+     * scoring
+     *
+     * @return
+     */
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+
+        if (CollectionUtils.isEmpty(scorers)) {
+            log.error("scorers is empty");
+            return rankItems;
+        }
+        List<RankItem> items = rankItems;
+
+        for (final AbstractScorer scorer : scorers) {
+            if (!scorer.isEnable()) {
+                continue;
+            }
+
+            final int beforeSize = rankItems.size();
+            final long startTime = System.currentTimeMillis();
+
+            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
+            String[] scorerNames = fullScorerName.split("\\.");
+            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
+
+            final List<RankItem> scoreRankerItems = items;
+            Callable<List<RankItem>> callable = () -> scorer.scoring(param, userFeature, scoreRankerItems);
+
+            // execute score use thread to protected score worst time
+            List<RankItem> scoredItems = new ArrayList<RankItem>();
+            try {
+                List<Future<List<RankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
+                for (Future<List<RankItem>> future : futures) {
+                    try {
+                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                            scoredItems.addAll(future.get());
+                        } else {
+                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
+                                    new Object[]{scorerName, scoreRankerItems.size()});
+                        }
+                    } catch (Exception e) {
+                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
+                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+                    }
+                }
+            } catch (Exception e) {
+                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
+                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+            }
+
+            //  变更item
+            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
+                items = scoreRankerItems;
+            } else {
+                items = new ArrayList<>(items);
+            }
+
+            int position = 0;
+            for (RankItem item : items) {
+                item.getRankerIndex().put(scorerName, position++);
+                item.getRankerScore().put(scorerName, item.getScore());
+            }
+
+            //
+            long spentTime = System.currentTimeMillis() - startTime;
+            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
+                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
+        }
+
+        int position = 0;
+        for (RankItem item : items) {
+            item.getRankerIndex().put("finalScore", position++);
+            item.getRankerScore().put("finalScore", item.getScore());
+        }
+
+        return items;
+    }
+
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                  final Map<String, String> userFeatureMap,
+                                  final List<RankItem> rankItems) {
+
+        if (CollectionUtils.isEmpty(scorers)) {
+            log.error("scorers is empty");
+            return rankItems;
+        }
+        List<RankItem> items = rankItems;
+
+        for (final AbstractScorer scorer : scorers) {
+            if (!scorer.isEnable()) {
+                continue;
+            }
+
+            final int beforeSize = rankItems.size();
+            final long startTime = System.currentTimeMillis();
+
+            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
+            String[] scorerNames = fullScorerName.split("\\.");
+            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
+
+            final List<RankItem> scoreRankerItems = items;
+            Callable<List<RankItem>> callable = () -> scorer.scoring(sceneFeatureMap, userFeatureMap, scoreRankerItems);
+
+            // execute score use thread to protected score worst time
+            List<RankItem> scoredItems = new ArrayList<RankItem>();
+            try {
+                List<Future<List<RankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
+                for (Future<List<RankItem>> future : futures) {
+                    try {
+                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                            scoredItems.addAll(future.get());
+                        } else {
+                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
+                                    new Object[]{scorerName, scoreRankerItems.size()});
+                        }
+                    } catch (Exception e) {
+                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
+                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+                    }
+                }
+            } catch (Exception e) {
+                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
+                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+            }
+
+            //  变更item
+            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
+                items = scoreRankerItems;
+            } else {
+                items = new ArrayList<>(items);
+            }
+
+            int position = 0;
+            for (RankItem item : items) {
+                item.getRankerIndex().put(scorerName, position++);
+                if(scorerName.equals("finalScore")) {
+                    item.getRankerScore().put(scorerName, item.getScore());
+                }
+            }
+
+            //
+            long spentTime = System.currentTimeMillis() - startTime;
+            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
+                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
+        }
+
+        int position = 0;
+        for (RankItem item : items) {
+            item.getRankerIndex().put("finalScore", position++);
+            item.getRankerScore().put("finalScore", item.getScore());
+        }
+
+        return items;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerUtils.java

@@ -0,0 +1,151 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+
+import com.typesafe.config.Config;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+@Slf4j
+public final class ScorerUtils {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
+
+    private static final Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
+
+    public static String BASE_CONF_FEED = "feeds_score_config_new_baseline.conf";
+
+    public static String FLOWPOOL_CONF = "feeds_score_config_thompson_new.conf";
+
+
+    public static void warmUp() {
+        log.info("scorer warm up ");
+        ScorerUtils.init(BASE_CONF_FEED);
+        ScorerUtils.init(FLOWPOOL_CONF);
+    }
+
+    private ScorerUtils() {
+        // init(BASE_CONF);
+    }
+
+    public static void init(String configFile) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(configFile);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+
+        initLoadModel(scorers);
+        scorerPipelineCache.put(configFile, new ScorerPipeline(scorers));
+    }
+
+    /**
+     * init load model
+     *
+     * @param scorers
+     */
+    public static void initLoadModel(List<AbstractScorer> scorers) {
+        for (AbstractScorer scorer : scorers) {
+            if (scorer.isEnable()) {
+                scorer.loadModel();
+            }
+        }
+    }
+
+    public static void initLoadModel(String configFile) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(configFile);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        initLoadModel(scorers);
+    }
+
+
+    public static void initLoadModel(Config config) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(config);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        initLoadModel(scorers);
+    }
+
+    public static ScorerPipeline getScorerPipeline(String configFile) {
+        // 不需要保证严格意义的单例
+        if (scorerPipelineCache.containsKey(configFile)) {
+            return scorerPipelineCache.get(configFile);
+        }
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(configFile);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        ScorerPipeline pipeline = new ScorerPipeline(scorers);
+        scorerPipelineCache.put(configFile, pipeline);
+        return pipeline;
+    }
+
+    public static ScorerPipeline getScorerPipeline(Config mergeConfig) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(mergeConfig);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        return new ScorerPipeline(scorers);
+    }
+
+    public static ScorerPipeline getScorerPipeline(Config baselineConfig, List<Config> configList) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(baselineConfig, configList);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        return new ScorerPipeline(scorers);
+    }
+
+    /**
+     * construct scorers
+     *
+     * @param scorerConfig
+     * @return
+     */
+    public static List<AbstractScorer> constructScorers(ScorerConfig scorerConfig) {
+
+        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
+        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
+            if (!configInfo.getDisableSwitch()) {
+                try {
+                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
+                            .getConstructor(ScorerConfigInfo.class)
+                            .newInstance(configInfo);
+                    scorers.add(scorer);
+                    LOGGER.debug("construct score [{}]", configInfo.getScorerName());
+                } catch (Exception e) {
+                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
+                }
+
+            }
+        }
+        return scorers;
+    }
+
+    /**
+     * construct scorers
+     *
+     * @param scorerConfig
+     * @return
+     */
+    public static List<AbstractScorer> constructQueueScorers(ScorerConfig scorerConfig, String queueName) {
+
+        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
+        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
+            if (!configInfo.getDisableSwitch() && configInfo.isQueueEnable(queueName)) {
+                try {
+                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
+                            .getConstructor(ScorerConfigInfo.class)
+                            .newInstance(configInfo);
+                    scorers.add(scorer);
+                    LOGGER.debug("construct queue scorer [{}] [{}]", queueName, configInfo.getScorerName());
+                } catch (Exception e) {
+                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        return scorers;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/AbstractUserAttentionExtractor.java

@@ -0,0 +1,28 @@
+package com.tzld.piaoquan.recommend.server.framework.userattention;
+
+
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+public abstract class AbstractUserAttentionExtractor {
+    protected UserAttentionExtractorConfig extractorConfig;
+
+    public AbstractUserAttentionExtractor(UserAttentionExtractorConfig extractorConfig) {
+        this.extractorConfig = extractorConfig;
+    }
+
+    public String getExtractorName() {
+        String[] arr = this.extractorConfig.getName().split(".");
+        return arr.length > 0 ? arr[arr.length - 1] : "UnKnownExtractor";
+    }
+
+    public UserAttentionExtractorConfig getExtractorConfig() {
+        return extractorConfig;
+    }
+
+
+    public abstract void extractAttention(final RecommendRequest requestData, final User user);
+
+    public void loadModel() {
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorConfig.java

@@ -0,0 +1,44 @@
+package com.tzld.piaoquan.recommend.server.framework.userattention;
+
+import com.typesafe.config.Config;
+
+
+
+public class UserAttentionExtractorConfig {
+    private final String name;  // attention extractor name
+    private final Integer priority;
+    private final String modelPath;
+    private final Config paramConfig; // param config
+
+    public UserAttentionExtractorConfig(String name,
+                                        Config paramConfig,
+                                        Integer priority,
+                                        String modelPath) {
+        this.name = name;
+        this.paramConfig = paramConfig;
+        this.priority = priority;
+        this.modelPath = modelPath;
+    }
+
+    public Integer getPriority() {
+        return priority;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Config getParamConfig() {
+        return paramConfig;
+    }
+
+    @Override
+    public String toString() {
+        return "AttentionExtractorConfig{" +
+                "name='" + name + '\'' +
+                ", priority=" + priority +
+                ", modelPath='" + modelPath + '\'' +
+                ", paramConfig=" + paramConfig +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorPipeline.java

@@ -0,0 +1,36 @@
+package com.tzld.piaoquan.recommend.server.framework.userattention;
+
+
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.common.UserAttention;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class UserAttentionExtractorPipeline {
+    private List<AbstractUserAttentionExtractor> extractorList = new ArrayList<AbstractUserAttentionExtractor>();
+
+    public UserAttentionExtractorPipeline(List<AbstractUserAttentionExtractor> extractorList) {
+        this.extractorList = extractorList;
+    }
+
+    public void extractAttention(final RecommendRequest requestData, final User user) {
+        if (user.getUserAttention() == null) {
+            UserAttention userAttention = new UserAttention();
+            user.setUserAttention(userAttention);
+        }
+
+        long startTime = System.currentTimeMillis();
+
+        for (AbstractUserAttentionExtractor extractor : extractorList) {
+            try {
+                long currStartTime = System.currentTimeMillis();
+                extractor.extractAttention(requestData, user);
+                // log
+            } catch (Exception e) {
+                // log
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorUtils.java

@@ -0,0 +1,134 @@
+package com.tzld.piaoquan.recommend.server.framework.userattention;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class UserAttentionExtractorUtils {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(UserAttentionExtractorUtils.class);
+
+    public static final String BASE_CONF = "";
+
+    /**
+     * 主调用函数
+     *
+     * @param config
+     */
+    public static void initLoadModel(Config config) {
+        initLoadModel(constructExtractorList(constructExtractorConfigList(config)));
+    }
+
+
+    public static UserAttentionExtractorPipeline getAtttentionPipeline(String configFile) {
+
+        Config attentionConfig = ConfigFactory.parseResources(configFile);
+        List<AbstractUserAttentionExtractor> attentionExtractors = constructExtractorList(constructExtractorConfigList(attentionConfig));
+        UserAttentionExtractorPipeline pipeline = new UserAttentionExtractorPipeline(attentionExtractors);
+
+        return pipeline;
+    }
+
+
+    public static List<UserAttentionExtractorConfig> constructExtractorConfigList(Config config) {
+        long startTime = System.currentTimeMillis();
+        List<UserAttentionExtractorConfig> configList = new ArrayList<UserAttentionExtractorConfig>();
+        try {
+            Config mergedConfig = config.getConfig("attention-extractor-config");
+            for (ConfigObject.Entry<String, ConfigValue> it : mergedConfig.root().entrySet()) {
+                LOGGER.debug("deal with base config, attention key: [{}]", it.getKey());
+                Config entryConf = ((ConfigObject) it.getValue()).toConfig();
+                UserAttentionExtractorConfig extractorConfig = parseExtractorConfig(entryConf);
+                addConfigByPriority(configList, extractorConfig);
+            }
+            // debug log
+            if (LOGGER.isDebugEnabled()) {
+                for (UserAttentionExtractorConfig extractorConfig : configList) {
+                    LOGGER.debug("attention extractor config [{}] [{}]", extractorConfig.getName(), extractorConfig);
+                }
+            }
+        } catch (Exception e) {
+            LOGGER.error("construct extractor config {} failed {}", config.toString(), ExceptionUtils.getFullStackTrace(e));
+        }
+        return configList;
+    }
+
+
+    private static List<AbstractUserAttentionExtractor> constructExtractorList(List<UserAttentionExtractorConfig> configList) {
+        long startTime = System.currentTimeMillis();
+        List<AbstractUserAttentionExtractor> extractorList = new ArrayList<AbstractUserAttentionExtractor>();
+        for (UserAttentionExtractorConfig config : configList) {
+            try {
+                AbstractUserAttentionExtractor extractor = (AbstractUserAttentionExtractor) Class.forName(config.getName())
+                        .getConstructor(UserAttentionExtractorConfig.class)
+                        .newInstance(config);
+                extractorList.add(extractor);
+            } catch (Exception e) {
+                LOGGER.error("instance extractor {} failed {}", config.getName(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        return extractorList;
+    }
+
+    private static String loadOptionStringConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getString(path) : null;
+    }
+
+    private static Config loadOptionConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
+    }
+
+    private static int loadOptionIntConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getInt(path) : 0;
+    }
+
+    /**
+     * attention层的配置化参数
+     */
+    private static UserAttentionExtractorConfig parseExtractorConfig(Config extractorConfig) {
+        String attentionExtractorName = extractorConfig.getString("name");
+        Integer priority = loadOptionIntConfig(extractorConfig, "attention-priority");
+        Config paramConfig = loadOptionConfig(extractorConfig, "param-config");
+        String modelPath = loadOptionStringConfig(extractorConfig, "model-path");
+        return new UserAttentionExtractorConfig(attentionExtractorName,
+                paramConfig,
+                priority,
+                modelPath);
+    }
+
+
+    /**
+     * attention支持模型获取
+     *
+     * @param attentionExtractors
+     */
+    private static void initLoadModel(List<AbstractUserAttentionExtractor> attentionExtractors) {
+        for (AbstractUserAttentionExtractor attentionExtractor : attentionExtractors) {
+            attentionExtractor.loadModel();
+        }
+    }
+
+    /**
+     * 按优先级处理逻辑
+     *
+     * @param configInfoList
+     * @param extractorConfig
+     */
+    private static void addConfigByPriority(List<UserAttentionExtractorConfig> configInfoList, UserAttentionExtractorConfig extractorConfig) {
+        int pos = 0;
+        for (; pos < configInfoList.size(); pos++) {
+            if (configInfoList.get(pos).getPriority() <= extractorConfig.getPriority()) {
+                break;
+            }
+        }
+        configInfoList.add(pos, extractorConfig);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/FixedThreadPoolHelper.java

@@ -0,0 +1,11 @@
+package com.tzld.piaoquan.recommend.server.framework.utils;
+
+public class FixedThreadPoolHelper extends ThreadPoolHelper {
+    public FixedThreadPoolHelper(int nThread) {
+        super(ThreadPoolUtils.newFixedThreadPool(nThread));
+    }
+
+    public FixedThreadPoolHelper(int nThread, String name) {
+        super(ThreadPoolUtils.newFixedThreadPool(nThread, name), "FixedPool_" + name);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/RedisSmartClient.java

@@ -0,0 +1,64 @@
+package com.tzld.piaoquan.recommend.server.framework.utils;
+
+
+import com.google.common.base.Function;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Service;
+import redis.clients.jedis.JedisCluster;
+import redis.clients.jedis.Tuple;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Service
+public class RedisSmartClient {
+
+    private final static ExecutorService executorService = Executors.newCachedThreadPool();
+    @Qualifier("redisTemplate")
+    @Autowired
+    protected RedisTemplate<String, String> redisTemplate;
+
+
+    public RedisSmartClient() {
+    }
+    public void set(final String key, final String value) {
+        redisTemplate.opsForValue().set(key, value);
+    }
+
+    public String get(final String key) {
+        return redisTemplate.opsForValue().get(key);
+    }
+
+    // hget
+    public String hget(final String key, final String field) {
+        return redisTemplate.opsForHash().get(key, field).toString();
+    }
+
+
+    // getRandomValueFromSet
+    public List<String> getRandomList(final String key, long num) {
+        return redisTemplate.opsForSet().randomMembers(key, num);
+    }
+
+
+    // hgetall
+    public Map<Object, Object> hgetall(String key) {
+        return redisTemplate.opsForHash().entries(key);
+    }
+
+    // TODO 异步写入redis 作为用户的分发历史
+    public <T> T write(String key, String value) {
+        T t = null;
+        return t;
+    }
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/ThreadPoolHelper.java

@@ -0,0 +1,64 @@
+package com.tzld.piaoquan.recommend.server.framework.utils;
+
+
+import lombok.Data;
+
+import java.util.Timer;
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * 线程池Helper
+ *   可以命名线程池; 保存ThreadPoolExecutor实例, 方便监控线程池信息
+ */
+@Data
+public class ThreadPoolHelper {
+    private String name;
+    private ThreadPoolExecutor threadPoolExecutor;
+    private Timer counterTimer;
+
+    ThreadPoolHelper(ThreadPoolExecutor threadPoolExecutor) {
+        this.threadPoolExecutor = threadPoolExecutor;
+        this.name = "";
+    }
+
+    ThreadPoolHelper(ThreadPoolExecutor threadPoolExecutor, String name) {
+        this.threadPoolExecutor = threadPoolExecutor;
+        this.name = name;
+    }
+
+
+    public long getTaskCount() {
+        return this.threadPoolExecutor.getTaskCount();
+    }
+
+    public long getActiveCount() {
+        return this.threadPoolExecutor.getActiveCount();
+    }
+
+    public long getCorePoolSize() {
+        return this.threadPoolExecutor.getCorePoolSize();
+    }
+
+    public long getPoolSize() {
+        return this.threadPoolExecutor.getPoolSize();
+    }
+
+    public long getMaximumPoolSize() {
+        return this.threadPoolExecutor.getMaximumPoolSize();
+    }
+
+    public long getLargestPoolSize() {
+        return this.threadPoolExecutor.getLargestPoolSize();
+    }
+
+    public long getTaskQueueSize() {
+        return this.threadPoolExecutor.getQueue().size();
+    }
+
+    public void cancelCounter() {
+        if (counterTimer != null) {
+            counterTimer.cancel();
+            counterTimer.purge();
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/ThreadPoolUtils.java

@@ -0,0 +1,46 @@
+package com.tzld.piaoquan.recommend.server.framework.utils;
+
+
+import java.util.concurrent.*;
+
+/**
+ * 命名线程池工场类
+ * 最后创建的线程名称:
+ *   name_{counter}
+ */
+class NamedThreadFactory implements ThreadFactory {
+    private int counter = 0;
+    private String name;
+    NamedThreadFactory(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public Thread newThread(Runnable r) {
+        counter += 1;
+        return new Thread(r, name + "_" + counter);
+    }
+}
+
+
+public class ThreadPoolUtils {
+    public static ThreadPoolExecutor newFixedThreadPool(int nThreads, String poolName) {
+        return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
+                new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("Fixed_" + poolName));
+    }
+    public static ThreadPoolExecutor newFixedThreadPool(int nThreads) {
+        return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
+                new LinkedBlockingQueue<Runnable>());
+    }
+
+    public static ThreadPoolExecutor newCachedThreadPool(String poolName) {
+        return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
+                new SynchronousQueue<Runnable>(), new NamedThreadFactory("Cached_" + poolName));
+    }
+
+    public static ThreadPoolExecutor newCachedThreadPool() {
+        return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
+                new SynchronousQueue<Runnable>());
+    }
+}
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/grpcservice/RecommendGrpcService.java

@@ -3,7 +3,7 @@ package com.tzld.piaoquan.recommend.server.grpcservice;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendResponse;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendServiceGrpc;
-import com.tzld.piaoquan.recommend.server.service.RecommendService;
+import com.tzld.piaoquan.recommend.server.service.VideoRecommendService;
 import io.grpc.stub.StreamObserver;
 import net.devh.boot.grpc.server.service.GrpcService;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -14,18 +14,18 @@ import org.springframework.beans.factory.annotation.Autowired;
 @GrpcService
 public class RecommendGrpcService extends RecommendServiceGrpc.RecommendServiceImplBase {
     @Autowired
-    private RecommendService recommendService;
+    private VideoRecommendService videoRecommendService;
 
     @Override
     public void homepageRecommend(RecommendRequest request, StreamObserver<RecommendResponse> responseObserver) {
-        RecommendResponse response = recommendService.homepageRecommend(request);
+        RecommendResponse response = videoRecommendService.homepageRecommend(request);
         responseObserver.onNext(response);
         responseObserver.onCompleted();
     }
 
     @Override
     public void relevantRecommend(RecommendRequest request, StreamObserver<RecommendResponse> responseObserver) {
-        RecommendResponse response = recommendService.relevantRecommend(request);
+        RecommendResponse response = videoRecommendService.relevantRecommend(request);
         responseObserver.onNext(response);
         responseObserver.onCompleted();
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/FlowPoolRecommendPipeline.java

@@ -0,0 +1,211 @@
+package com.tzld.piaoquan.recommend.server.implement;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.MachineInfo;
+import com.tzld.piaoquan.recommend.server.model.RecommendParam;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.RecommendService;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.FlowPoolWithLevelRecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.FlowPoolWithLevelScoreRecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.FlowPoolWithScoreRecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+@Service
+public class FlowPoolRecommendPipeline implements ApplicationContextAware {
+
+    private final Logger log = LoggerFactory.getLogger(FlowPoolRecommendPipeline.class);
+
+    private final static Map<String, RecallStrategy> strategyMap = new HashMap<>();
+
+    @Resource
+    private RecommendService recommendService;
+
+    private ApplicationContext applicationContext;
+
+    private final ExecutorService pool = ThreadPoolFactory.recallPool();
+
+    @PostConstruct
+    public void init() {
+        Map<String, RecallStrategy> type = applicationContext.getBeansOfType(RecallStrategy.class);
+        for (Map.Entry<String, RecallStrategy> entry : type.entrySet()) {
+            RecallStrategy value = entry.getValue();
+            strategyMap.put(value.getClass().getSimpleName(), value);
+        }
+    }
+
+
+    public List<Video> feedByRec(final RecommendRequest requestData,
+                                 final int recommendType, Boolean logPrint) {
+        List<RecallStrategy> strategies = new ArrayList<>();
+
+        RecommendParam param = recommendService.genRecommendParam(requestData, recommendType);
+        RecallParam recallParam = recommendService.convertToRecallParam(param);
+        RecallResult recallResult = getRecallResult(strategies, recallParam);
+        if (logPrint) {
+            log.info("traceId = {}, recallResult [{}]", requestData.getRequestId(),
+                    JSONObject.toJSONString(recallResult));
+        }
+
+        if (recallResult == null || CollectionUtils.isEmpty(recallResult.getData())) {
+            return Collections.emptyList();
+        }
+
+        RankParam rankParam = recommendService.convertToRankParam(param, recallResult);
+        List<Video> videoList = mergeAndRankFlowPoolRecall(rankParam);
+        if (logPrint) {
+            log.info("traceId = {}, videoList [{}]", requestData.getRequestId(),
+                    JSONObject.toJSONString(videoList));
+        }
+        return videoList;
+    }
+
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+        return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
+    }
+
+    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
+
+        //初始化 userid
+        UserFeature userFeature = new UserFeature();
+        userFeature.setMid(param.getMid());
+
+        // 初始化RankItem
+        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                .filter(d -> d.getPushFrom().equals(pushFrom))
+                .findFirst();
+        if (!data.isPresent()) {
+            return Collections.emptyList();
+        }
+        List<Video> videoList = data.get().getVideos();
+        if (videoList == null) {
+            return Collections.emptyList();
+        }
+        List<RankItem> rankItems = new ArrayList<>();
+        for (int i = 0; i < videoList.size(); i++) {
+            RankItem rankItem = new RankItem(videoList.get(i));
+            rankItems.add(rankItem);
+        }
+
+        // 初始化上下文参数
+        ScoreParam scoreParam = convert(param);
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
+                .scoring(scoreParam, userFeature, rankItems);
+
+        if (rovRecallScore == null) {
+            return Collections.emptyList();
+        }
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
+
+    protected ScoreParam convert(RankParam param) {
+        ScoreParam scoreParam = new ScoreParam();
+
+        scoreParam.setMid(param.getMid());
+
+        // TODO hardcode 为了兼容写入逻辑
+        RequestContext context = new RequestContext();
+        context.setApptype(param.getAppType() + "");
+
+        // TODO 地域转换
+        context.setRegion(param.getProvince());
+        context.setCity(param.getCity());
+
+        Calendar calendar = Calendar.getInstance();
+        context.setWeek((calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        context.setDay(new SimpleDateFormat("yyyyMMdd").format(calendar.getTime()));
+        context.setHour(new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        MachineInfo machineInfo = param.getMachineInfo();
+        if (machineInfo != null) {
+            context.setMachineinfo_brand(machineInfo.getBrand());
+            context.setMachineinfo_model(machineInfo.getModel());
+            context.setMachineinfo_platform(machineInfo.getPlatform());
+            context.setMachineinfo_sdkversion(machineInfo.getSdkVersion());
+            context.setMachineinfo_system(machineInfo.getSystem());
+            context.setMachineinfo_wechatversion(machineInfo.getWechatVersion());
+        }
+
+        scoreParam.setRequestContext(context);
+        return scoreParam;
+    }
+
+    private RecallResult getRecallResult(List<RecallStrategy> strategies, RecallParam param) {
+        if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL)) {
+            strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategy.class.getSimpleName()));
+        } else if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL_SCORE)) {
+            strategies.add(strategyMap.get(FlowPoolWithLevelScoreRecallStrategy.class.getSimpleName()));
+        } else {
+            strategies.add(strategyMap.get(FlowPoolWithScoreRecallStrategy.class.getSimpleName()));
+        }
+
+        CountDownLatch cdl = new CountDownLatch(strategies.size());
+        List<Future<RecallResult.RecallData>> recallResultFutures = new ArrayList<>();
+        for (final RecallStrategy strategy : strategies) {
+            Future<RecallResult.RecallData> future = pool.submit(() -> {
+                List<Video> result = strategy.recall(param);
+                cdl.countDown();
+                return new RecallResult.RecallData(strategy.pushFrom(), result);
+            });
+            recallResultFutures.add(future);
+        }
+        try {
+            cdl.await(3000, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            log.error("recall error", e);
+            return null;
+        }
+
+        List<RecallResult.RecallData> results = new ArrayList<>();
+        for (Future<RecallResult.RecallData> f : recallResultFutures) {
+            try {
+                RecallResult.RecallData data = f.get();
+                results.add(data);
+            } catch (Exception e) {
+                log.error("future get error ", e);
+            }
+        }
+
+        return new RecallResult(results);
+    }
+
+
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+        this.applicationContext = applicationContext;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/TopRecommendPipeline.java

@@ -0,0 +1,704 @@
+package com.tzld.piaoquan.recommend.server.implement;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.base.Stopwatch;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.MergeUtils;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueue;
+import com.tzld.piaoquan.recommend.server.framework.recaller.BaseRecaller;
+import com.tzld.piaoquan.recommend.server.framework.recaller.provider.RedisBackedQueue;
+import com.tzld.piaoquan.recommend.server.framework.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import org.apache.commons.collections4.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Service
+public class TopRecommendPipeline {
+
+    private static final Logger log = LoggerFactory.getLogger(TopRecommendPipeline.class);
+
+    public static final String MERGE_CONF = "merge_config.conf";
+
+    @Resource
+    private RedisSmartClient client;
+    @Resource
+    public RedisTemplate<String, String> redisTemplate;
+    private RedisBackedQueue queueProvider;
+
+    @PostConstruct
+    public void init() {
+        queueProvider = new RedisBackedQueue(client, 15 * 60 * 1000L);
+    }
+
+    public List<Video> feeds(final RecommendRequest requestData,
+                             final int requestIndex,
+                             final User userInfo, Boolean logPrint) {
+        // Step 1: Attention extraction
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        stopwatch.reset().start();
+        List<RankItem> rankItems = feedByRec(requestData, requestIndex, userInfo, logPrint);
+        if (rankItems == null || rankItems.isEmpty()) {
+            return new ArrayList<>();
+        }
+        if (logPrint) {
+            log.info("traceId = {}, cost = {}, feeds rankItems = {}", requestData.getRequestId(),
+                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(rankItems));
+        }
+        stopwatch.reset().start();
+        List<Video> videos = rankItem2Video(rankItems);
+        if (logPrint) {
+            log.info("traceId = {}, cost = {}, videos = {}", requestData.getRequestId(),
+                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(videos));
+        }
+        return videos;
+    }
+
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+
+    public List<RankItem> feedByRec(final RecommendRequest requestData,
+                                    final int requestIndex,
+                                    final User userInfo, Boolean logPrint) {
+        int recallNum = 150;
+
+        Stopwatch stopwatch = Stopwatch.createStarted();
+
+        // Step 2: create top queue
+        stopwatch.reset().start();
+        StrategyQueue topQueue = MergeUtils.createTopQueue(MERGE_CONF, "top-queue");
+        if (logPrint) {
+            log.info("traceId = {}, cost = {}, topQueue = {}", requestData.getRequestId(),
+                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(topQueue));
+        }
+
+        // Step 3: Candidate
+        stopwatch.reset().start();
+        Map<String, Candidate> candidates = new HashMap<String, Candidate>();
+        topQueue.candidate(candidates, recallNum, userInfo, requestData, 0, 0);
+        if (logPrint) {
+            log.info("traceId = {}, cost = {}, candidates = {}", requestData.getRequestId(),
+                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(candidates));
+        }
+
+
+        // Step 4: Recalling & Basic Scoring
+        stopwatch.reset().start();
+        BaseRecaller recaller = new BaseRecaller(queueProvider);
+        List<RankItem> items = recaller.recalling(requestData, userInfo, new ArrayList<>(candidates.values()));
+        if (logPrint) {
+            log.info("traceId = {}, cost = {}, items = {}", requestData.getRequestId(),
+                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(items));
+        }
+
+
+        // Step 4: Advance Scoring
+//        timestamp = System.currentTimeMillis();
+//        ScorerPipeline scorerPipeline = getScorerPipeline(requestData);
+//        items = scorerPipeline.scoring(requestData, userInfo, requestIndex, items);
+        if (CollectionUtils.isEmpty(items)) {
+            return new ArrayList<>();
+        }
+
+        stopwatch.reset().start();
+        // Step 5: Merger
+        MergeUtils.distributeItemsToMultiQueues(topQueue, items);
+        topQueue.merge(recallNum * 3, userInfo, requestData, requestIndex, 0);
+
+        // 多样性融合
+        List<RankItem> mergeItems = topQueue.getItems();
+        if (CollectionUtils.isEmpty(mergeItems)) {
+            return new ArrayList<>();
+        }
+        duplicate(mergeItems);
+
+        if (logPrint) {
+            log.info("traceId = {}, cost = {}, mergeItems = {}", requestData.getRequestId(),
+                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(mergeItems));
+        }
+//        MergeUtils.diversityRerank(mergeItems, SimilarityUtils.getIsSameUserTagOrCategoryFunc(), recallNum, 6, 2);
+
+        // Step 6: Global Rank & subList
+        // TODO 前置和后置处理逻辑 hardcode，后续优化
+        stopwatch.reset().start();
+        List<RankItem> rovRecallRankNewScore = rankByScore(mergeItems, requestData);
+        if (logPrint) {
+            log.info("traceId = {}, cost = {}, rovRecallRankNewScore = {}", requestData.getRequestId(),
+                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(rovRecallRankNewScore));
+        }
+
+        return rovRecallRankNewScore;
+    }
+
+    private List<Video> rankItem2Video(List<RankItem> items) {
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : items){
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0) *
+                    Math.log(1 + item.scoresMap.getOrDefault("allreturnsScore", 0.0));
+            Video video = new Video();
+            video.setVideoId(Long.parseLong(item.getId()));
+            video.setPushFrom(item.getQueue());
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+
+    private void duplicate(List<RankItem> items) {
+        Set<String> ids = new HashSet<>();
+        List<RankItem> result = new ArrayList<>();
+        for (RankItem item : items) {
+            if (ids.contains(item.getId())) {
+                continue;
+            }
+            ids.add(item.getId());
+            result.add(item);
+        }
+        items.clear();
+        items.addAll(result);
+    }
+
+    public List<RankItem> rankByScore(List<RankItem> rankItems, RecommendRequest param){
+        List<RankItem> result = new ArrayList<>();
+        if (rankItems.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.getClass().getSimpleName(), e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.getClass().getSimpleName());
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<String> videoIds = CommonCollectionUtils.toListDistinct(rankItems, RankItem::getId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.getClass().getSimpleName(), e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}",
+                            this.getClass().getSimpleName(), e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}",
+                            this.getClass().getSimpleName(), e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF_FEED)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.getClass().getSimpleName());
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getUserFeatureMap(RecommendRequest param, List<RankItem> rankItems) {
+        Map<String, String> userFeatureMap = new HashMap<>(64);
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.getClass().getSimpleName(), e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.getClass().getSimpleName());
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(rankItems, RankItem::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.getClass().getSimpleName(), e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                ++j;
+                if (j >= rankItems.size()) {
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try {
+                    String vF = videoRtFeatures.get(j);
+                    if (vF == null) {
+                        continue;
+                    }
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
+                    }, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
+                        String value = entry.getValue();
+                        if (value == null) {
+                            continue;
+                        }
+                        String[] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1) {
+                            String[] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}",
+                            this.getClass().getSimpleName(), e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                ++j;
+                if (j >= rankItems.size()) {
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try {
+                    String vF = videoRtFeatures.get(j);
+                    if (vF == null) {
+                        continue;
+                    }
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
+                    }, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
+                        String value = entry.getValue();
+                        if (value == null) {
+                            continue;
+                        }
+                        String[] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1) {
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}",
+                            this.getClass().getSimpleName(), e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+        return userFeatureMap;
+    }
+
+    private Map<String, String> getSceneFeature(RecommendRequest param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/attention/SimpleAttentionExtractor.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.implement.attention;
+
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.userattention.AbstractUserAttentionExtractor;
+import com.tzld.piaoquan.recommend.server.framework.userattention.UserAttentionExtractorConfig;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+public class SimpleAttentionExtractor extends AbstractUserAttentionExtractor {
+
+    public SimpleAttentionExtractor(UserAttentionExtractorConfig config) {
+        super(config);
+    }
+
+    @Override
+    public void extractAttention(RecommendRequest recommendRequestData, User user) {
+        //TODO
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global1hHotCandidate.java

@@ -0,0 +1,62 @@
+package com.tzld.piaoquan.recommend.server.implement.candidate;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.math.BigDecimal;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Global1hHotCandidate extends IndexCandidateQueue {
+
+    public static final String ItemType = "video";
+
+    public Global1hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int currRecall = 0;
+        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
+
+
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.6)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+
+
+        QueueName queueName = new QueueName(ItemType, "rov")
+                .addMatch("type", "global1h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
+
+        queueName = new QueueName(ItemType, "short_rov")
+                .addMatch("type", "global1h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
+
+        queueName = new QueueName(ItemType, "short_ros")
+                .addMatch("type", "global1h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum, getStrategyQueueInfo().getQueueName());
+
+
+        candidates.putAll(simplifiedCandidates);
+        return currRecall;
+
+    }
+
+    @Override
+    public String toString() {
+        return "Global24hHotCandidate{" +
+                "myCandidates=" + myCandidates +
+                ", rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global24hHotCandidate.java

@@ -0,0 +1,56 @@
+package com.tzld.piaoquan.recommend.server.implement.candidate;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.math.BigDecimal;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Global24hHotCandidate extends IndexCandidateQueue {
+
+    public static final String ItemType = "video";
+
+    public Global24hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int currRecall = 0;
+        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<>();
+
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.6)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
+
+
+        QueueName queueName = new QueueName(ItemType, "rov")
+                .addMatch("type", "global24h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
+
+
+        queueName = new QueueName(ItemType, "short_ros")
+                .addMatch("type", "global24h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum, getStrategyQueueInfo().getQueueName());
+
+        candidates.putAll(simplifiedCandidates);
+        return currRecall;
+
+    }
+
+    @Override
+    public String toString() {
+        return "Global24hHotCandidate{" +
+                "myCandidates=" + myCandidates +
+                ", rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global3hHotCandidate.java

@@ -0,0 +1,62 @@
+package com.tzld.piaoquan.recommend.server.implement.candidate;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.math.BigDecimal;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Global3hHotCandidate extends IndexCandidateQueue {
+
+    public static final String ItemType = "video";
+
+    public Global3hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int currRecall = 0;
+        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
+
+
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.6)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+
+
+        QueueName queueName = new QueueName(ItemType, "rov")
+                .addMatch("type", "global3h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
+
+        queueName = new QueueName(ItemType, "short_rov")
+                .addMatch("type", "global3h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
+
+        queueName = new QueueName(ItemType, "short_ros")
+                .addMatch("type", "global3h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum, getStrategyQueueInfo().getQueueName());
+
+
+        candidates.putAll(simplifiedCandidates);
+        return currRecall;
+
+    }
+
+    @Override
+    public String toString() {
+        return "Global24hHotCandidate{" +
+                "myCandidates=" + myCandidates +
+                ", rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/HotCandidateQueue.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.recommend.server.implement.candidate;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.util.Map;
+
+/**
+ * 对应 SimHotVideo 和 ReturnVideo
+ *
+ * @author sunxy
+ */
+public class HotCandidateQueue extends IndexCandidateQueue {
+
+    public HotCandidateQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user,
+                               RecommendRequest requestData, int requestIndex, int expId) {
+
+        return 0;
+    }
+
+    @Override
+    public String toString() {
+        return "HotCandidateQueue{" +
+                "myCandidates=" + myCandidates +
+                ", rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region1hHotCandidate.java

@@ -0,0 +1,67 @@
+package com.tzld.piaoquan.recommend.server.implement.candidate;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.math.BigDecimal;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Region1hHotCandidate extends IndexCandidateQueue {
+
+    public static final String ItemType = "video";
+
+    public Region1hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int currRecall = 0;
+        String region = user.getRegion();
+        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
+
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.6)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+
+
+        QueueName queueName = new QueueName(ItemType, "short_rov")
+                .addMatch("type", "region1h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum,
+                getStrategyQueueInfo().getQueueName());
+
+
+        queueName = new QueueName(ItemType, "rov")
+                .addMatch("type", "region1h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum,
+                getStrategyQueueInfo().getQueueName());
+
+        queueName = new QueueName(ItemType, "short_ros")
+                .addMatch("type", "region1h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
+                getStrategyQueueInfo().getQueueName());
+
+        candidates.putAll(simplifiedCandidates);
+        return currRecall;
+    }
+
+    @Override
+    public String toString() {
+        return "Region1hHotCandidate{" +
+                "myCandidates=" + myCandidates +
+                ", rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region24hHotCandidate.java

@@ -0,0 +1,61 @@
+package com.tzld.piaoquan.recommend.server.implement.candidate;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.math.BigDecimal;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Region24hHotCandidate extends IndexCandidateQueue {
+
+    public static final String ItemType = "video";
+
+    public Region24hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int currRecall = 0;
+        String region = user.getRegion();
+        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
+
+
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.6)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
+
+        // index key  video:queue:type=region24h:region=北京:ordering=rov
+        QueueName queueName = new QueueName(ItemType, "rov")
+                .addMatch("type", "region24h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
+
+        // index key  video:queue:type=region24h:region=北京:ordering=str
+        queueName = new QueueName(ItemType, "short_ros")
+                .addMatch("type", "region24h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
+                getStrategyQueueInfo().getQueueName());
+
+        candidates.putAll(simplifiedCandidates);
+        return currRecall;
+
+    }
+
+    @Override
+    public String toString() {
+        return "Region24hHotCandidate{" +
+                "myCandidates=" + myCandidates +
+                ", rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region3hHotCandidate.java

@@ -0,0 +1,70 @@
+package com.tzld.piaoquan.recommend.server.implement.candidate;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.math.BigDecimal;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Region3hHotCandidate extends IndexCandidateQueue {
+
+    public static final String ItemType = "video";
+
+    public Region3hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int currRecall = 0;
+        String region = user.getRegion();
+        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<>();
+
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.6)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+
+
+        // index key video:queue:type=region3h:region=北京:ordering=short_rov
+        QueueName queueName = new QueueName(ItemType, "short_rov")
+                .addMatch("type", "region3h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum,
+                getStrategyQueueInfo().getQueueName());
+
+        // index key video:queue:type=region3h:region=北京:ordering=rov
+        queueName = new QueueName(ItemType, "rov")
+                .addMatch("type", "region3h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum,
+                getStrategyQueueInfo().getQueueName());
+
+        // index key video:queue:type=region3h:region=北京:ordering=str
+        queueName = new QueueName(ItemType, "short_ros")
+                .addMatch("type", "region3h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
+                getStrategyQueueInfo().getQueueName());
+
+
+        candidates.putAll(simplifiedCandidates);
+        return currRecall;
+    }
+
+    @Override
+    public String toString() {
+        return "Region3hHotCandidate{" +
+                "myCandidates=" + myCandidates +
+                ", rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/AllowListFilter.java

@@ -0,0 +1,113 @@
+package com.tzld.piaoquan.recommend.server.implement.recall;
+
+
+import com.google.common.collect.Lists;
+import com.google.common.hash.Hashing;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
+import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.PreViewedService;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+
+import java.util.*;
+
+public class AllowListFilter extends AbstractFilter<Video> {
+
+    @Autowired
+    @Qualifier("longVideoRedisTemplate")
+    private RedisTemplate<String, String> redisTemplate;
+
+    private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX = "movie:videoid:allowSet:";
+    private static final String RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY = "mp:religion:allowlist:videoid:bitmap";
+
+    public AllowListFilter(FilterConfigInfo filterConfigInfo,
+                           RecommendRequest recommendRequest,
+                           User user) {
+        super(filterConfigInfo, recommendRequest, user);
+
+    }
+
+
+
+    @Override
+    public void doFilter(Candidate candidate, List<Video> list) {
+        if (CollectionUtils.isEmpty(list)) {
+            return;
+        }
+        List<Video> result = Lists.newArrayList();
+        for (Video video : list) {
+            if (!isAllowList(user, video)) {
+                result.add(video);
+            }
+        }
+        list.clear();
+        list.addAll(result);
+    }
+
+
+    public boolean isAllowList(User user, Video video) {
+        //不是新小程序 在白名单则不显示
+        Set<Long> retainVideoIds = new LinkedHashSet<>();
+        int appType = requestContext.getAppType();
+        if ( appType != AppTypeEnum.WAN_NENG_VIDEO.getCode()
+                && appType != AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
+                && appType != AppTypeEnum.ZUI_JING_QI.getCode()
+                && appType!= AppTypeEnum.H5.getCode()) {
+            // 如果不在新小程序白名单，则允许
+            return !isMemberOfVideoAllowList(video.getVideoId());
+
+        } else if (appType == AppTypeEnum.WAN_NENG_VIDEO.getCode()
+                || appType == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
+                || appType == AppTypeEnum.ZUI_JING_QI.getCode()
+                || appType == AppTypeEnum.H5.getCode()) {
+            return !isMemberOfReligionVideoAllowList(video.getVideoId());
+
+        }
+        return true;
+    }
+
+    private boolean isMemberOfVideoAllowList(Long videoId) {
+        if (Objects.isNull(videoId)) {
+            return false;
+        }
+        try {
+            int newIdx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 100;
+            String newPrefix = VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX + newIdx;
+            Boolean result = redisTemplate.opsForSet().isMember(newPrefix, String.valueOf(videoId));
+            return result;
+        } catch (Exception e) {
+            LOGGER.error("isMemberOfVideoAllowList error {}", videoId, e);
+        }
+        return false;
+    }
+
+    public boolean isMemberOfReligionVideoAllowList(Long videoId) {
+        if (Objects.isNull(videoId)) {
+            return false;
+        }
+        try {
+            return redisTemplate.opsForValue().getBit(RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY, videoId);
+        } catch (Exception e) {
+            LOGGER.error("isMemberOfReligionVideoAllowList error {}", e, videoId);
+        }
+        return false;
+    }
+
+
+
+
+
+}
+
+
+
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/HistoryLongPeriodFilter.java

@@ -0,0 +1,65 @@
+package com.tzld.piaoquan.recommend.server.implement.recall;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
+import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.RecommendParam;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.RecommendService;
+import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * TODO 有性能问题，暂时废弃
+ *
+ * @author sunxy
+ */
+public class HistoryLongPeriodFilter extends AbstractFilter<Video> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(HistoryLongPeriodFilter.class);
+
+    private final RegionFilterService filterService;
+    private final RecommendService recommendService;
+
+    public HistoryLongPeriodFilter(FilterConfigInfo filterConfigInfo, RecommendRequest requestContext, User user) {
+        super(filterConfigInfo, requestContext, user);
+        filterService = ServiceBeanFactory.getBean(RegionFilterService.class);
+        recommendService = ServiceBeanFactory.getBean(RecommendService.class);
+    }
+
+    @Override
+    public void doFilter(Candidate candidate, List<Video> list) {
+        if (list == null || list.isEmpty()) {
+            return;
+        }
+        RecommendParam recommendParam = recommendService.genRecommendParam(requestContext, 1);
+        RecallParam recallParam = recommendService.convertToRecallParam(recommendParam);
+        FilterParam filterParam = FilterParamFactory.create(recallParam,
+                list.stream().map(Video::getVideoId).collect(Collectors.toList()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        LOGGER.info("HistoryLongPeriodFilter doFilter traceId:{}, filterResult:{}", requestContext.getRequestId(),
+                filterResult);
+        LOGGER.info("HistoryLongPeriodFilter doFilter traceId:{}, list:{}", requestContext.getRequestId(),
+                list.stream().map(Video::getVideoId).map(Object::toString).collect(Collectors.joining(",")));
+        if (filterResult != null && filterResult.getVideoIds() != null) {
+            list.removeIf(video -> !filterResult.getVideoIds().contains(video.getVideoId()));
+        }
+        LOGGER.info("HistoryLongPeriodFilter doFilter end traceId:{}, list:{}", requestContext.getRequestId(),
+                list.stream().map(Video::getVideoId).map(Object::toString).collect(Collectors.joining(",")));
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/PreViewedFilter.java

@@ -0,0 +1,63 @@
+package com.tzld.piaoquan.recommend.server.implement.recall;
+
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
+import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.PreViewedService;
+import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class PreViewedFilter extends AbstractFilter<Video> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(PreViewedFilter.class);
+
+    private final Set<Long> preViewedVideoIds;
+
+    public PreViewedFilter(FilterConfigInfo filterConfigInfo,
+                           RecommendRequest recommendRequest,
+                           User user) {
+        super(filterConfigInfo, recommendRequest, user);
+        PreViewedService preViewedService = ServiceBeanFactory.getBean(PreViewedService.class);
+        preViewedVideoIds = preViewedService.getVideoIds(this.requestContext.getAppType(), user.getMid());
+    }
+
+    @Override
+    public void doFilter(Candidate candidate, List<Video> list) {
+        LOGGER.info("PreViewedFilter doFilter start traceId:{}, user:{}, apptype:{}", requestContext.getRequestId(),
+                user, requestContext.getAppType());
+        if (StringUtils.isBlank(user.getMid())
+                || CollectionUtils.isEmpty(list)) {
+            return;
+        }
+        LOGGER.info("PreViewedFilter doFilter traceId:{}, preViewedVideoIds:{}", requestContext.getRequestId(),
+                preViewedVideoIds);
+
+        if (CollectionUtils.isEmpty(preViewedVideoIds)) {
+            return;
+        }
+        LOGGER.info("PreViewedFilter doFilter traceId:{}, list:{}", requestContext.getRequestId(),
+                list.stream().map(Video::getVideoId).map(Objects::toString).collect(Collectors.joining(",")));
+        list.removeIf(video -> preViewedVideoIds.contains(video.getVideoId()));
+        LOGGER.info("PreViewedFilter doFilter end traceId:{}, list:{}", requestContext.getRequestId(),
+                list.stream().map(Video::getVideoId).map(Objects::toString).collect(Collectors.joining(",")));
+    }
+
+
+
+}
+
+
+
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/RecommendStatusFilter.java

@@ -0,0 +1,57 @@
+package com.tzld.piaoquan.recommend.server.implement.recall;
+
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
+import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoStatusRepository;
+import com.tzld.piaoquan.recommend.server.service.PreViewedService;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+
+import java.util.List;
+
+public class RecommendStatusFilter extends AbstractFilter<Video> {
+
+    @Autowired
+    @Qualifier("redisTemplate")
+    private RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private WxVideoStatusRepository wxVideoStatusRepository;
+
+    private String keyFormat = "video:recommend:status:%s";
+
+    private static final int RECOMMEND_STATUS = -6;
+
+
+    @Autowired
+    private PreViewedService preViewedService;
+
+    public RecommendStatusFilter(FilterConfigInfo filterConfigInfo,
+                                 RecommendRequest recommendRequest,
+                                 User user) {
+        super(filterConfigInfo, recommendRequest, user);
+
+    }
+
+    @Override
+    public void doFilter(Candidate candidate, List<Video> t) {
+        if (CollectionUtils.isEmpty(t)) {
+            return;
+        }
+    }
+
+
+
+
+}
+
+
+
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/RiskVideoFilter.java

@@ -0,0 +1,28 @@
+package com.tzld.piaoquan.recommend.server.implement.recall;
+
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
+import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.Video;
+
+import java.util.List;
+
+public class RiskVideoFilter extends AbstractFilter<Video> {
+
+
+    public RiskVideoFilter(FilterConfigInfo filterConfigInfo,
+                           RecommendRequest recommendRequest,
+                           User user) {
+        super(filterConfigInfo, recommendRequest, user);
+    }
+
+    @Override
+    public void doFilter(Candidate candidate, List<Video> t) {
+
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/TagFilter.java

@@ -0,0 +1,105 @@
+package com.tzld.piaoquan.recommend.server.implement.recall;
+
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
+import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+public class TagFilter extends AbstractFilter<Video> {
+
+    @Autowired
+    private WxVideoTagRelRepository repository;
+
+    @Value("${video.filter.tagids:}")
+    private String videoFilterTagIds;
+
+    private LoadingCache<Long, Set<Long>> videoTagCache = CacheBuilder.newBuilder()
+            .maximumSize(100)
+            .refreshAfterWrite(60, TimeUnit.SECONDS)
+            .expireAfterWrite(60, TimeUnit.SECONDS)
+            .expireAfterAccess(60, TimeUnit.SECONDS)
+            .build(new CacheLoader<Long, Set<Long>>() {
+                @Override
+                public Set<Long> load(Long tagId) {
+                    List<WxVideoTagRel> rels = repository.findAllByTagId(tagId);
+                    return CommonCollectionUtils.toSet(rels, WxVideoTagRel::getVideoId);
+                }
+            });
+
+
+    public TagFilter(FilterConfigInfo filterConfigInfo,
+                     RecommendRequest recommendRequest,
+                     User user) {
+        super(filterConfigInfo, recommendRequest, user);
+
+        if (org.apache.commons.lang.StringUtils.isNotBlank(this.videoFilterTagIds)) {
+            String[] tags = this.videoFilterTagIds.split(",");
+            for (String tag : tags) {
+                if (org.apache.commons.lang.StringUtils.isBlank(tag)) {
+                    continue;
+                }
+                this.videoTagCache.getUnchecked(NumberUtils.toLong(tag));
+            }
+        }
+    }
+
+    @Override
+    public void doFilter(Candidate candidate, List<Video> t) {
+        if (CollectionUtils.isEmpty(t)) {
+            return;
+        }
+        List<Video> videos = Lists.newArrayList();
+        for (Video video : t) {
+            if (!hasVideoRelTagId(video.getVideoId(), videoFilterTagIds)) {
+                videos.add(video);
+            }
+        }
+        t.removeAll(videos);
+    }
+
+    private boolean hasVideoRelTagId(Long videoId, String videoFilterTagIds) {
+        // TODO 主要是涉政标签
+        List<Long> tagIds = new ArrayList<>();
+
+        String[] tags = videoFilterTagIds.split(",");
+        for (String tag : tags) {
+            if (Objects.isNull(tag) || Objects.equals("", tag)) {
+                continue;
+            }
+            tagIds.add(Long.parseLong(tag));
+        }
+
+        for (Long tagId : tagIds) {
+            if (Objects.isNull(videoId) || videoId <= 0L || Objects.isNull(tagId) || tagId <= 0L) {
+                return false;
+            }
+            Set<Long> videos = this.videoTagCache.getUnchecked(tagId);
+            if (CollectionUtils.isEmpty(videos)) {
+                return false;
+            }
+            return videos.contains(videoId);
+        }
+        return false;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/ViewedHistoryFilter.java

@@ -0,0 +1,112 @@
+package com.tzld.piaoquan.recommend.server.implement.recall;
+
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
+import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
+import com.tzld.piaoquan.recommend.server.service.SpringContextHolder;
+import com.tzld.piaoquan.recommend.server.service.filter.strategy.VideoView;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.StopWatch;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.redis.core.RedisTemplate;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public class ViewedHistoryFilter extends AbstractFilter<Video> {
+
+
+    protected Set<String> historySet;
+
+    private final RedisTemplate<String, String> redisTemplate;
+
+    private final MongoTemplate mongoTemplate;
+
+    private final String keyFormat = "user:exclude:videoidset:%s";
+
+    public ViewedHistoryFilter(FilterConfigInfo filterConfigInfo,
+                               RecommendRequest recommendRequest,
+                               User user) {
+        super(filterConfigInfo, recommendRequest, user);
+        mongoTemplate = ServiceBeanFactory.getBean(MongoTemplate.class);
+        redisTemplate = SpringContextHolder.getBean("filterRedisTemplate", RedisTemplate.class);
+        historySet = this.loadUserHistory(user);
+        if (historySet == null) {
+            historySet = new HashSet<>();
+        }
+    }
+
+    @Override
+    public void doFilter(Candidate candidate, List<Video> videoList) {
+        if (CollectionUtils.isEmpty(videoList)) {
+            return;
+        }
+        LOGGER.info("ViewedHistoryFilter doFilter start traceId:{}, user:{}, apptype:{}", requestContext.getRequestId(),
+                user, requestContext.getAppType());
+        videoList.removeIf(video -> this.historySet.contains(String.valueOf(video.getVideoId())));
+        LOGGER.info("ViewedHistoryFilter doFilter end traceId:{}, list:{}", requestContext.getRequestId(),
+                videoList.stream().map(Video::getVideoId).map(Object::toString).collect(Collectors.joining(",")));
+    }
+
+
+    public Set<String> loadUserHistory(User user) {
+        String userid = StringUtils.isNotBlank(user.getUid()) ? user.getUid() : user.getMid();
+        LOGGER.info("ViewedHistoryFilter loadUserHistory start traceId:{}, userid:{}, apptype:{}", requestContext.getRequestId(),
+                userid, requestContext.getAppType());
+        if (StringUtils.isBlank(userid)) {
+            return new HashSet<>();
+        }
+
+        String key = String.format(keyFormat, userid);
+        Set<String> viewedVideoIds = redisTemplate.opsForSet().members(key);
+        if (CollectionUtils.isEmpty(viewedVideoIds)) {
+            viewedVideoIds = new HashSet<>();
+            StopWatch stopWatch = new StopWatch();
+            stopWatch.start();
+            // 从mongodb中取曝光记录
+            Criteria criteriatime = new Criteria();
+            criteriatime.and("uid").is(userid);
+            Query query = new Query();
+            query.addCriteria(criteriatime);
+            //查新库
+            List<VideoView> list = mongoTemplate.find(query, VideoView.class);
+            //限制最多10000条
+            if (Objects.nonNull(list) && list.size() > 10000) {
+                list = list.subList(list.size() - 10000, list.size());
+            }
+            if (CollectionUtils.isNotEmpty(list)) {
+                viewedVideoIds.addAll(list.stream().map(VideoView::getVideoId).map(String::valueOf).collect(Collectors.toSet()));
+            } else {
+                // 避免缓存击穿
+                viewedVideoIds.add("-1");
+            }
+            stopWatch.stop();
+            LOGGER.info("ViewedHistoryFilter loadUserHistory end traceId:{}, userid:{}, apptype:{}, viewedVideoIds:{}, time:{}",
+                    requestContext.getRequestId(), userid, requestContext.getAppType(), viewedVideoIds, stopWatch.getTime());
+
+            redisTemplate.opsForSet().add(key, viewedVideoIds.toArray(new String[0]));
+            redisTemplate.expire(key, 1, TimeUnit.DAYS);
+        }
+
+        return viewedVideoIds;
+    }
+
+
+
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/BaseLRModelScorer.java

@@ -0,0 +1,22 @@
+package com.tzld.piaoquan.recommend.server.implement.score;
+
+import com.tzld.piaoquan.recommend.server.framework.score.AbstractScorer;
+import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
+import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseLRModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRModelScorer.class);
+
+    public BaseLRModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(LRModel.class);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogShareLRScorer.java

@@ -0,0 +1,328 @@
+package com.tzld.piaoquan.recommend.server.implement.score;
+
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
+import com.tzld.piaoquan.recommend.feature.domain.video.feature.VlogShareLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.score.ScoreParam;
+import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
+import com.tzld.piaoquan.recommend.server.service.rank.strategy.OfflineVlogShareLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+
+public class VlogShareLRScorer extends BaseLRModelScorer {
+
+    private final static int CORE_POOL_SIZE = 64;
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogShareLRScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+    private static final double defaultUserCtrGroupNumber = 10.0;
+    private static final int enterFeedsScoreRatio = 10;
+    private static final int enterFeedsScoreNum = 20;
+
+
+    public VlogShareLRScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+
+    @Override
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(rankItems, param.getRequestContext(),
+                userFeature == null ? UserFeature.defaultInstance(param.getMid()) : userFeature);
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final List<RankItem> items,
+                                      final RequestContext requestContext,
+                                      final UserFeature user) {
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // userBytes
+        UserBytesFeature userInfoBytes = null;
+        userInfoBytes = new UserBytesFeature(user);
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userInfoBytes, requestContext, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+
+    /**
+     * 计算 predict ctr
+     */
+    public double calcScore(final LRModel lrModel,
+                            final RankItem item,
+                            final UserBytesFeature userInfoBytes,
+                            final RequestContext requestContext) {
+
+        LRSamples lrSamples = null;
+        VlogShareLRFeatureExtractor bytesFeatureExtractor;
+        bytesFeatureExtractor = new VlogShareLRFeatureExtractor();
+
+        try {
+            VideoBytesFeature newsInfoBytes = new VideoBytesFeature(item.getItemFeature() == null
+                    ? ItemFeature.defaultInstance(item.getVideoId() + "")
+                    : item.getItemFeature());
+            lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
+                    new RequestContextBytesFeature(requestContext));
+        } catch (Exception e) {
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
+        }
+
+
+        double pro = 0.0;
+        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
+            try {
+                pro = lrModel.score(lrSamples);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+            // 增加实时特征后打开在线存储日志逻辑
+            //
+            // CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
+            // samples.setLr_samples(lrSamples);
+            // item.setSamples(samples);
+            //
+        }
+        item.setScore(pro);
+        return pro;
+    }
+
+
+    /**
+     * 并行打分
+     *
+     * @param items
+     * @param userInfoBytes
+     * @param requestContext
+     * @param model
+     */
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final UserBytesFeature userInfoBytes,
+                                  final RequestContext requestContext,
+                                  final LRModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", requestContext.getRequest_id(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
+    }
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                           final Map<String, String> userFeatureMap,
+                                           final List<RankItem> rankItems){
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(
+                sceneFeatureMap, userFeatureMap, rankItems
+        );
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<RankItem> items) {
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+        // userBytes
+        Map<String, byte[]> userFeatureMapByte = new HashMap<>();
+        for(Map.Entry<String, String> entry: userFeatureMap.entrySet()){
+            userFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
+        }
+        //sceneBytes
+        Map<String, byte[]> sceneFeatureMapByte = new HashMap<>();
+        for(Map.Entry<String, String> entry: sceneFeatureMap.entrySet()){
+            sceneFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
+        }
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMapByte, sceneFeatureMapByte, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final Map<String, byte[]> userFeatureMapByte,
+                                  final Map<String, byte[]> sceneFeatureMapByte,
+                                  final LRModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            // items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMapByte, sceneFeatureMapByte);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMapByte.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
+    }
+
+    public double calcScore(final LRModel lrModel,
+                            final RankItem item,
+                            final Map<String, byte[]> userFeatureMapByte,
+                            final Map<String, byte[]> sceneFeatureMapByte) {
+
+        LRSamples lrSamples = null;
+        OfflineVlogShareLRFeatureExtractor bytesFeatureExtractor;
+        bytesFeatureExtractor = new OfflineVlogShareLRFeatureExtractor();
+
+        try {
+
+            Map<String, byte[]> itemFeatureByte = new HashMap<>();
+            for (Map.Entry<String, String> entry: item.getFeatureMap().entrySet()){
+                itemFeatureByte.put(entry.getKey(), entry.getValue().getBytes());
+            }
+            lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
+        } catch (Exception e) {
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
+        }
+
+
+        double pro = 0.0;
+        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
+            try {
+                pro = lrModel.score(lrSamples);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.setScoreStr(pro);
+        return pro;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogShareLRScorer4Ros.java

@@ -0,0 +1,328 @@
+package com.tzld.piaoquan.recommend.server.implement.score;
+
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
+import com.tzld.piaoquan.recommend.feature.domain.video.feature.VlogShareLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.score.ScoreParam;
+import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
+import com.tzld.piaoquan.recommend.server.service.rank.strategy.OfflineVlogShareLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+
+public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
+
+    private final static int CORE_POOL_SIZE = 64;
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogShareLRScorer4Ros.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+    private static final double defaultUserCtrGroupNumber = 10.0;
+    private static final int enterFeedsScoreRatio = 10;
+    private static final int enterFeedsScoreNum = 20;
+
+
+    public VlogShareLRScorer4Ros(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+
+    @Override
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(rankItems, param.getRequestContext(),
+                userFeature == null ? UserFeature.defaultInstance(param.getMid()) : userFeature);
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final List<RankItem> items,
+                                      final RequestContext requestContext,
+                                      final UserFeature user) {
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // userBytes
+        UserBytesFeature userInfoBytes = null;
+        userInfoBytes = new UserBytesFeature(user);
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userInfoBytes, requestContext, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+
+    /**
+     * 计算 predict ctr
+     */
+    public double calcScore(final LRModel lrModel,
+                            final RankItem item,
+                            final UserBytesFeature userInfoBytes,
+                            final RequestContext requestContext) {
+
+        LRSamples lrSamples = null;
+        VlogShareLRFeatureExtractor bytesFeatureExtractor;
+        bytesFeatureExtractor = new VlogShareLRFeatureExtractor();
+
+        try {
+            VideoBytesFeature newsInfoBytes = new VideoBytesFeature(item.getItemFeature() == null
+                    ? ItemFeature.defaultInstance(item.getVideoId() + "")
+                    : item.getItemFeature());
+            lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
+                    new RequestContextBytesFeature(requestContext));
+        } catch (Exception e) {
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
+        }
+
+
+        double pro = 0.0;
+        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
+            try {
+                pro = lrModel.score(lrSamples);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+            // 增加实时特征后打开在线存储日志逻辑
+            //
+            // CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
+            // samples.setLr_samples(lrSamples);
+            // item.setSamples(samples);
+            //
+        }
+        item.setScore(pro);
+        return pro;
+    }
+
+
+    /**
+     * 并行打分
+     *
+     * @param items
+     * @param userInfoBytes
+     * @param requestContext
+     * @param model
+     */
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final UserBytesFeature userInfoBytes,
+                                  final RequestContext requestContext,
+                                  final LRModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", requestContext.getRequest_id(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
+    }
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                  final Map<String, String> userFeatureMap,
+                                  final List<RankItem> rankItems){
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(
+                sceneFeatureMap, userFeatureMap, rankItems
+        );
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<RankItem> items) {
+        long startTime = System.currentTimeMillis();
+        LRModel model = (LRModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+        // userBytes
+        Map<String, byte[]> userFeatureMapByte = new HashMap<>();
+        for(Map.Entry<String, String> entry: userFeatureMap.entrySet()){
+            userFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
+        }
+        //sceneBytes
+        Map<String, byte[]> sceneFeatureMapByte = new HashMap<>();
+        for(Map.Entry<String, String> entry: sceneFeatureMap.entrySet()){
+            sceneFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
+        }
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMapByte, sceneFeatureMapByte, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final Map<String, byte[]> userFeatureMapByte,
+                                  final Map<String, byte[]> sceneFeatureMapByte,
+                                  final LRModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+//            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMapByte, sceneFeatureMapByte);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMapByte.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
+    }
+
+    public double calcScore(final LRModel lrModel,
+                            final RankItem item,
+                            final Map<String, byte[]> userFeatureMapByte,
+                            final Map<String, byte[]> sceneFeatureMapByte) {
+
+        LRSamples lrSamples = null;
+        OfflineVlogShareLRFeatureExtractor bytesFeatureExtractor;
+        bytesFeatureExtractor = new OfflineVlogShareLRFeatureExtractor();
+
+        try {
+
+            Map<String, byte[]> itemFeatureByte = new HashMap<>();
+            for (Map.Entry<String, String> entry: item.getFeatureMap().entrySet()){
+                itemFeatureByte.put(entry.getKey(), entry.getValue().getBytes());
+            }
+            lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
+        } catch (Exception e) {
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
+        }
+
+
+        double pro = 0.0;
+        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
+            try {
+                pro = lrModel.score(lrSamples);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.setScoreRos(pro);
+        return pro;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogThompsonScorer.java

@@ -0,0 +1,143 @@
+package com.tzld.piaoquan.recommend.server.implement.score;
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.score.BaseThompsonSamplingScorer;
+import com.tzld.piaoquan.recommend.server.framework.score.ScoreParam;
+import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
+import com.tzld.piaoquan.recommend.server.service.score.model.ThompsonSamplingModel;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+
+
+//@Service
+public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogThompsonScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+    public VlogThompsonScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+
+        if (userFeature == null || CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
+
+        LOGGER.debug("thompson sampling ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final List<RankItem> items,
+                                      final RequestContext requestContext,
+                                      final UserFeature user) {
+        long startTime = System.currentTimeMillis();
+        ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ROV Thompson排序
+        multipleCtrScore(items, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
+            }
+        }
+
+        LOGGER.debug("thompson ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[thompson ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+
+    /**
+     * 计算 predict ROV
+     */
+    public double calcScore(final ThompsonSamplingModel model,
+                            final RankItem item) {
+        double score = 0d;
+        try {
+            score = model.score(item);
+        } catch (Exception e) {
+            LOGGER.error("score error for doc={} exception={}", item.getVideo(), ExceptionUtils.getFullStackTrace(e));
+        }
+        item.setScore(score);
+        return score;
+    }
+
+
+    /**
+     * 并行打分 Thompson ROV
+     *
+     * @param items
+     * @param model
+     */
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final ThompsonSamplingModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            items.get(fIndex).setScore(0.0);   //设置为原始值为0
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex));
+                    } catch (Exception e) {
+                        LOGGER.error("thompson exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        LOGGER.debug("ROV-Thompson Score {}, Total: {}, Cancel: {}", items.size(), cancel);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/ModelService.java

@@ -32,7 +32,7 @@ public class ModelService {
     @Autowired
     private RedisTemplate<String, String> featureRedisTemplate;
 
-    private Map<String, String> modelNameMap = new HashMap<>();
+    private final Map<String, String> modelNameMap = new HashMap<>();
 
     @PostConstruct
     public void init() {
@@ -170,7 +170,6 @@ public class ModelService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ) {
-            ;
         } else {
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -55,7 +55,7 @@ public class RecommendService {
     private RedisTemplate<String, String> redisTemplate;
 
 
-    private Map<Integer, String> ab_initial_config_map = new HashMap<>();
+    private final Map<Integer, String> ab_initial_config_map = new HashMap<>();
 
     @ApolloJsonValue("${ab_exp_code:{}}")
     private Map<String, Map<String, String>> abExpCodeMap;
@@ -167,16 +167,16 @@ public class RecommendService {
                 map.put("traceId", String.valueOf(TraceUtils.currentTraceId()));
 
                 // TODO user
-                map.put("sessionId", String.valueOf(request.getSessionId()));
-                map.put("subsessionid", String.valueOf(request.getSubSessionId()));
+                map.put("sessionId", request.getSessionId());
+                map.put("subsessionid", request.getSubSessionId());
                 map.put("mid", param.getMid());
 
                 // ab
-                map.put("newexpgroup", String.valueOf(request.getNewExpGroup()));
+                map.put("newexpgroup", request.getNewExpGroup());
 
                 // scene
                 map.put("apptype", String.valueOf(request.getAppType()));
-                map.put("pagesource", String.valueOf(request.getPageSource()));
+                map.put("pagesource", request.getPageSource());
 
                 // video
                 map.put("videoId", String.valueOf(v.getVideoId()));
@@ -205,7 +205,7 @@ public class RecommendService {
         }
     }
 
-    private RecommendResponse specialMidRecommend(RecommendRequest request) {
+    public RecommendResponse specialMidRecommend(RecommendRequest request) {
         log.info("hit special mid recommend request={}", JSONUtils.toJson(request));
         if (request == null) {
             return RecommendResponse.newBuilder()
@@ -258,7 +258,7 @@ public class RecommendService {
                 .build();
     }
 
-    private RecommendParam genRecommendParam(RecommendRequest request, int recommendType) {
+    public RecommendParam genRecommendParam(RecommendRequest request, int recommendType) {
         RecommendParam param = new RecommendParam();
         param.setTopK(3);
         param.setFlowPoolP(0.3);
@@ -468,7 +468,7 @@ public class RecommendService {
         return videos;
     }
 
-    private RecallParam convertToRecallParam(RecommendParam param) {
+    public RecallParam convertToRecallParam(RecommendParam param) {
         RecallParam recallParam = new RecallParam();
         recallParam.setAppType(param.getAppType());
         // hard code 算法实验配置化之前，复用abcode做AB验证
@@ -520,7 +520,7 @@ public class RecommendService {
     }
 
 
-    private RankParam convertToRankParam(RecommendParam param, RecallResult recallResult) {
+    public RankParam convertToRankParam(RecommendParam param, RecallResult recallResult) {
         RankParam rankParam = new RankParam();
         rankParam.setRecallResult(recallResult);
         // hard code 算法实验配置化之前，复用abcode做AB验证
@@ -588,7 +588,7 @@ public class RecommendService {
         }
     }
 
-    private void updateCache(RecommendRequest request, RecommendParam param, List<Video> videos) {
+    public void updateCache(RecommendRequest request, RecommendParam param, List<Video> videos) {
         if (StringUtils.isBlank(request.getMid())
                 || CollectionUtils.isEmpty(videos)) {
             return;
@@ -633,6 +633,10 @@ public class RecommendService {
 
     private void updateLastVideoCache(List<Video> videos) {
 
+        if (CollectionUtils.isEmpty(videos)) {
+            return;
+        }
+
         Consumer<String> consumer = (p) -> {
             for (int i = videos.size() - 1; i >= 0; i--) {
                 if (videos.get(i).getPushFrom().equals(p)) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/SpringContextHolder.java

@@ -0,0 +1,33 @@
+package com.tzld.piaoquan.recommend.server.service;
+
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Service;
+
+@Service
+public class SpringContextHolder implements ApplicationContextAware {
+
+    private static ApplicationContext applicationContext;
+
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+        SpringContextHolder.applicationContext = applicationContext;
+    }
+
+    public static ApplicationContext getApplicationContext() {
+        return applicationContext;
+    }
+
+    public static Object getBean(String beanName) {
+        return applicationContext.getBean(beanName);
+    }
+
+    public static <T> T getBean(String beanName, Class<T> clazz) {
+        return applicationContext.getBean(beanName, clazz);
+    }
+
+    public static <T> T getBean(Class<T> clazz) {
+        return applicationContext.getBean(clazz);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/VideoRecommendService.java

@@ -0,0 +1,188 @@
+package com.tzld.piaoquan.recommend.server.service;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Strings;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.gen.common.Result;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendResponse;
+import com.tzld.piaoquan.recommend.server.gen.recommend.VideoProto;
+import com.tzld.piaoquan.recommend.server.implement.FlowPoolRecommendPipeline;
+import com.tzld.piaoquan.recommend.server.implement.TopRecommendPipeline;
+import com.tzld.piaoquan.recommend.server.model.RecommendParam;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.rank.strategy.RankStrategy4RegionMergeModelV2;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author sunxy
+ */
+@Service
+@Slf4j
+public class VideoRecommendService {
+
+    @Value("${recommend.new.log.print:true}")
+    private Boolean newLogPrint;
+    @Resource
+    private RecommendService recommendService;
+    @Resource
+    private TopRecommendPipeline topRecommendPipeline;
+    @Resource
+    private FlowPoolRecommendPipeline flowPoolRecommendPipeline;
+    @Resource
+    private RankStrategy4RegionMergeModelV2 rankStrategy4RegionMergeModelV2;
+    @Autowired
+    @Qualifier("redisTemplate")
+    private RedisTemplate<String, String> redisTemplate;
+
+    public RecommendResponse homepageRecommend(RecommendRequest request) {
+        RecommendParam recommendParam = recommendService.genRecommendParam(request, 0);
+        if (StringUtils.equals(recommendParam.getAbCode(), "60140")) {
+            return recommendNew(request, recommendParam, 0);
+        }
+        return recommend(request, 0);
+    }
+
+
+    public RecommendResponse relevantRecommend(RecommendRequest request) {
+        RecommendParam recommendParam = recommendService.genRecommendParam(request, 1);
+        if (StringUtils.equals(recommendParam.getAbCode(), "60140")) {
+            return recommendNew(request, recommendParam, 1);
+        }
+        return recommend(request, 1);
+    }
+
+    private RecommendResponse recommendNew(RecommendRequest request, RecommendParam param, int recommendType) {
+        if (request == null) {
+            return RecommendResponse.newBuilder()
+                    .setResult(Result.newBuilder().setCode(1).setMessage("success"))
+                    .build();
+        }
+        if (request.getVersionAuditStatus() == 1) {
+            return recommendService.specialMidRecommend(request);
+        }
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        if (StringUtils.isNotBlank(request.getMid())
+                && redisTemplate.opsForSet().isMember("special:mid", request.getMid())) {
+            return recommendService.specialMidRecommend(request);
+        }
+        stopwatch.reset().start();
+
+        log.info("genRecommendParam={},genRecommendParam cost={}", JSONUtils.toJson(param),
+                stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+        stopwatch.reset().start();
+        List<Video> videos = videoRecommend(request, param, recommendType);
+        // log.info("videoRecommend={}, videoRecommend cost={}", JSONUtils.toJson(videos),
+//        stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+        recommendService.updateCache(request, param, videos);
+
+        // 更新position
+        List<VideoProto> vps = new ArrayList<>();
+        for (int i = 0; i < videos.size(); i++) {
+            vps.add(VideoProto.newBuilder()
+                    .setPosition(i + 1)
+                    .setPushFrom(Strings.nullToEmpty(videos.get(i).getPushFrom()))
+                    .setAbCode(Strings.nullToEmpty(videos.get(i).getAbCode()))
+                    .setVideoId(videos.get(i).getVideoId())
+                    .setRovScore(videos.get(i).getRovScore())
+                    .setSortScore(videos.get(i).getSortScore())
+                    .setFlowPool(Strings.nullToEmpty(videos.get(i).getFlowPool()))
+                    .setIsInFlowPool(videos.get(i).isInFlowPool() ? 1 : 0)
+                    .setRand(videos.get(i).getRand())
+                    .build());
+        }
+
+        return RecommendResponse.newBuilder()
+                .setResult(Result.newBuilder().setCode(1).setMessage("success"))
+                .addAllVideo(vps)
+                .build();
+    }
+
+    private List<Video> videoRecommend(RecommendRequest request, RecommendParam param, int recommendType) {
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        User user = getUser(request);
+        List<Video> topRecommendVideoList = new ArrayList<>();
+        List<Video> flowPoolRecommendVideoList = new ArrayList<>();
+        try {
+            topRecommendVideoList = topRecommendPipeline.feeds(request, 0, user, newLogPrint);
+            if (newLogPrint) {
+                log.info("traceId = {}, cost = {}, topRecommendVideoList [{}]", request.getRequestId(),
+                        stopwatch.elapsed().toMillis(), JSONObject.toJSONString(topRecommendVideoList));
+            }
+
+        } catch (Exception e) {
+            log.error("traceId = {}, topRecommendVideoList error", request.getRequestId(), e);
+        }
+        try {
+            stopwatch.reset().start();
+            flowPoolRecommendVideoList = flowPoolRecommendPipeline.feedByRec(request, recommendType, newLogPrint);
+            if (newLogPrint) {
+                log.info("traceId = {}, cost = {}, flowPoolRecommendVideoList [{}]", request.getRequestId(),
+                        stopwatch.elapsed().toMillis(), JSONObject.toJSONString(flowPoolRecommendVideoList));
+            }
+        } catch (Exception e) {
+            log.error("traceId = {}, flowPoolRecommendVideoList error", request.getRequestId(), e);
+        }
+
+        stopwatch.reset().start();
+        RankParam rankParam = recommendService.convertToRankParam(param, null);
+        RankResult rankResult = rankStrategy4RegionMergeModelV2.mergeAndSort(rankParam, topRecommendVideoList,
+                flowPoolRecommendVideoList);
+        if (newLogPrint) {
+            log.info("traceId = {}, cost = {}, final rankResult [{}]", request.getRequestId(),
+                    stopwatch.elapsed().toMillis(), JSONObject.toJSONString(rankResult));
+        }
+
+        // 只返回size条数据
+        List<Video> videos = rankResult.getVideos();
+        if (param.getSize() < rankResult.getVideos().size()) {
+            videos = rankResult.getVideos().subList(0, param.getSize());
+        }
+
+        // 更新position
+        List<VideoProto> vps = new ArrayList<>();
+        for (int i = 0; i < videos.size(); i++) {
+            vps.add(VideoProto.newBuilder()
+                    .setPosition(i + 1)
+                    .setPushFrom(Strings.nullToEmpty(videos.get(i).getPushFrom()))
+                    .setAbCode(Strings.nullToEmpty(videos.get(i).getAbCode()))
+                    .setVideoId(videos.get(i).getVideoId())
+                    .setRovScore(videos.get(i).getRovScore())
+                    .setSortScore(videos.get(i).getSortScore())
+                    .setFlowPool(Strings.nullToEmpty(videos.get(i).getFlowPool()))
+                    .setIsInFlowPool(videos.get(i).isInFlowPool() ? 1 : 0)
+                    .setRand(videos.get(i).getRand())
+                    .build());
+        }
+        return videos;
+    }
+
+
+    private User getUser(RecommendRequest request) {
+        User user = new User();
+        user.setMid(request.getMid());
+        user.setUid(request.getUid());
+        user.setRegion(request.getProvince());
+        return user;
+    }
+
+    private RecommendResponse recommend(RecommendRequest request, int recommendType) {
+        return recommendService.recommend(request, recommendType);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/WarmUpService.java

@@ -1,7 +1,6 @@
 package com.tzld.piaoquan.recommend.server.service;
 
 import com.tzld.piaoquan.recommend.server.repository.WxVideoStatusRepository;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.core.annotation.Order;
@@ -33,7 +32,8 @@ public class WarmUpService {
         redisTemplate.opsForValue().get("");
         featureRedisTemplate.opsForValue().get("");
         longVideoRedisTemplate.opsForValue().get("");
-        ScorerUtils.warmUp();
+        com.tzld.piaoquan.recommend.server.service.score.ScorerUtils.warmUp();
+        com.tzld.piaoquan.recommend.server.framework.score.ScorerUtils.warmUp();
         wxVideoStatusRepository.count();
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -219,7 +219,7 @@ public abstract class AbstractFilterService {
                 log.info("{} param {} result {} cost {} ms", strategy.getClass().getSimpleName(),
                         JSONUtils.toJson(param.getVideoIds()),
                         JSONUtils.toJson(result),
-                        stopwatch.elapsed().toMillis());
+                        stopwatch.elapsed(TimeUnit.MILLISECONDS));
                 return result;
             });
             futures.add(future);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -49,6 +49,8 @@ public class RankRouter {
             return rankService.rank(param);
         }
         switch (abCode) {
+            case "60097":
+                return rankStrategy4Density.rank(param);
             case "60106":
                 return rankStrategy4Rankv2Model.rank(param);
             case "60101":
@@ -64,8 +66,6 @@ public class RankRouter {
             case "60116":
             case "60117":
             case "60118":
-            case "60119":
-                return rankStrategy4Density.rank(param);
             case "60107": // 556
                 return rankStrategyFlowThompsonModel.rank(param);
             case "60120": // 576
@@ -77,9 +77,9 @@ public class RankRouter {
             case "60123": // 541
                 return rankStrategy4RegionMergeModelV3.rank(param);
             case "60124": // 546
-                return rankStrategy4RegionMergeModelV4.rank(param);
+                return rankStrategy4RegionMergeModelV2.rank(param);
             case "60125": // 547
-                return rankStrategy4RegionMergeModelV5.rank(param);
+                return rankStrategy4RegionMergeModelV2.rank(param);
             case "60126": // 548
                 return rankStrategy4RegionMergeModelV6.rank(param);
             case "60130":

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -88,17 +88,12 @@ public class RankService {
 
         // 2 正常走分发 排序+冷启动
         List<Video> rovRecallRank = mergeAndRankRovRecall(param);
-        // log.info("mergeAndRankRovRecall rovRecallRank={}", JSONUtils.toJson(rovRecallRank));
         List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
-        // log.info("mergeAndRankFlowPoolRecall flowPoolRank={}", JSONUtils.toJson(flowPoolRank));
 
         rankFilter(param, rovRecallRank, flowPoolRank);
 
         removeDuplicate(param, rovRecallRank, flowPoolRank);
 
-//        log.info("removeDuplicate rovRecallRank={}, flowPoolRank={}",
-//                JSONUtils.toJson(rovRecallRank),
-//                JSONUtils.toJson(flowPoolRank));
 
         // 融合排序
         return mergeAndSort(param, rovRecallRank, flowPoolRank);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorDensity.java

@@ -71,7 +71,7 @@ public class RankProcessorDensity {
             String push = pushes.get(j);
             List<Video> candidate;
             if (!push.isEmpty()) {
-                // 5 如果是flow的video 取不到 不做替换
+                // 5 如果是flow的video0 取不到 不做替换
                 candidate = flow;
             } else {
                 // 5 如果是rov的video  取不到 不做替换

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Density.java

@@ -35,8 +35,8 @@ public class RankStrategy4Density extends RankService {
     @Override
     public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
 
-         //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos) || filterRules == null || filterRules.isEmpty()) {
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
             if (param.getSize() < flowVideos.size()) {
                 return new RankResult(flowVideos.subList(0, param.getSize()));
             } else {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RankModel.java

@@ -8,6 +8,8 @@ import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
@@ -18,11 +20,10 @@ import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -368,7 +369,6 @@ public class RankStrategy4RankModel extends RankService {
             "新竹市".equals(city) |
             "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Rankv2Model.java

@@ -8,6 +8,8 @@ import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
@@ -18,11 +20,10 @@ import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -368,7 +369,6 @@ public class RankStrategy4Rankv2Model extends RankService {
             "新竹市".equals(city) |
             "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV1.java

@@ -34,7 +34,7 @@ import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -49,7 +49,7 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
     @ApolloJsonValue("${rank.score.merge.weightv1:}")
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
+    private final Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
     @Override
     public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
@@ -566,7 +566,6 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV2.java

@@ -1,6 +1,5 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
-
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
@@ -569,7 +568,6 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV3.java

@@ -6,7 +6,6 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
@@ -50,7 +49,7 @@ public class RankStrategy4RegionMergeModelV3 extends RankService {
     @ApolloJsonValue("${rank.score.merge.weightv3:}")
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
+    private final Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
@@ -524,7 +523,6 @@ public class RankStrategy4RegionMergeModelV3 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV4.java

@@ -530,7 +530,6 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV5.java

@@ -543,7 +543,6 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV6.java

@@ -2,21 +2,26 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 
 import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeatureV2;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeatureV2;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
@@ -33,10 +38,10 @@ import org.springframework.stereotype.Service;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+
 /**
  * @author zhangbo
- * @desc 地域召回融合
+ * @desc 地域召回融合 流量池汤姆森
  */
 @Service
 @Slf4j
@@ -44,8 +49,9 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
     @ApolloJsonValue("${rank.score.merge.weightv6:}")
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
+    private final Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
+
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
         while(iterator.hasNext()){
@@ -59,53 +65,64 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
     }
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        List<Video> oldRovs = new ArrayList<>();
+        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        int sizeReturn = param.getSize();
+        removeDuplicate(oldRovs);
+        oldRovs = oldRovs.size() <= sizeReturn
+                ? oldRovs
+                : oldRovs.subList(0, sizeReturn);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, oldRovs);
+
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1_default.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3_default.PUSH_FORM);
         List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v1);
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
+        //-------------------节日扶持召回 融合+去重-------------------
         List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
         this.duplicate(setVideo, v7);
 
+        rovRecallRank.addAll(oldRovs);
         rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
         rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
         rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 0.0).intValue(), v4.size())));
         rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
         rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
         rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
+
+
+
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
-//        List<String> videoIdKeys = rovRecallRank.stream()
-//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-//                .collect(Collectors.toList());
-//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-//                JSONUtils.toJson(videoScores));
-//        if (CollectionUtils.isNotEmpty(videoScores)
-//                && videoScores.size() == rovRecallRank.size()) {
-//            for (int i = 0; i < videoScores.size(); i++) {
-//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-//            }
-//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        }
+
         // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<RankItem> items = model(rovRecallRank, param);
         List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
         List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
         Calendar calendar = Calendar.getInstance();
@@ -119,7 +136,7 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         }
         // 2 统计分
         String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
         for (int i=0; i<24; ++i){
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
@@ -146,9 +163,22 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流的rov和ros
+            List<Double> share2allreturn = getRateData(returns, shares, 1.0, 10.0);
+            Double share2allreturnScore = calScoreWeight(share2allreturn);
+            List<Double> view2allreturn = getRateData(returns, views, 0.0, 0.0);
+            Double view2allreturnScore = calScoreWeight(view2allreturn);
+            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
+            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
+
+            // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
 
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
             // rov的趋势
             double trendScore = calTrendScore(view2return);
             item.scoresMap.put("trendScore", trendScore);
@@ -160,22 +190,45 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
-        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
-        double beta = this.mergeWeight.getOrDefault("beta", 1.0);
+        double a = mergeWeight.getOrDefault("a", 0.1);
+        double b = mergeWeight.getOrDefault("b", 0.0);
+        double bb = mergeWeight.getOrDefault("bb", 0.005);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double f = mergeWeight.getOrDefault("f", 0.1);
+        double g = mergeWeight.getOrDefault("g", 1.0);
+        double h = mergeWeight.getOrDefault("h", 20.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
         for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 0.0 ?
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
-                    + alpha * trendScore
-                    + beta * newVideoScore
-                    ;
+            double strScore = item.getScoreStr();
+            double rosScoreModel = item.getScoreRos();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
+            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScore + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+
+            }
+            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
+            if (allreturnsScore > h){
+                score += (bb * rosScoreModel + f * share2allreturnScore + g * view2allreturnScore);
+            }
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
             video.setScoreStr(item.getScoreStr());
+            video.setScoreRos(item.getScoreRos());
             video.setScoresMap(item.getScoresMap());
             result.add(video);
         }
@@ -184,10 +237,10 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
     }
     public double calNewVideoScore(Map<String, String> itemBasicMap){
         double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 8){
+        if (existenceDays > 5){
             return 0.0;
         }
-        double score = 1.0 / (existenceDays + 5.0);
+        double score = 1.0 / (existenceDays + 10.0);
         return score;
     }
     public double calTrendScore(List<Double> data){
@@ -207,7 +260,6 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         }
         return sum;
     }
-
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -220,9 +272,13 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
     public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
         List<Double> data = new LinkedList<>();
         for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
+            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)){
+                data.add(0.0);
+            }else{
+                data.add(
+                        (ups.get(i) + up) / (downs.get(i) + down)
+                );
+            }
         }
         return data;
     }
@@ -237,9 +293,7 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         }
         return views;
     }
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart){
+    public List<RankItem> model(List<Video> videos, RankParam param){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -275,14 +329,13 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                 JSONObject obj = new JSONObject();
                 obj.put("name", "user_key_in_model_is_null");
                 obj.put("class", this.CLASS_NAME);
-                log.info(obj.toString());
-//                return videos;
             }
         }
         final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
                 "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
                 "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
         ));
         Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
         while (iterator.hasNext()) {
@@ -291,27 +344,29 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                 iterator.remove();
             }
         }
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+        Map<String, String> f1 = RankExtractorUserFeatureV2.getOriginFeature(userFeatureMap,
                 new HashSet<String>(Arrays.asList(
                         "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
                 ))
         );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+        Map<String, Double> f2__ = RankExtractorUserFeatureV2.getUserRateFeature(userFeatureMap);
+        Map<String, String> f2 = RankExtractorUserFeatureV2.rateFeatureChange(f2__);
+        Map<String, String> f3 = RankExtractorUserFeatureV2.cntFeatureChange(userFeatureMap,
                 new HashSet<String>(Arrays.asList(
                         "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                        "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
                 ))
         );
         f1.putAll(f2);
         f1.putAll(f3);
-//        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
                 "total_time", "play_count_total",
                 "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"
         ));
 
         List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
@@ -337,12 +392,14 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                             iteratorIn.remove();
                         }
                     }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                    Map<String, Double> f4__ = RankExtractorItemFeatureV2.getItemRateFeature(vfMap);
+                    Map<String, String> f4 = RankExtractorItemFeatureV2.rateFeatureChange(f4__);
+                    Map<String, String> f5 = RankExtractorItemFeatureV2.cntFeatureChange(vfMap,
                             new HashSet<String>(Arrays.asList(
                                     "total_time", "play_count_total",
                                     "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                                    "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"))
                     );
                     f4.putAll(f5);
                     rankItems.get(i).setFeatureMap(f4);
@@ -375,7 +432,6 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         videoRtKeys1.addAll(videoRtKeys2);
         List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
 
-
         if (videoRtFeatures != null){
             int j = 0;
             for (RankItem item: rankItems){
@@ -404,7 +460,8 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                 }catch (Exception e){
                     log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
                 item.getFeatureMap().putAll(f8);
             }
             for (RankItem item: rankItems){
@@ -435,26 +492,19 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                 }catch (Exception e){
                     log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
                 item.getFeatureMap().putAll(f8);
             }
         }
 
-
-//        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
-
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline("feeds_score_config_20240228.conf")
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-//        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
         JSONObject obj = new JSONObject();
         obj.put("name", "user_key_in_model_is_not_null");
         obj.put("class", this.CLASS_NAME);
-        log.info(obj.toString());
         return rovRecallScore;
     }
-
     private Map<String, String> getSceneFeature(RankParam param) {
         Map<String, String> sceneFeatureMap = new HashMap<>();
         String provinceCn = param.getProvince();
@@ -472,7 +522,6 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }
@@ -484,7 +533,6 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
 
         return sceneFeatureMap;
     }
-
     @Override
     public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -96,6 +96,7 @@ public class RecallService implements ApplicationContextAware {
         } else {
             switch (abCode) {
                 case "60121": // 536
+                case "60126": // 548
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1_default.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3_default.class.getSimpleName()));
@@ -103,6 +104,8 @@ public class RecallService implements ApplicationContextAware {
                     strategies.addAll(getRegionRecallStrategy(param));
                     break;
                 case "60122": // 537
+                case "60124": // 546
+                case "60125": // 547
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
@@ -111,9 +114,6 @@ public class RecallService implements ApplicationContextAware {
                     break;
                 case "60120": // 576
                 case "60123": // 541
-                case "60124": // 546
-                case "60125": // 547
-                case "60126": // 548
                 case "60112": // 562
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
@@ -126,10 +126,18 @@ public class RecallService implements ApplicationContextAware {
             //2:通过“流量池标记”控制“流量池召回子策略” 其中有9组会走EXPERIMENTAL_FLOW_SET_LEVEL 有1组会走EXPERIMENTAL_FLOW_SET_LEVEL_SCORE
             if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL)) {
                 strategies.add(strategyMap.get(QuickFlowPoolWithLevelRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategy.class.getSimpleName()));
+                if ("60126".equals(abCode)){
+                    strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategyTomson.class.getSimpleName()));
+                }else {
+                    strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategy.class.getSimpleName()));
+                }
             } else if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL_SCORE)) {
                 strategies.add(strategyMap.get(QuickFlowPoolWithLevelScoreRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(FlowPoolWithLevelScoreRecallStrategy.class.getSimpleName()));
+                if ("60126".equals(abCode)){
+                    strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategyTomson.class.getSimpleName()));
+                }else {
+                    strategies.add(strategyMap.get(FlowPoolWithLevelScoreRecallStrategy.class.getSimpleName()));
+                }
             } else {
                 strategies.add(strategyMap.get(QuickFlowPoolWithScoreRecallStrategy.class.getSimpleName()));
                 strategies.add(strategyMap.get(FlowPoolWithScoreRecallStrategy.class.getSimpleName()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractFlowPoolWithScoreRecallStrategy.java

@@ -6,7 +6,6 @@ import com.tzld.piaoquan.recommend.server.service.filter.FlowPoolWithScoreFilter
 import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.math.NumberUtils;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithLevelRecallStrategyTomson.java

@@ -0,0 +1,163 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConfigService;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants.KEY_WITH_LEVEL_FORMAT;
+
+/**
+ * @author zhangbo
+ */
+@Service
+@Slf4j
+public class FlowPoolWithLevelRecallStrategyTomson extends AbstractFlowPoolWithLevelRecallStrategy {
+
+    @Autowired
+    private FlowPoolConfigService flowPoolConfigService;
+
+    @Override
+    Pair<String, String> flowPoolKeyAndLevel(RecallParam param) {
+        //# 1. 获取流量池各层级分发概率权重
+        Map<String, Double> levelWeightMap = flowPoolConfigService.getLevelWeight();
+
+        // 2. 判断各层级是否有视频需分发
+        List<LevelWeight> availableLevels = new ArrayList<>();
+        for (Map.Entry<String, Double> entry : levelWeightMap.entrySet()) {
+            String levelKey = String.format(KEY_WITH_LEVEL_FORMAT, param.getAppType(), entry.getKey());
+            if (redisTemplate.hasKey(levelKey)) {
+                LevelWeight lw = new LevelWeight();
+                lw.setLevel(entry.getKey());
+                lw.setLevelKey(levelKey);
+                lw.setWeight(entry.getValue());
+                availableLevels.add(lw);
+            }
+        }
+        if (CollectionUtils.isEmpty(availableLevels)) {
+            return Pair.of("", "");
+        }
+
+        // 3. 根据可分发层级权重设置分发概率
+        Collections.sort(availableLevels, Comparator.comparingDouble(LevelWeight::getWeight));
+
+        double weightSum = availableLevels.stream().mapToDouble(o -> o.getWeight()).sum();
+        BigDecimal weightSumBD = new BigDecimal(weightSum);
+        double level_p_low = 0;
+        double weight_temp = 0;
+        double level_p_up = 0;
+        Map<String, LevelP> level_p_mapping = new HashMap<>();
+        for (LevelWeight lw : availableLevels) {
+            BigDecimal bd = new BigDecimal(weight_temp + lw.getWeight());
+            level_p_up = bd.divide(weightSumBD, 2, RoundingMode.HALF_UP).doubleValue();
+            LevelP levelP = new LevelP();
+            levelP.setMin(level_p_low);
+            levelP.setMax(level_p_up);
+            levelP.setLevelKey(lw.getLevelKey());
+            level_p_mapping.put(lw.level, levelP);
+            level_p_low = level_p_up;
+
+            weight_temp += lw.getWeight();
+        }
+
+        // 4. 随机生成[0,1)之间数，返回相应概率区间的key
+        double random_p = RandomUtils.nextDouble(0, 1);
+        for (Map.Entry<String, LevelP> entry : level_p_mapping.entrySet()) {
+            if (random_p >= entry.getValue().getMin()
+                    && random_p <= entry.getValue().getMax()) {
+                return Pair.of(entry.getValue().getLevelKey(), entry.getKey());
+            }
+        }
+        return Pair.of("", "");
+    }
+
+    @Data
+    static class LevelWeight {
+        private String level;
+        private String levelKey;
+        private Double weight;
+    }
+
+    @Data
+    static class LevelP {
+        private String levelKey;
+        private double min;
+        private double max;
+    }
+
+    @Override
+    public String pushFrom() {
+        return FlowPoolConstants.PUSH_FORM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        Pair<String, String> flowPoolKeyAndLevel = flowPoolKeyAndLevel(param);
+        String flowPoolKey = flowPoolKeyAndLevel.getLeft();
+        String level = flowPoolKeyAndLevel.getRight();
+        Set<String> data = redisTemplate.opsForSet().members(flowPoolKey);
+        if (CollectionUtils.isEmpty(data)) {
+            return null;
+        }
+        Map<String, String> videoFlowPoolMap = new LinkedHashMap<>();
+        Map<Long, String> videoFlowPoolMap_ = new LinkedHashMap<>();
+        for (String value : data) {
+            String[] values = value.split("-");
+            videoFlowPoolMap.put(values[0], values[1]);
+            videoFlowPoolMap_.put(NumberUtils.toLong(values[0], 0), values[1]);
+        }
+        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_tomson.conf");
+        List<List<Pair<Long, Double>>> results = pipeline.recall(videoFlowPoolMap);
+        List<Pair<Long, Double>> result = results.get(0);
+        Map<Long, Double> resultmap = result.stream()
+                .collect(Collectors.toMap(
+                        Pair::getLeft, // 键是Pair的left值
+                        Pair::getRight, // 值是Pair的right值
+                        (existingValue, newValue) -> existingValue, // 如果键冲突，选择保留现有的值（或者你可以根据需要定义其他合并策略）
+                        LinkedHashMap::new // 使用LinkedHashMap来保持插入顺序（如果需要的话）
+                ));
+        // 3 召回内部过滤
+        FilterParam filterParam = FilterParamFactory.create(param, result.stream()
+                .map(Pair::getLeft)
+                .collect(Collectors.toList()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        List<Video> videosResult = new ArrayList<>();
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            filterResult.getVideoIds().forEach(vid -> {
+                Video recallData = new Video();
+                recallData.setVideoId(vid);
+                recallData.setAbCode(param.getAbCode());
+                recallData.setRovScore(resultmap.getOrDefault(vid, 0.0));
+                recallData.setPushFrom(pushFrom());
+                recallData.setFlowPool(videoFlowPoolMap_.get(vid));
+                recallData.setFlowPoolAbtestGroup(param.getFlowPoolAbtestGroup());
+                recallData.setLevel(level);
+                videosResult.add(recallData);
+            });
+        }
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
+        return videosResult;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/AbstractScorer.java

@@ -1,8 +1,8 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
 
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.service.score.model.Model;
 import com.tzld.piaoquan.recommend.server.service.score.model.ModelManager;
 import org.apache.commons.lang3.StringUtils;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerConfig.java

@@ -5,7 +5,6 @@ import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import com.typesafe.config.ConfigObject;
 import com.typesafe.config.ConfigValue;
-
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -18,8 +17,8 @@ import java.util.Set;
 
 public class ScorerConfig {
 
-    private static Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
-    private List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
+    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
+    private final List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
 
     public ScorerConfig(Config config) {
         this.load(config);
@@ -62,7 +61,7 @@ public class ScorerConfig {
             int pos = 0;
             for (ScorerConfigInfo scorerConfigInfo : configInfoList) {
                 LOGGER.debug("scorer at position [{}], priority [{}], scorer name [{}]",
-                        new Object[]{pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName()});
+                        pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName());
             }
             LOGGER.debug("Load scorer config success");
         } catch (Exception e) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -17,10 +17,10 @@ import java.util.concurrent.ConcurrentHashMap;
 
 @Slf4j
 public final class ScorerUtils {
-    private static Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
 
-    private static Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
-    private static Map<String, ScorerPipeline4Recall> scorerPipelineCache4Recall = new ConcurrentHashMap<>();
+    private static final Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
+    private static final Map<String, ScorerPipeline4Recall> scorerPipelineCache4Recall = new ConcurrentHashMap<>();
 
     public static String BASE_CONF = "feeds_score_config_baseline.conf";
     public static String VIDEO_SCORE_CONF_FOR_AD = "video_score_config_for_ad.conf";
@@ -40,6 +40,7 @@ public final class ScorerUtils {
         ScorerUtils.init4Recall("feeds_recall_config_region_v4.conf");
         ScorerUtils.init4Recall("feeds_score_config_festival.conf");
         ScorerUtils.init4Recall("feeds_score_config_bless.conf");
+        ScorerUtils.init4Recall("feeds_recall_config_tomson.conf");
     }
 
     private ScorerUtils() {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareLRScorer.java

@@ -2,12 +2,11 @@ package com.tzld.piaoquan.recommend.server.service.score;
 
 
 import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
-import com.tzld.piaoquan.recommend.server.common.base.*;
-import com.tzld.piaoquan.recommend.feature.model.sample.*;
 import com.tzld.piaoquan.recommend.feature.domain.video.feature.VlogShareLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.service.rank.strategy.OfflineVlogShareLRFeatureExtractor;
 import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
@@ -106,8 +105,8 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
                     new RequestContextBytesFeature(requestContext));
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getUid()), item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
         }
 
 
@@ -116,8 +115,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             try {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
             // 增加实时特征后打开在线存储日志逻辑
             //
@@ -184,7 +182,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                 }
             }
         }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{requestContext.getRequest_id(), items.size(), cancel});
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
     }
     @Override
     public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
@@ -289,7 +287,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                 }
             }
         }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{sceneFeatureMapByte.size(), items.size(), cancel});
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
     }
 
     public double calcScore(final LRModel lrModel,
@@ -309,8 +307,8 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             }
             lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{"", item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
         }
 
 
@@ -319,8 +317,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             try {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
         }
         item.setScoreStr(pro);