v0

recommend-server-service/pom.xml

@@ -205,6 +205,12 @@
             <groupId>org.mongodb</groupId>
             <artifactId>mongodb-driver-sync</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>18.0</version>
+        </dependency>
+
     </dependencies>
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -1,10 +1,16 @@
 package com.tzld.piaoquan.recommend.server.common.base;
 
+import com.google.common.collect.Lists;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.CandidateInfo;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import lombok.Data;
 
+
+import com.google.common.collect.Maps;
+
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 
@@ -13,6 +19,7 @@ public class RankItem implements Comparable<RankItem> {
 
     // featureMap中保存所有的特征
     public Map<String, String> featureMap = new HashMap<>();
+    public String id;
     public long videoId;
     private double score; // 记录最终的score
     private Video video;
@@ -22,7 +29,34 @@ public class RankItem implements Comparable<RankItem> {
     // 记录Item侧用到的特征
     private ItemFeature itemFeature;
 
+    // 记录召回信息
+    private String queue;
+    private CandidateInfo candidateInfo;
+    private List<CandidateInfo> candidateInfoList; // 兼容多个召回key命中
+
+    // merge信息
+    private List<String> mergeQueuePath;
+    private List<String> mergeDecisionLabels;
+
+    // 多样性过滤因子
+    // 排序因子
+    private Map<String, Double> rankItemCategories = Maps.newHashMap();
+    private Map<String, Double> rankItemSubCategories = Maps.newHashMap();
+    private Map<String, Double> rankItemUserTags = Maps.newHashMap();
+    private Map<String, Double> rankItemTitleTags = Maps.newHashMap();
+
+
+    // 排序侧信息
+    private Map<String, Double> rankerScore;
+    private Map<String, Integer> rankerIndex;
+    public RankItem(){
+
+    }
+
+
+
     public RankItem(Video video) {
+        this.id = String.valueOf(video.getVideoId());
         this.videoId = video.getVideoId();
         this.score = 0.0;
         this.scoreRos = 0.0;
@@ -31,12 +65,40 @@ public class RankItem implements Comparable<RankItem> {
     }
 
     public RankItem(Long videoId) {
+        this.id = videoId.toString();
         this.videoId = videoId;
         this.score = 0.0;
     }
 
-    private Map<String, Double> rankerScore = new HashMap<>();
-    private Map<String, Integer> rankerIndex = new HashMap<>();
+    public RankItem(RankItem other) {
+        this.id = other.id;
+        this.videoId = other.videoId;
+        this.score = other.score;
+        this.scoreRos = other.scoreRos;
+        this.scoreStr = other.scoreStr;
+        this.video = other.video;
+
+        // merge queue
+        this.queue = other.queue;
+        this.mergeQueuePath = Lists.newArrayList();
+        this.mergeDecisionLabels = Lists.newArrayList();
+        if (other.getMergeQueuePath() != null)
+            this.mergeQueuePath.addAll(other.getMergeQueuePath());
+        if (other.getMergeDecisionLabels() != null)
+            this.mergeDecisionLabels.addAll(other.getMergeDecisionLabels());
+
+
+        // candidateinfo
+        this.candidateInfo = other.getCandidateInfo() != null ? other.getCandidateInfo().deepcopy() : null;
+        this.candidateInfoList = Lists.newArrayList();
+        if (other.getCandidateInfoList() != null) {
+            for (CandidateInfo tmpCandidateInfo : other.getCandidateInfoList()) {
+                this.candidateInfoList.add(tmpCandidateInfo.deepcopy());
+            }
+        }
+
+
+    }
 
 
     @Override
@@ -53,5 +115,39 @@ public class RankItem implements Comparable<RankItem> {
         }
     }
 
+    public RankItem deepcopy() {
+        return new RankItem(this);
+    }
+
+
+    public void addToCandidateInfoList(CandidateInfo candidateInfo) {
+        this.candidateInfoList.add(candidateInfo);
+    }
+
+
+    public void addMergeQueuePath(String mergeQueueName) {
+        this.mergeQueuePath.add(mergeQueueName);
+    }
+
+    public void addMergeDecisionLabel(String label) {
+        this.mergeDecisionLabels.add(label);
+    }
+
+    public void putRankerScore(String rankerName, Double rankerScore) {
+        this.rankerScore.put(rankerName, rankerScore);
+    }
+
+    public String getLastMergeQueueName() {
+        int pathSize = this.mergeQueuePath.size();
+        if (pathSize > 0) {
+            return this.mergeQueuePath.get(pathSize - 1);
+        } else {
+            return getCandidateInfo().getCandidateQueueName();
+        }
+    }
+
+
+
+
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/VlogRecommendPipeline.java

@@ -0,0 +1,85 @@
+package com.tzld.piaoquan.recommend.server.framework;
+
+
+import com.tzld.piaoquan.recommend.server.framework.merger.MergeUtils;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueue;
+import com.tzld.piaoquan.recommend.server.framework.recaller.BaseRecaller;
+import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfig;
+import com.tzld.piaoquan.recommend.server.framework.recaller.provider.InMemoryItem;
+import com.tzld.piaoquan.recommend.server.framework.recaller.provider.ItemProvider;
+import com.tzld.piaoquan.recommend.server.framework.recaller.provider.RedisBackedQueueWithoutMeta;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.userattention.UserAttentionExtractorPipeline;
+import com.tzld.piaoquan.recommend.server.framework.userattention.UserAttentionExtractorUtils;
+import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+
+public class VlogRecommendPipeline {
+
+    public static final String FILTER_CONF = "filter_config.conf";
+    public static final String MERGE_CONF = "merge_config.conf";
+
+    public static final String PREFIX = "";
+
+
+    private List<RankItem> feedByRec(final RecommendRequest requestData,
+                                       final int requestIndex,
+                                       final User userInfo) {
+        int recallNum =  200;
+
+        // Step 1: Attention extraction
+        long timestamp = System.currentTimeMillis();
+        UserAttentionExtractorPipeline attentionExtractorPipeline = UserAttentionExtractorUtils.getAtttentionPipeline(UserAttentionExtractorUtils.BASE_CONF);
+        attentionExtractorPipeline.extractAttention(requestData, userInfo);
+
+        // Step 2: create top queue
+        StrategyQueue topQueue = MergeUtils.createTopQueue(MERGE_CONF, "top-queue");
+
+
+        // Step 3: Candidate
+        Map<String, Candidate> candidates = new HashMap<String, Candidate>();
+        topQueue.candidate(candidates, recallNum, userInfo, requestData, 0, 0);
+
+        // Step 4: Recalling & Basic Scoring
+        RedisSmartClient client = new RedisSmartClient();
+        ItemProvider<InMemoryItem> itemProvider = new ItemProvider<>(client, PREFIX);
+        RedisBackedQueueWithoutMeta queueProvider = new RedisBackedQueueWithoutMeta(client, 1000L);
+
+        FilterConfig filterConfig = new FilterConfig();
+        filterConfig.load(FILTER_CONF);
+
+        BaseRecaller recaller = new BaseRecaller(itemProvider, queueProvider, filterConfig);
+        List<RankItem> items = recaller.recalling(requestData, userInfo, requestIndex, new ArrayList<Candidate>(candidates.values()));
+
+
+        /*
+        // Step 4: Advance Scoring
+        timestamp = System.currentTimeMillis();
+        ScorerPipeline scorerPipeline = getScorerPipeline(requestData);
+        items = scorerPipeline.scoring(requestData, userInfo, requestIndex, items);
+
+
+        // Step 5: Rerank
+        timestamp = System.currentTimeMillis();
+        RankPipeline rankPipeline = getRankPipeline(requestData;
+        List<RankItem> resultItems = rankPipeline.doRank(requestData, userInfo, requestIndex, items);
+
+        // Step 6: Global Rank & subList
+        // TODO: Global Rank
+        */
+
+
+        return items;
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Candidate.java

@@ -0,0 +1,40 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import com.google.gson.Gson;
+import lombok.Data;
+
+
+/**
+ * 记录召回下一路队列索引信息
+ */
+@Data
+public class Candidate {
+    private String candidateKey;
+    private int candidateNum;
+    private QueueName candidateQueueName;
+    // 上层待合并的queue name
+    private String mergeQueueName;
+    private int mergeQueueNum;
+
+    public Candidate() {
+    }
+
+    public Candidate(Candidate other) {
+        this.candidateKey = other.getCandidateKey();
+        this.candidateNum = other.getCandidateNum();
+        this.candidateQueueName = other.getCandidateQueueName();
+        this.mergeQueueName = other.getMergeQueueName();
+        this.mergeQueueNum = other.getMergeQueueNum();
+    }
+
+    public Candidate deepcopy() {
+        return new Candidate(this);
+    }
+
+
+    @Override
+    public String toString() {
+        return new Gson().toJson(this);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/CandidateInfo.java

@@ -0,0 +1,31 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import com.google.gson.Gson;
+import lombok.Data;
+
+@Data
+public class CandidateInfo {
+    private String candidateQueueName;     // 队列名称
+    private int position;               // 在召回中的位置
+    private Candidate candidate;        // 队列配置的信息
+
+    public CandidateInfo() {
+    }
+
+    public CandidateInfo(CandidateInfo other) {
+        this.candidateQueueName = other.getCandidateQueueName();
+        this.position = other.getPosition();
+        this.candidate = other.candidate != null ? new Candidate(other.candidate) : null;
+    }
+
+    public CandidateInfo deepcopy() {
+        return new CandidateInfo(this);
+    }
+
+
+    @Override
+    public String toString() {
+        return new Gson().toJson(this);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Entry.java

@@ -0,0 +1,38 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * 一个队列召回的内容实体
+ * @param <T>
+ */
+public class Entry<T> {
+    public final T item;
+    public final Map<String, Double> scores;
+    public final Map<String, String> explanations;
+    public final String id;
+
+    public Entry(T item, String id) {
+        this.item = item;
+        this.scores = new HashMap<String, Double>();
+        this.id = id;
+        this.explanations = new HashMap<String, String>();
+    }
+
+    public Entry(Entry<T> other) {
+        this.item = other.item;
+        this.id = other.id;
+        this.scores = other.scores;
+        this.explanations = other.explanations;
+    }
+
+    public void addScore(String name, double score) {
+        scores.put(name, score);
+    }
+
+    public void addExplanation(String name, String explanation) {
+        explanations.put(name, explanation);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/IndexCandidateQueue.java

@@ -0,0 +1,79 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.merger.*;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * 定义召回下子队列的Queue 融合信息
+ */
+public abstract class IndexCandidateQueue extends StrategyQueue {
+
+    protected Map<String, Candidate> myCandidates = new HashMap<String, Candidate>();
+
+    public IndexCandidateQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    public abstract int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId);
+
+
+    @Override
+    public int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        // TODO: sublist by every queue
+        return MergeUtils.simpleMergeWithProtection(items, rankerItemsListMap, recNum, user, requestData, requestIndex, expId);
+    }
+
+    @Override
+    public int candidate(final Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        myCandidates.clear();
+        int n = addCandidateKey(candidateMap, recallNum, user, requestData, requestIndex, expId);
+        candidateMap.putAll(myCandidates);
+
+        return n;
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, String key, int num, String queue) {
+        if (candidates.containsKey(key)) {
+            return 0;
+        } else {
+            Candidate candidate = new Candidate();
+            candidate.setCandidateKey(key);
+            candidate.setCandidateNum(num);
+            candidate.setMergeQueueName(queue);
+            candidates.put(key, candidate);
+            return num;
+        }
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue) {
+        return this.addCandidateKey(candidates, queueName, num, queue, true, true, true);
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue, boolean needScore, boolean needPornRank) {
+        return this.addCandidateKey(candidates, queueName, num, queue, needScore, needPornRank, true);
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue, boolean needScore, boolean needPornRank, boolean needExposeHistoryFilter) {
+        if (candidates.containsKey(queueName.toString())) {
+            return 0;
+        } else {
+            Candidate candidate = new Candidate();
+            candidate.setCandidateKey(queueName.toString());
+            candidate.setCandidateNum(num);
+            candidate.setMergeQueueName(queue);
+            candidates.put(queueName.toString(), candidate);
+            return num;
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/IndexDescription.java

@@ -0,0 +1,137 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+
+import com.google.common.collect.Lists;
+import com.typesafe.config.Config;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 描述检索的key
+ */
+public class IndexDescription {
+    public static Logger LOGGER = LoggerFactory.getLogger(IndexDescription.class);
+    public String name;
+
+    public int recallWeight;
+
+    /**
+     * redis,knn
+     */
+    public String db;
+    public String itemType;
+    public String indexName;
+    public String keyType;
+    public String key;
+    public List<Pair<String, String>> matches;
+    public String ordering;
+
+    /**
+     * 处理类似item_cf:when=day:score=click 的情况，提取出
+     * item_cf, when->day
+     */
+    public void processIndexName() {
+        if (indexName.contains(":")) {
+            String[] indics = indexName.split(":");
+            indexName = indics[0];
+            if (indics.length > 1) {
+                matches = Lists.newArrayList();
+                for (int i = 1; i < indics.length; ++i) {
+                    String[] pair = indics[i].split("=");
+                    if (pair.length == 2) {
+                        matches.add(Pair.of(pair[0], pair[1]));
+                    } else {
+                        LOGGER.error("index name [{}] match failed", indics[i]);
+                    }
+                }
+            }
+        }
+    }
+
+
+    /**
+     * 根据当前indexdesc的字段信息，生成candidate，用于后续召回
+     * @param queue
+     * @param indexDescription
+     * @return
+     */
+    public static Candidate convertToCandidate(String queue, IndexDescription indexDescription) {
+        Candidate candidate = new Candidate();
+        indexDescription.processIndexName();
+        QueueName queueName;
+
+        //knn类召回，不需要type
+        if ("knn".equalsIgnoreCase(indexDescription.keyType)) {
+            queueName = new QueueName(indexDescription.itemType, indexDescription.ordering);
+        } else {
+            queueName = new QueueName(indexDescription.itemType, indexDescription.ordering)
+                    .addMatch("type", indexDescription.indexName);
+        }
+
+
+        if (indexDescription.matches != null && !indexDescription.matches.isEmpty()) {
+            for (Map.Entry<String, String> entry : indexDescription.matches) {
+                queueName.addMatch(entry.getKey(), entry.getValue());
+            }
+        }
+
+        //添加类似tag->范冰冰 这类索引
+        if (StringUtils.isNotBlank(indexDescription.keyType)) {
+            queueName.addMatch(indexDescription.keyType, indexDescription.key);
+        }
+
+        candidate.setCandidateKey(queueName.toString());
+        candidate.setCandidateNum(indexDescription.recallWeight);
+        candidate.setMergeQueueNum(indexDescription.recallWeight);
+        candidate.setCandidateQueueName(queueName);
+        candidate.setMergeQueueName(queue);
+
+        return candidate;
+    }
+
+    public static final String RECALL_WEIGHT = "recall-weight";
+    public static final String INDEX_DESC = "index-desc";
+    public static final String DB = "db"; // source
+    public static final String ITEM_TYPE = "item-type";
+    public static final String NAME = "name";
+    public static final String KEY_TYPE = "key-type";
+    public static final String KEY = "key";
+    public static final String ORDERING = "ordering";
+    public static final String MATCHES = "matches";
+
+    public static IndexDescription parseIndexDescription(String name, Config conf) {
+        if (!conf.hasPath(RECALL_WEIGHT) || !conf.hasPath(INDEX_DESC)) {
+            LOGGER.error("name=" + name + ",必须输入recall-weight和indexdesc");
+            return null;
+        }
+
+        Config desc = conf.getConfig(INDEX_DESC);
+        if (!desc.hasPath(DB) || !desc.hasPath(ITEM_TYPE) || !desc.hasPath(NAME) || !desc.hasPath(ORDERING)) {
+            LOGGER.error("name=" + name + ",必须输入recall-weight和indexdesc");
+            return null;
+        }
+
+        IndexDescription indexDescription = new IndexDescription();
+        indexDescription.name = name;
+        indexDescription.recallWeight = conf.getInt(RECALL_WEIGHT);
+        indexDescription.db = desc.getString(DB);
+        indexDescription.itemType = desc.getString(ITEM_TYPE);
+        indexDescription.indexName = desc.getString(NAME);
+        indexDescription.ordering = desc.getString(ORDERING);
+
+        if (desc.hasPath(KEY_TYPE)) {
+            indexDescription.keyType = desc.getString(KEY_TYPE);
+        }
+        if (desc.hasPath(KEY)) {
+            indexDescription.key = desc.getString(KEY);
+        }
+
+        return indexDescription;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Queue.java

@@ -0,0 +1,54 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * An abstract data type represents a queue of certain item.
+ *   一个队列内召回Items完整list
+ */
+public class Queue<T> implements Iterable<Entry<T>> {
+    private final List<Entry<T>> entries;
+    private final String name;
+    private final String explain;
+
+    public Queue(String name) {
+        entries = new ArrayList<Entry<T>>();
+        this.name = name;
+        this.explain = null;
+    }
+
+    public Queue(String name, String explain) {
+        this.name = name;
+        this.explain = explain;
+        this.entries = new ArrayList<Entry<T>>();
+    }
+
+    public void reverse() {
+        Collections.reverse(this.entries);
+    }
+
+    public void add(Entry<T> entry) {
+        entries.add(entry);
+    }
+
+    public void addAll(Collection<Entry<T>> paramEntries) {
+        entries.addAll(paramEntries);
+    }
+
+    public List<Entry<T>> get() {
+        return entries;
+    }
+
+    public Iterator<Entry<T>> iterator() {
+        return entries.iterator();
+    }
+
+    public int size() {
+        return entries.size();
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/QueueName.java

@@ -0,0 +1,131 @@
+package com.tzld.piaoquan.recommend.server.framework.candidiate;
+
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.FluentIterable;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Queue names.召回队列命名处理
+ */
+public class QueueName implements Serializable, Comparable<QueueName> {
+    private static final Function<Pair<String, String>, String> MATCH_PATTERN =
+            new Function<Pair<String, String>, String>() {
+                @Override
+                public String apply(Pair<String, String> input) {
+                    return input.getLeft() + "=" + input.getRight();
+                }
+            };
+    public static final long DEFAULT_LOCAL_CACHE_TTL = 15 * 60 * 1000;
+    private final List<Pair<String, String>> matches = new ArrayList<Pair<String, String>>();
+    private final String itemType;
+    private final String ordering;
+    private final long ttl;
+
+    private String metaChannel;  // meta 渠道
+
+    public QueueName(String itemType, String ordering) {
+        this(itemType, ordering, DEFAULT_LOCAL_CACHE_TTL);
+    }
+
+    public QueueName(String itemType, String ordering, long ttl) {
+        this.itemType = itemType;
+        this.ordering = ordering;
+        this.ttl = ttl;
+    }
+
+    public static QueueName fromString(String string, long ttl) {
+        String[] parts = string.split(":");
+        String itemType = parts[0];
+        String ordering = parts[parts.length - 1].split("=")[1];
+
+        QueueName result = new QueueName(itemType, ordering, ttl);
+        for (int i = 2; i < parts.length - 1; ++i) {
+            String[] sides = parts[i].split("=");
+            result.addMatch(sides[0], sides[1]);
+        }
+        return result;
+    }
+
+    public static QueueName fromString(String string) {
+        return QueueName.fromString(string, DEFAULT_LOCAL_CACHE_TTL);
+    }
+
+    public static Pair<String, String> parseMatchPair(String match) {
+        String[] arr = match.split("=");
+        if (arr.length == 2) {
+            return Pair.of(arr[0], arr[1]);
+        }
+        return Pair.of("", "");
+    }
+
+    public String getMetaChannel() {
+        return metaChannel;
+    }
+
+    public void setMetaChannel(String metaChannel) {
+        this.metaChannel = metaChannel;
+    }
+
+    public String getItemType() {
+        return itemType;
+    }
+
+    public String getOrdering() {
+        return ordering;
+    }
+
+    public long getTtl() {
+        return ttl;
+    }
+
+    public QueueName addMatch(String key, String value) {
+        if (value == null || value.equals("")) {
+            value = "_";
+        } else {
+            value = value.replace("=", "_");
+            value = value.replace(":", "_");
+        }
+        if (key.equals("channel")) {
+            this.setMetaChannel(value);
+        }
+        matches.add(Pair.of(key, value));
+        return this;
+    }
+
+    public Iterable<String> getMatches() {
+        return FluentIterable.from(matches).transform(MATCH_PATTERN);
+    }
+
+    public long getTTL() {
+        return this.ttl;
+    }
+
+    @Override
+    public String toString() {
+        Iterable<String> matchesString = FluentIterable.from(matches)
+                .transform(MATCH_PATTERN);
+
+        return itemType + ":queue:" + Joiner.on(":").join(matchesString) + ":ordering=" + ordering;
+    }
+
+    @Override
+    public int hashCode() {
+        return toString().hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return other instanceof QueueName && ((QueueName) other).toString().equals(toString());
+    }
+
+    @Override
+    public int compareTo(QueueName other) {
+        return other.toString().compareTo(ordering.toString());
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/ArticleInfo.java

@@ -0,0 +1,13 @@
+package com.tzld.piaoquan.recommend.server.framework.common;
+
+
+import lombok.Data;
+
+@Data
+public class ArticleInfo {
+    private String id;
+    private String articleName;
+    private String title;
+    private String titleVector;
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/User.java

@@ -0,0 +1,53 @@
+package com.tzld.piaoquan.recommend.server.framework.common;
+
+import lombok.Data;
+
+@Data
+public class User {
+
+    private String id;
+    private UserAttention userAttention;
+
+    // user profile 基础画像信息
+    private String machine_model;
+    private String machine_brand;
+    private String sdk;
+    private String gender;
+    private String ifCreater;
+    private String userType;
+    private String region;
+    private String city;
+
+
+    // user-content info 用户内容tag统计数据
+    private UserAction last1monthVideoTags;
+    private UserAction last7dayVideoTags;
+    private UserAction last1dayVideoTags;
+    private UserAction lastSessionVideoTags;
+
+    private UserAction last1monthTitleTags;
+    private UserAction last7dayTitleTags;
+    private UserAction last1dayTitleTags;
+    private UserAction lastSessionTitleTags;
+
+
+    // 用户关注up主信息
+    private UserAction last1monthPublishers;
+
+    // user-action info 用户
+    private UserAction last1monthUserAction;
+    private UserAction last7dayUserAction;
+    private UserAction last1dayUserAction;
+    private UserAction last1hourUserAction;
+    private UserAction lastSessionUserAction;
+
+
+    // user-group info
+    private String userGroup;
+
+    // user-vector
+    private String userVector;
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/UserAction.java

@@ -0,0 +1,21 @@
+package com.tzld.piaoquan.recommend.server.framework.common;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class UserAction {
+    //记录用户行为中 itemids，tags等信息
+
+    private List<String> followed;
+    private List<String> follow_and_realplay;
+    private List<String> played;
+    private List<String> shared;
+
+    private List<String> feedsPlayed;
+    private List<String> realPlayed;
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/UserAttention.java

@@ -0,0 +1,26 @@
+package com.tzld.piaoquan.recommend.server.framework.common;
+
+public class UserAttention {
+    /**
+     分维度的用户特征数据, 包含：
+     1、分不同时间维度的ItemList
+     30天、7天、1天、1H、实时，可以提供给I2I类召回策略用
+
+     2、分不同时间维度的categoryList
+     30天、7天、1天、1H、实时，可以提供给CB类召回策略用
+
+     3、用户group信息
+     天级别更新数据，可以提供给usergroup -> Item召回策略用。
+
+     4、分时间维度的发布者List
+     30天、7天、1天、1H、实时，可以提供 发布者 -> Item类召回用
+
+     */
+
+
+
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/MergeRule.java

@@ -0,0 +1,45 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * Merge 规则配置
+ */
+@Data
+public class MergeRule {
+    public String queueName;
+
+    public Set<Integer> enableExpIdSet = null;
+    public Set<Integer> disableExpIdSet = null;
+
+    public double recallPercentage = 0.1; // 召回百分比, 默认占 10%
+    public int minMergeNum = 0; // 合并保护规则: 最小合并条数
+    public int maxMergeNum = 100; // 合并保护规则: 最大合并条数
+
+    private Map<String, String> properties;
+
+    public MergeRule() {
+        this.properties = new HashMap<String, String>();
+    }
+
+    public boolean isDisabled(final int expId) {
+        if (enableExpIdSet != null && !enableExpIdSet.contains(expId)) {
+            return true;
+        }
+        if (disableExpIdSet != null && disableExpIdSet.contains(expId)) {
+            return true;
+        }
+        return false;
+    }
+
+    public void putProperty(String key, String value) {
+        this.properties.put(key, value);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/MergeUtils.java

@@ -0,0 +1,313 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.google.common.base.Function;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+
+
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.CandidateInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+
+
+public class MergeUtils {
+    public static final Logger LOGGER = LoggerFactory.getLogger(MergeUtils.class);
+
+    public static StrategyQueue createTopQueue(String mergeConfFile, String topQueueName) {
+        StrategyQueueConfig strategyQueueConfig = new StrategyQueueConfig();
+        // Config mergeQueueConf = Configuration.getConfigByFileName(mergeConfFile, configDefault);
+
+        Config mergeQueueConf = ConfigFactory.parseResources(mergeConfFile);
+        if (strategyQueueConfig.load(mergeQueueConf)) {
+            LOGGER.debug("Merger config init succ");
+        } else {
+            LOGGER.error("Merge config init failed: init MergeBiz using queue config {}", mergeConfFile);
+        }
+
+        StrategyQueue topQueue = null;
+        try {
+            topQueue = MergeUtils.constructStrategyQueue(topQueueName, strategyQueueConfig);
+        } catch (InstantiationException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (IllegalAccessException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (ClassNotFoundException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (NoSuchMethodException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (InvocationTargetException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        }
+        return topQueue;
+    }
+
+    public static StrategyQueue constructStrategyQueue(String queueName, StrategyQueueConfig strategyQueueConfig) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
+        if (!strategyQueueConfig.getStrategyQueueInfoMap().containsKey(queueName)) {
+            return null;
+        }
+
+        StrategyQueueInfo strategyQueueInfo = strategyQueueConfig.getStrategyQueueInfoMap().get(queueName);
+        StrategyQueue strategyQueue = (StrategyQueue) Class.forName(strategyQueueInfo.getQueueClass())
+                .getConstructor(StrategyQueueInfo.class, strategyQueueConfig.getClass())
+                .newInstance(strategyQueueInfo, strategyQueueConfig);
+
+        return strategyQueue;
+    }
+
+    /**
+     * 分发items到策略树中
+     * 分发的过程中不要破坏相对顺序
+     *
+     * @param strategyQueue
+     * @param items
+     */
+    public static void distributeItemsToMultiQueues(StrategyQueue strategyQueue, List<RankItem> items) {
+        List<StrategyQueue> strategyQueueList = strategyQueue.getAllQueues();
+
+        Multimap<String, RankItem> mergeQueuesItems = ArrayListMultimap.create();
+        for (RankItem item : items) {
+            for (CandidateInfo candidateInfo : item.getCandidateInfoList()) {
+                String mergeQueue = candidateInfo.getCandidate().getMergeQueueName();
+                RankItem currentItem = item.deepcopy();
+
+                // set candidate info
+                currentItem.setQueue(mergeQueue);
+                currentItem.setCandidateInfo(candidateInfo);
+
+                mergeQueuesItems.put(mergeQueue, currentItem);
+            }
+        }
+
+        for (StrategyQueue queue : strategyQueueList) {
+            String mergeQueueName = queue.getStrategyQueueInfo().getQueueName();
+            if (mergeQueuesItems.containsKey(mergeQueueName)) {
+                List<RankItem> currMergeQueueItems = new ArrayList<RankItem>(mergeQueuesItems.get(mergeQueueName));
+                queue.setItems(currMergeQueueItems);
+            }
+        }
+    }
+
+    /**
+     * 基于 score 字段融合: score 较大的 Item 优先
+     *
+     * @param resultRankerItems
+     * @param rankerItemsListMap
+     * @param freeRecNum
+     * @param expId
+     */
+    public static void simpleMergeByScore(List<RankItem> resultRankerItems, final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, int freeRecNum, int expId) {
+        PriorityQueue<Pair<String, Integer>> mergePriorityQueue = new PriorityQueue<Pair<String, Integer>>(freeRecNum, new Comparator<Pair<String, Integer>>() {
+            @Override
+            public int compare(Pair<String, Integer> o1, Pair<String, Integer> o2) {
+                return rankerItemsListMap.get(o1.getLeft()).getRight().get(o1.getRight()).compareTo(rankerItemsListMap.get(o2.getLeft()).getRight().get(o2.getRight()));
+            }
+        });
+        for (Pair<MergeRule, List<RankItem>> entry : rankerItemsListMap.values()) {
+            if (entry.getLeft().isDisabled(expId) || entry.getRight() == null || entry.getRight().isEmpty()) {
+                continue;
+            }
+
+            MergeRule myRule = entry.getLeft();
+            if (myRule.minMergeNum < myRule.maxMergeNum && entry.getRight().size() > myRule.minMergeNum) {
+                Collections.sort(entry.getRight().subList(myRule.minMergeNum, entry.getRight().size()));
+                mergePriorityQueue.add(Pair.of(myRule.queueName, myRule.minMergeNum));
+            }
+        }
+
+        while (freeRecNum > 0 && !mergePriorityQueue.isEmpty()) {
+            Pair<String, Integer> item = mergePriorityQueue.poll();
+            String myName = item.getLeft();
+            int myIndex = item.getRight();
+
+            resultRankerItems.add(rankerItemsListMap.get(myName).getRight().get(myIndex));
+            resultRankerItems.get(resultRankerItems.size() - 1).addMergeDecisionLabel("score:" + myName);
+            freeRecNum -= 1;
+            myIndex += 1;
+
+            if (freeRecNum > 0 && rankerItemsListMap.get(myName).getRight().size() > myIndex && myIndex < rankerItemsListMap.get(myName).getLeft().maxMergeNum) {
+                mergePriorityQueue.add(Pair.of(myName, myIndex));
+            }
+        }
+    }
+
+    /**
+     * 队列融合:
+     * 1. 各队列基于排序不变;
+     * 2. 各队列保证最小合并数量 (minMergeNum) 条目;
+     * 3. 各队列限制最大合并数量 (maxMergeNum) 条目;
+     * 4. 从各队列取 Score 最高条目.
+     *
+     * @param rankerItemsListMap
+     * @param recNum
+     * @param user
+     * @param requestData
+     * @param requestIndex
+     * @param expId
+     * @return
+     */
+    public static int simpleMergeWithProtection(List<RankItem> resultRankerItems,
+                                                final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap,
+                                                int recNum,
+                                                User user,
+                                                RecommendRequest requestData,
+                                                int requestIndex,
+                                                int expId) {
+        if (resultRankerItems == null) {
+            resultRankerItems = new LinkedList<RankItem>();
+        }
+        // 保证各队列最小合并条目后剩余可自由调配条目数量
+        int freeRecNum = recNum;
+        for (Pair<MergeRule, List<RankItem>> entry : rankerItemsListMap.values()) {
+
+            if (entry.getLeft().isDisabled(expId) || entry.getRight() == null || entry.getRight().isEmpty()) {
+                continue;
+            }
+
+            MergeRule myRule = entry.getLeft();
+            int mySize = entry.getRight().size();
+            int myMinMergeNum = Math.min(myRule.minMergeNum, mySize);
+            freeRecNum -= myMinMergeNum;
+
+            int prevSize = resultRankerItems.size();
+            resultRankerItems.addAll(entry.getRight().subList(0, myMinMergeNum));
+            int afterSize = resultRankerItems.size();
+            for (int ix = prevSize; ix < afterSize; ++ix) {
+                resultRankerItems.get(ix).addMergeDecisionLabel("min_protect:" + myRule.queueName);
+            }
+        }
+
+        if (freeRecNum > 0) {
+            simpleMergeByScore(resultRankerItems, rankerItemsListMap, freeRecNum, expId);
+        }
+
+        return resultRankerItems.size();
+    }
+
+    /**
+     * 按优先级队列融合:
+     * 1. 各队列基于 priority 依次融合: 各队列保证最小合并数量 (minMergeNum) 条目;
+     * 2. 若没有凑足 recNum, 各队列限制最大合并数量 (maxMergeNum) 条目条件下按 score 优先选择;
+     *
+     * @param rankerItemsListMap
+     * @param recNum
+     * @param user
+     * @param requestData
+     * @param requestIndex
+     * @param expId
+     * @return
+     */
+    public static int simpleMergeByPriority(List<RankItem> resultRankerItems, final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        if (resultRankerItems == null) {
+            resultRankerItems = new LinkedList<RankItem>();
+        }
+
+        final List<Map.Entry<String, Pair<MergeRule, List<RankItem>>>> rankerItemsList = new LinkedList<Map.Entry<String, Pair<MergeRule, List<RankItem>>>>(rankerItemsListMap.entrySet());
+
+        // 保证各队列最小合并条目后剩余可自由调配条目数量
+        Collections.sort(rankerItemsList, new Comparator<Map.Entry<String, Pair<MergeRule, List<RankItem>>>>() {
+            private final int DEFAULT_PRIORITY = 0;
+
+            @Override
+            public int compare(Map.Entry<String, Pair<MergeRule, List<RankItem>>> o1, Map.Entry<String, Pair<MergeRule, List<RankItem>>> o2) {
+                int p1 = getPriority(o1.getValue().getLeft()), p2 = getPriority(o2.getValue().getLeft());
+                return p1 - p2;
+            }
+
+            private int getPriority(MergeRule rule) {
+                if (rule.getProperties().containsKey("priority")) {
+                    return Integer.valueOf(rule.getProperties().get("priority"));
+                }
+                return DEFAULT_PRIORITY;
+            }
+        });
+
+        int freeRecNum = recNum;
+        for (int i = 0; i < rankerItemsList.size(); ++i) {
+            if (rankerItemsList.get(i).getValue().getLeft().isDisabled(expId) || rankerItemsList.get(i).getValue().getRight() == null || rankerItemsList.get(i).getValue().getRight().isEmpty()) {
+                continue;
+            }
+
+            MergeRule myRule = rankerItemsList.get(i).getValue().getLeft();
+            int mySize = rankerItemsList.get(i).getValue().getRight().size();
+            int myMinMergeNum = Math.min(myRule.minMergeNum, mySize);
+            freeRecNum -= myMinMergeNum;
+
+            int prevSize = resultRankerItems.size();
+            resultRankerItems.addAll(rankerItemsList.get(i).getValue().getRight().subList(0, myMinMergeNum));
+            if (mySize > myMinMergeNum) {
+                Collections.sort(rankerItemsList.get(i).getValue().getRight().subList(myMinMergeNum, mySize));
+            }
+            int afterSize = resultRankerItems.size();
+            for (int ix = prevSize; ix < afterSize; ++ix) {
+                resultRankerItems.get(ix).addMergeDecisionLabel("min_protect:" + myRule.queueName);
+            }
+        }
+
+        if (freeRecNum > 0) {
+            simpleMergeByScore(resultRankerItems, rankerItemsListMap, freeRecNum, expId);
+        }
+
+        return resultRankerItems.size();
+    }
+
+    /**
+     * 基于 tag 和 category 重排序, 保证具有相同的分类或标签的 Item 在 windowSize 窗口内只有 similarLimit 条
+     *
+     * @param items
+     * @param expectedRecommendItemsCount
+     * @param windowSize
+     * @param similarLimit
+     */
+    public static void diversityRerank(List<RankItem> items, Function<Pair<RankItem, RankItem>, Boolean> similarFunc, int expectedRecommendItemsCount, final int windowSize, final int similarLimit) {
+        List<RankItem> rerankedItems = new LinkedList<RankItem>();
+        expectedRecommendItemsCount = Math.min(expectedRecommendItemsCount, items.size());
+        for (int ix = 0; ix < expectedRecommendItemsCount; ++ix) {
+            int windowStartPosition = Math.max(ix - windowSize + 1, 0);
+            int nextId = 0;
+            while (nextId < items.size()) {
+                int similarCount = 0;
+                for (int jx = windowStartPosition; jx < ix && similarCount < similarLimit; ++jx) {
+                    if (similarFunc.apply(Pair.of(items.get(nextId), rerankedItems.get(jx)))) {
+                        similarCount += 1;
+                    }
+                }
+                if (similarCount < similarLimit) {
+                    break;
+                }
+                nextId += 1;
+            }
+            if (nextId >= items.size()) {
+                nextId = 0;
+            }
+
+            RankItem nextItem = items.get(nextId);
+            nextItem.addMergeDecisionLabel("sim_rerank:" + windowStartPosition + "_" + windowSize);
+            items.remove(nextId);
+            rerankedItems.add(nextItem);
+        }
+
+        if (items.size() > 0) {
+            rerankedItems.addAll(items);
+        }
+
+        items.clear();
+        items.addAll(rerankedItems);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/SimilarityUtils.java

@@ -0,0 +1,116 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.google.common.collect.Sets;
+import com.google.common.base.Function;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import org.apache.commons.lang3.tuple.Pair;
+
+import javax.annotation.Nullable;
+
+
+public class SimilarityUtils {
+
+    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameUserTagOrCategoryFunc() {
+        return new Function<Pair<RankItem, RankItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
+                return isSameCategory(itemPair.getLeft(), itemPair.getRight()) || isSameUserTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameTitleTagFunc() {
+        return new Function<Pair<RankItem, RankItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
+                return isSameTitleTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameUserTagFunc() {
+        return new Function<Pair<RankItem, RankItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
+                return isSameUserTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameCategoryFunc() {
+        return new Function<Pair<RankItem, RankItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
+                return isSameCategory(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankItem, RankItem>, Boolean> getIsUserTagFunc() {
+        return new Function<Pair<RankItem, RankItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
+                return isSameUserTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static boolean isSameCategory(RankItem rankerItem1, RankItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getRankItemCategories() == null ||
+                rankerItem2.getRankItemCategories() == null) {
+            return false;
+        }
+
+        return !Sets.intersection(rankerItem1.getRankItemCategories().keySet(),
+                        rankerItem2.getRankItemCategories().keySet())
+                .isEmpty();
+    }
+
+    public static boolean isSameTitleTag(RankItem rankerItem1, RankItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getRankItemTitleTags() == null ||
+                rankerItem2.getRankItemTitleTags() == null) {
+            return false;
+        }
+        return !Sets.intersection(rankerItem1.getRankItemTitleTags().keySet(),
+                        rankerItem2.getRankItemTitleTags().keySet())
+                .isEmpty();
+    }
+
+    public static boolean isSameUserTag(RankItem rankerItem1, RankItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getRankItemUserTags() == null ||
+                rankerItem2.getRankItemUserTags() == null) {
+            return false;
+        }
+        return !Sets.intersection(rankerItem1.getRankItemUserTags().keySet(),
+                        rankerItem2.getRankItemUserTags().keySet())
+                .isEmpty();
+    }
+
+    public static boolean isSameQueue(RankItem rankerItem1, RankItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getQueue() == null) {
+            return false;
+        }
+        return rankerItem1.getQueue().equals(rankerItem2.getQueue());
+    }
+
+    public static boolean isSameSubCategory(RankItem item1, RankItem item2) {
+        if (item1 == null || item2 == null || item1.getRankItemSubCategories() == null
+                || item2.getRankItemSubCategories() == null) {
+            return false;
+        }
+
+        return !Sets.intersection(item1.getRankItemSubCategories().keySet(),
+                        item2.getRankItemSubCategories().keySet())
+                .isEmpty();
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/SimpleMergeQueue.java

@@ -0,0 +1,27 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.*;
+
+
+public class SimpleMergeQueue extends StrategyQueue {
+    public SimpleMergeQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        return MergeUtils.simpleMergeWithProtection(items, rankerItemsListMap, recNum, user, requestData, requestIndex, expId);
+    }
+
+    @Override
+    public int candidate(Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        return simpleCandidateByPercentage(candidateMap, recallNum, user, requestData, requestIndex, expId);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueue.java

@@ -0,0 +1,194 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import lombok.Data;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Data
+public abstract class StrategyQueue {
+    // 得等召回结果，定义为RankItem
+    protected final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsList = new HashMap<String, Pair<MergeRule, List<RankItem>>>();
+    private final Logger LOGGER = LoggerFactory.getLogger(StrategyQueue.class);
+    protected Map<String, StrategyQueue> children;
+
+    protected List<RankItem> items;
+    private StrategyQueueInfo strategyQueueInfo;
+    private List<StrategyQueue> allQueues = null;
+
+    public StrategyQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        this.strategyQueueInfo = strategyQueueInfo;
+
+        items = new LinkedList<RankItem>();
+        children = new HashMap<String, StrategyQueue>();
+
+        for (String childName : strategyQueueInfo.getChildren()) {
+            if (!strategyQueueConfig.getStrategyQueueInfoMap().containsKey(childName)) {
+                LOGGER.error("No config for Child queue [{}]", childName);
+                continue;
+            }
+            StrategyQueueInfo childQueueInfo = strategyQueueConfig.getStrategyQueueInfoMap().get(childName);
+
+            StrategyQueue childQueue = null;
+            String childQueueClass = childQueueInfo.getQueueClass();
+            try {
+                childQueue = (StrategyQueue) Class.forName(childQueueClass).getConstructor(childQueueInfo.getClass(), strategyQueueConfig.getClass()).newInstance(childQueueInfo, strategyQueueConfig);
+            } catch (InstantiationException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (IllegalAccessException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (ClassNotFoundException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (NoSuchMethodException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (InvocationTargetException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            }
+            if (childQueue != null) {
+                putChild(childName, childQueue);
+            } else {
+                LOGGER.error("construct child queue [{}] failed", childName);
+            }
+        }
+
+//        getAllQueues();
+    }
+
+    public List<StrategyQueue> getAllQueues() {
+        if (allQueues == null) {
+            allQueues = new LinkedList<StrategyQueue>();
+
+            if (!strategyQueueInfo.isLeaf()) {
+                for (Map.Entry<String, StrategyQueue> child : children.entrySet()) {
+                    allQueues.addAll(child.getValue().getAllQueues());
+                }
+            }
+
+            allQueues.add(this);
+        }
+        return allQueues;
+    }
+
+    private void putChild(String queueName, StrategyQueue child) {
+        children.put(queueName, child);
+    }
+
+
+
+    public void clearItems() {
+        items.clear();
+    }
+
+    public void addItem(RankItem item) {
+        this.items.add(item);
+    }
+
+    public abstract int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId);
+
+    // 定义截断函数
+    public abstract int candidate(Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId);
+
+    public final int merge(final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        this.beforeMerge(recNum, user, requestData, requestIndex, expId);
+        this.doMerge(rankerItemsList, recNum, user, requestData, requestIndex, expId);
+        this.doDeDup();
+        return this.afterMerge();
+    }
+
+    // Merge 前从每个Queue中获取Item
+    // 放入RankerItem中
+    public void beforeMerge(final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        if (strategyQueueInfo.isLeaf()) {
+            LOGGER.debug("leaf queue [{}] with [{}] items to merge", getStrategyQueueInfo().getQueueName(), items.size());
+        } else {
+            rankerItemsList.clear();
+            clearItems();
+            int queueCount = 0, itemCount = 0;
+            for (MergeRule rule : strategyQueueInfo.getRulesList()) {
+                if (rule.isDisabled(expId)) {
+                    continue;
+                }
+                int myRecNum = Math.min(recNum, rule.maxMergeNum);
+                int actualRecNum = children.get(rule.queueName).merge(myRecNum, user, requestData, requestIndex, expId);
+                queueCount += 1;
+                itemCount += actualRecNum;
+                rankerItemsList.put(rule.queueName, Pair.of(rule, children.get(rule.queueName).getItems()));
+            }
+            LOGGER.debug(getStrategyQueueInfo().getQueueName() + " merge info: [{}] queues with [{}] items to merge", queueCount, itemCount);
+        }
+    }
+
+    /**
+     * id dedup 去重操作
+     *
+     * @return
+     */
+    public final int doDeDup() {
+        List<RankItem> pureItems = new ArrayList<RankItem>();
+        Set<String> deDupItems = new HashSet<String>();
+        int deDupCount = 0;
+        for (RankItem item : items) {
+            if (deDupItems.contains(item.getVideoId())) {
+                if (LOGGER.isDebugEnabled()) {
+                    LOGGER.debug("merge queue dedup item, queue [{}] itemid [{}]",
+                            strategyQueueInfo.getQueueName(), item.getVideoId());
+                }
+                deDupCount++;
+                continue;
+            }
+            deDupItems.add(item.getId());
+            pureItems.add(item);
+        }
+        items = pureItems;
+        LOGGER.debug("dedup items, queue [{}], dedup number [{}], current number [{}]",
+                new Object[]{getStrategyQueueInfo().getQueueName(), deDupCount, items.size()});
+        return items.size();
+    }
+
+    // 给item增加融合队列名称
+    public int afterMerge() {
+        int n = items.size();
+        for (RankItem item : items) {
+            item.addMergeQueuePath(getStrategyQueueInfo().getQueueName());
+        }
+        LOGGER.debug("number of items after [{}] merge: [{}]", getStrategyQueueInfo().getQueueName(), n);
+        return n;
+    }
+
+    // 读取子队列的merge规则并按配比阶段
+    protected int simpleCandidateByPercentage(Map<String, Candidate> candidateMap, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int n = 0;
+
+        for (MergeRule rule : strategyQueueInfo.getRulesList()) {
+            if (rule.isDisabled(expId)) {
+                continue;
+            }
+
+            int myRecallNum = (int) (recallNum * rule.recallPercentage);
+            if (!children.containsKey(rule.queueName)) {
+                LOGGER.error("Rule for not exist child queue [{}]", rule.queueName);
+            }
+
+            Map<String, Candidate> myCandidates = new HashMap<String, Candidate>();
+            n += children.get(rule.queueName).candidate(myCandidates, myRecallNum, user, requestData, requestIndex, expId);
+            candidateMap.putAll(myCandidates);
+        }
+
+        return n;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueueConfig.java

@@ -0,0 +1,231 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import lombok.Data;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.*;
+
+@Data
+public class StrategyQueueConfig {
+    private final Logger LOGGER = LoggerFactory.getLogger(StrategyQueueConfig.class);
+
+    private static final String CHILDREN_STRING = "children";
+    private static final String STRATEGY_QUEUE_CLASS_CONF_STRING = "class";
+
+    public static final String DEFAULT_STRATEGY_CLASS_NAME = "com.tzld.piaoquan.recommend.server.framework.merger.SimpleMergeQueue";
+
+    private static final String MERGE_RULE_STRING = "merge-rule";
+    private static final String QUEUE_NAME_STRING = "queue-name";
+    private static final String RECALL_PERCENTAGE_STRING = "recall-percentage";
+    private static final String ENABLE_EXPERIMENT_LIST_STRING = "enable-exp";
+    private static final String DISABLE_EXPERIMENT_LIST_STRING = "disable-exp";
+    private static final String MIN_MERGE_NUM_STRING = "min-merge-num";
+    private static final String MAX_MERGE_NUM_STRING = "max-merge-num";
+
+    private static final Set<String> RESERVED_PROPERTIES = new HashSet<String>(Arrays.asList(QUEUE_NAME_STRING, RECALL_PERCENTAGE_STRING, ENABLE_EXPERIMENT_LIST_STRING, DISABLE_EXPERIMENT_LIST_STRING, MIN_MERGE_NUM_STRING, MAX_MERGE_NUM_STRING));
+
+    Map<String, StrategyQueueInfo> strategyQueueInfoMap;
+
+
+    public boolean load(Config config) {
+        config.checkValid(config, "queue-config");
+        Config queueConf = config.getConfig("queue-config");
+        if (loadMergeQueues(queueConf)) {
+            LOGGER.debug("Load queues config success");
+        } else {
+            LOGGER.error("Load queues config failed");
+            return false;
+        }
+
+        config.checkValid(config, "rule-config");
+        Config rulesConf = config.getConfig("rule-config");
+        if (loadMergeRules(rulesConf)) {
+            LOGGER.debug("Load rules config success");
+        } else {
+            LOGGER.error("Load rules config failed");
+            return false;
+        }
+        // TODO: validate queue structure & rules
+        for (String queueName : strategyQueueInfoMap.keySet()) {
+            StrategyQueue queue = null;
+            try {
+                queue = MergeUtils.constructStrategyQueue(queueName, this);
+            } catch (ClassNotFoundException e) {
+                e.printStackTrace();
+            } catch (NoSuchMethodException e) {
+                e.printStackTrace();
+            } catch (IllegalAccessException e) {
+                e.printStackTrace();
+            } catch (InvocationTargetException e) {
+                e.printStackTrace();
+            } catch (InstantiationException e) {
+                e.printStackTrace();
+            }
+
+            if (queue == null) {
+                LOGGER.error("Queue config info & rules error");
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    public boolean load(String configFile) {
+        Config queueConfig = ConfigFactory.parseResources(configFile);
+        return load(queueConfig);
+    }
+
+    private boolean loadMergeQueues(Config queueConf) {
+        strategyQueueInfoMap = new HashMap<String, StrategyQueueInfo>();
+
+        ConfigObject confObj = queueConf.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            String myName = it.getKey();
+            Config myConf = ((ConfigObject) it.getValue()).toConfig();
+            if (loadMergeQueueInfo(myName, myConf)) {
+                LOGGER.debug("Load queue [{}] info success", myName);
+            } else {
+                LOGGER.error("Load queue [{}] info failed", myName);
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean loadMergeQueueInfo(String name, Config conf) {
+        LOGGER.debug("Start load queue [{}]", name);
+        if (!strategyQueueInfoMap.containsKey(name)) {
+            strategyQueueInfoMap.put(name, new StrategyQueueInfo(name));
+        }
+        StrategyQueueInfo mergeQueueInfo = strategyQueueInfoMap.get(name);
+
+        if (conf.hasPath(STRATEGY_QUEUE_CLASS_CONF_STRING)) {
+            mergeQueueInfo.setQueueClass(conf.getString(STRATEGY_QUEUE_CLASS_CONF_STRING));
+        } else {
+            LOGGER.info("No Class config was found for queue [{}], use default class [{}]", name, DEFAULT_STRATEGY_CLASS_NAME);
+            mergeQueueInfo.setQueueClass(DEFAULT_STRATEGY_CLASS_NAME);
+        }
+
+        if (conf.hasPath(CHILDREN_STRING)) {
+            ConfigObject childrenConf = conf.getObject(CHILDREN_STRING);
+            for (ConfigObject.Entry<String, ConfigValue> it : childrenConf.entrySet()) {
+                String key = it.getKey();
+                Config childConf = ((ConfigObject) it.getValue()).toConfig();
+                String childName = key;
+
+                boolean isLoadSuccess = loadMergeQueueInfo(childName, childConf);
+                if (isLoadSuccess) {
+                    LOGGER.debug("Load queue [{}] SUCCESS", childName);
+                } else {
+                    LOGGER.error("Load queue [{}] FAILED: [{}]", childName, it.getValue());
+                    return false;
+                }
+
+                mergeQueueInfo.addChild(childName);
+            }
+        } else {
+            mergeQueueInfo.setLeaf(true);
+        }
+
+        LOGGER.debug("Load queue [{}] success", name);
+        return true;
+    }
+
+    private boolean loadMergeRules(Config conf) {
+        ConfigObject confObj = conf.root();
+
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            String myName = it.getKey();
+            Config subTreeConf = ((ConfigObject) it.getValue()).toConfig();
+            boolean isParseSuccess = parseMergeRule(myName, subTreeConf);
+            if (isParseSuccess) {
+                LOGGER.debug("Parse rules for queue [{}] SUCCESS", myName);
+            } else {
+                LOGGER.error("Parse rules for queue [{}] FAILED: [{}]", myName, it.getValue());
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private boolean parseMergeRule(String name, Config ruleConf) {
+        if (!strategyQueueInfoMap.containsKey(name)) {
+            LOGGER.error("skip rules for not exist queue [{}]", name);
+            return true;
+        } else {
+            LOGGER.debug("parse rules for queue [{}]", name);
+        }
+
+        if (!ruleConf.hasPath(MERGE_RULE_STRING)) {
+            LOGGER.debug("no merge rules for queue [{}]", name);
+            return true;
+        }
+
+        ConfigObject ruleConfObj = ruleConf.getObject(MERGE_RULE_STRING);
+
+        for (ConfigObject.Entry<String, ConfigValue> it : ruleConfObj.entrySet()) {
+            String ruleName = it.getKey();
+            MergeRule rule = new MergeRule();
+
+            Config myConf = ((ConfigObject) it.getValue()).toConfig();
+
+            rule.recallPercentage = getOrDefault(myConf, RECALL_PERCENTAGE_STRING, 0.1);
+            rule.minMergeNum = getOrDefault(myConf, MIN_MERGE_NUM_STRING, 0);
+            rule.maxMergeNum = getOrDefault(myConf, MAX_MERGE_NUM_STRING, 500);
+
+            // 子队列名规则: 用 queue-name 显式指定, 默认为对应规则名
+            rule.queueName = getOrDefault(myConf, QUEUE_NAME_STRING, ruleName);
+            if (myConf.hasPath(ENABLE_EXPERIMENT_LIST_STRING)) {
+                List<Integer> expIds = myConf.getIntList(ENABLE_EXPERIMENT_LIST_STRING);
+                rule.enableExpIdSet = new HashSet<Integer>(expIds);
+            }
+            if (myConf.hasPath(DISABLE_EXPERIMENT_LIST_STRING)) {
+                List<Integer> expIds = myConf.getIntList(DISABLE_EXPERIMENT_LIST_STRING);
+                rule.disableExpIdSet = new HashSet<Integer>(expIds);
+            }
+
+            for (ConfigObject.Entry<String, ConfigValue> properties : myConf.entrySet()) {
+                if (!RESERVED_PROPERTIES.contains(properties.getKey())) {
+                    rule.putProperty(properties.getKey(), myConf.getString(properties.getKey()));
+                }
+            }
+
+            if (strategyQueueInfoMap.get(name).getChildren().contains(rule.queueName)) {
+                strategyQueueInfoMap.get(name).addRule(rule);
+            } else {
+                LOGGER.info("Skip rule={} for not exist queue={}", ruleName, rule.queueName);
+            }
+        }
+        return true;
+    }
+
+    private int getOrDefault(Config conf, String path, int defaultValue) {
+        if (conf.hasPath(path)) {
+            return conf.getInt(path);
+        }
+        return defaultValue;
+    }
+
+    private String getOrDefault(Config conf, String path, String defaultValue) {
+        if (conf.hasPath(path)) {
+            return conf.getString(path);
+        }
+        return defaultValue;
+    }
+
+    private double getOrDefault(Config conf, String path, double defaultValue) {
+        if (conf.hasPath(path)) {
+            return conf.getDouble(path);
+        }
+        return defaultValue;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueueInfo.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.recommend.server.framework.merger;
+
+
+import lombok.Data;
+
+import java.util.*;
+
+
+/**
+ * 每一个召回 or 队列的融合配置信息
+ */
+
+@Data
+public class StrategyQueueInfo {
+    private String queueName;
+
+    private boolean isLeaf = false;
+
+    private String queueClass; // Full path of StrategyQueue Class
+
+    private List<MergeRule> rulesList;
+
+    private Set<String> children;
+
+    public StrategyQueueInfo(String name) {
+        this.queueName = name;
+        rulesList = new LinkedList<MergeRule>();
+        children = new HashSet<String>();
+    }
+
+    public void addRule(MergeRule rule) {
+        this.rulesList.add(rule);
+    }
+
+    public void addChild(String childName) {
+        this.children.add(childName);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/GBDTModel.java

@@ -0,0 +1,196 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.*;
+
+
+
+public class GBDTModel extends Model {
+    private static final Logger LOGGER = LoggerFactory.getLogger(GBDTModel.class);
+    private static final float LINEAR_TRANSFORM_LOWER_BOUND = 0f;
+    private static final float LINEAR_TRANSFORM_SLOPE = 1f / 1f;
+    private int featureCount = 0;
+    private Map<String, Integer> featureIdMap = null;
+    private List<HashMap<Integer, Node>> boosterModel = null;
+
+    private float transform(final float score) {
+        return (score - LINEAR_TRANSFORM_LOWER_BOUND) * LINEAR_TRANSFORM_SLOPE;
+    }
+
+    private int getFeatureId(final String feature) {
+        return featureIdMap.containsKey(feature) ? featureIdMap.get(feature) : -1;
+    }
+
+    @Override
+    public int getModelSize() {
+        return boosterModel.size();
+    }
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws Exception {
+        List<HashMap<Integer, Node>> model = new ArrayList<HashMap<Integer, Node>>();
+        HashMap<Integer, Node> tree = null;
+
+        featureIdMap = new HashMap<String, Integer>();
+        featureCount = 0;
+
+        BufferedReader input = new BufferedReader(in);
+        String line;
+        while ((line = input.readLine()) != null) {
+            String[] tokens = line.trim().split(":");
+            if (tokens.length == 1) {
+                if (tokens[0].startsWith("booster")) {
+                    if (tree != null && tree.size() > 0) {
+                        model.add(tree);
+                    }
+
+                    tree = new HashMap<Integer, Node>();
+                }
+            } else if (tokens.length == 2) {
+                Node node = new Node();
+
+                Integer id = Integer.parseInt(tokens[0]);
+
+                String[] items = tokens[1].split(" ");
+                if (items.length == 1) {
+                    node.isLeaf = true;
+
+                    String[] dt = items[0].split("=");
+                    node.value = Float.parseFloat(dt[1]);
+                } else if (items.length == 2) {
+                    node.isLeaf = false;
+                    String[] fieldDescriptions = null;
+                    node.compareType = NodeCompareType.LT;
+                    if (items[0].substring(1, items[0].length() - 1).contains("<=")) {
+                        fieldDescriptions = items[0].substring(1, items[0].length() - 1).split("<=");
+                        node.compareType = NodeCompareType.LE;
+                    } else {
+                        fieldDescriptions = items[0].substring(1, items[0].length() - 1).split("<");
+                    }
+                    // feature to id
+                    int featureId = getFeatureId(fieldDescriptions[0]);
+                    if (featureId < 0) {
+                        featureIdMap.put(fieldDescriptions[0], featureCount);
+                        featureId = featureCount;
+                        featureCount += 1;
+                    }
+                    node.splitFeatureId = featureId;
+
+                    if (fieldDescriptions.length == 1) {
+                        node.fieldType = NodeFieldType.BINARY;
+                    } else {
+                        node.fieldType = NodeFieldType.QUANTITATIVE;
+                        node.splitCondition = Float.parseFloat(fieldDescriptions[1]);
+                    }
+
+                    String[] childrenDescriptions = items[1].split(",");
+                    for (String childDescription : childrenDescriptions) {
+                        String[] descs = childDescription.split("=");
+                        if (descs[0].equals("yes")) {
+                            node.yes = Integer.parseInt(descs[1]);
+                        } else if (descs[0].equals("no")) {
+                            node.no = Integer.parseInt(descs[1]);
+                        } else if (descs[0].equals("missing")) {
+                            node.missing = Integer.parseInt(descs[1]);
+                        }
+                    }
+                }
+                tree.put(id, node);
+            }
+        }
+        if (tree != null && !tree.isEmpty()) {
+            model.add(tree);
+        }
+
+        LOGGER.info("Boosted tree model load over and tree number is " + model.size());
+        input.close();
+        in.close();
+        if (model != null && model.size() > 0) {
+            boosterModel = model;
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private float score(final Map<Integer, Node> tree, final boolean[] featureIsExists, final Float[] featureValues) {
+        int id = 0;
+        while (true) {
+            Node currentNode = tree.get(id);
+            if (currentNode.isLeaf) {
+                return currentNode.value;
+            }
+
+            if (currentNode.fieldType == NodeFieldType.BINARY) {
+                if (!featureIsExists[currentNode.splitFeatureId]) {
+                    id = currentNode.no;
+                } else {
+                    id = currentNode.yes;
+                }
+            } else if (currentNode.fieldType == NodeFieldType.QUANTITATIVE) {
+                if (!featureIsExists[currentNode.splitFeatureId]) {
+                    id = currentNode.missing;
+                } else {
+                    Float value = featureValues[currentNode.splitFeatureId];
+
+                    if ((currentNode.compareType == NodeCompareType.LT && value < currentNode.splitCondition)
+                            || (currentNode.compareType == NodeCompareType.LE && value <= currentNode.splitCondition)) {
+                        id = currentNode.yes;
+                    } else {
+                        id = currentNode.no;
+                    }
+                }
+            } else {
+                LOGGER.error("Reach undefined condition: {}", id);
+            }
+        }
+    }
+
+    public Float score(final Map<String, Double> features, Map<String, Double> featuresScore) {
+        Float result = 0f;
+
+        boolean[] featureIsExists = new boolean[featureCount];
+        Float[] featureValues = new Float[featureCount];
+        for (Map.Entry<String, Double> entry : features.entrySet()) {
+            int featureId = getFeatureId(entry.getKey());
+            if (featureId >= 0) {
+                featureIsExists[featureId] = true;
+                featureValues[featureId] = entry.getValue().floatValue();
+            }
+        }
+
+        for (Map<Integer, Node> tree : boosterModel) {
+            result += score(tree, featureIsExists, featureValues);
+        }
+
+        Float transformedResult = transform(result);
+
+        LOGGER.debug("[calc_dwelltime]features: " + Arrays.toString(features.entrySet().toArray()) + ", prediction:" + result + " , transformed:" + transformedResult);
+
+        return transformedResult;
+    }
+
+    private enum NodeFieldType {
+        BINARY, QUANTITATIVE
+    }
+
+    private enum NodeCompareType {
+        LT, LE
+    }
+
+    private class Node {
+
+        boolean isLeaf;
+        float value;
+        NodeFieldType fieldType;
+        NodeCompareType compareType;
+        int splitFeatureId;
+        Float splitCondition;
+        Integer yes, no, missing;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/LRModel.java

@@ -0,0 +1,169 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+
+import com.tzld.piaoquan.recommend.feature.model.sample.BaseFeature;
+import com.tzld.piaoquan.recommend.feature.model.sample.GroupedFeature;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
+import it.unimi.dsi.fastutil.longs.Long2FloatMap;
+import it.unimi.dsi.fastutil.longs.Long2FloatOpenHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+
+public class LRModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(LRModel.class);
+    private final int bucketBits = 10;  // power(2, 10) => 1024 个槽位
+    private List<Long2FloatMap> lrModel;
+
+    public LRModel() {
+        //配置不同环境的hdfs conf
+        this.lrModel = constructModel();
+    }
+
+    public List<Long2FloatMap> getLrModel() {
+        return lrModel;
+    }
+
+    public List<Long2FloatMap> constructModel() {
+        List<Long2FloatMap> initModel = new ArrayList<Long2FloatMap>();
+        int buckets = (int) Math.pow(2, bucketBits);
+        for (int i = 0; i < buckets; i++) {
+            Long2FloatMap internalModel = new Long2FloatOpenHashMap();
+            internalModel.defaultReturnValue(0.0f);
+            initModel.add(internalModel);
+        }
+        return initModel;
+    }
+
+    public int getBucket(long featureHash) {
+        return (int) (((featureHash >> bucketBits) << bucketBits) ^ featureHash);
+    }
+
+    public void putFeature(List<Long2FloatMap> model, long featureHash, float weight) {
+        model.get(getBucket(featureHash)).put(featureHash, weight);
+    }
+
+    public float getWeight(List<Long2FloatMap> model, long featureHash) {
+        return model.get(getBucket(featureHash)).get(featureHash);
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.lrModel == null)
+            return 0;
+        int sum = 0;
+        for (Map<Long, Float> model : this.lrModel) {
+            sum += model.size();
+        }
+        return sum;
+    }
+
+    public void cleanModel() {
+        this.lrModel = null;
+    }
+
+    public Float score(LRSamples lrSamples) {
+        float sum = 0.0f;
+        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
+            GroupedFeature groupedFeature = lrSamples.getFeatures(i);
+            if (groupedFeature != null && groupedFeature.getFeaturesCount() != 0) {
+                for (int j = 0; j < groupedFeature.getFeaturesCount(); j++) {
+                    BaseFeature baseFeature = groupedFeature.getFeatures(j);
+                    if (baseFeature != null) {
+                        float weight = getWeight(this.lrModel, baseFeature.getIdentifier());
+                        baseFeature.toBuilder().setWeight(weight);
+                        sum += weight;
+                    }
+                }
+            }
+        }
+
+        float pro = (float) (1.0f / (1 + Math.exp(-sum)));
+        lrSamples.toBuilder().setPredictCtr(pro);
+        return pro;
+    }
+
+    public Float getWeights(LRSamples lrSamples) {
+        float sum = 0.0f;
+
+        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
+            GroupedFeature gf = lrSamples.getFeatures(i);
+            if (gf != null && gf.getFeatures(i) != null) {
+                for (int j = 0; j < gf.getFeaturesCount(); j++) {
+                    BaseFeature fea = gf.getFeatures(j);
+                    if (fea != null) {
+                        float tmp = getWeight(this.lrModel, fea.getIdentifier());
+                        fea.toBuilder().setWeight(tmp);
+                        sum += tmp;
+                    }
+                }
+            }
+        }
+        lrSamples.toBuilder().setWeight(sum);
+        return sum;
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        List<Long2FloatMap> model = constructModel();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+
+            putFeature(model, new BigInteger(items[0].trim()).longValue(), Float.valueOf(items[1].trim()).floatValue());
+            if (cnt++ < 10) {
+                LOGGER.debug("fea: " + items[0] + ", weight: " + items[1]);
+            }
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.lrModel = model;
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+            putFeature(model, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/Model.java

@@ -0,0 +1,11 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+
+import java.io.InputStreamReader;
+
+abstract public class Model {
+    public abstract int getModelSize();
+
+    public abstract boolean loadFromStream(InputStreamReader in) throws Exception;
+}
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ModelManager.java

@@ -0,0 +1,247 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.OSSClientBuilder;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.auth.DefaultCredentialProvider;
+import com.aliyun.oss.model.OSSObject;
+import com.ctrip.framework.apollo.Config;
+import com.ctrip.framework.apollo.ConfigService;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+
+@Slf4j
+public class ModelManager {
+    private static final int SCHEDULE_PERIOD = 10;
+    private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+    private static ModelManager instance;
+    Map<String, ModelLoadTask> loadTasks = new HashMap<>();
+    Map<String, String> modelPathMap = new HashMap<>();
+    private OSS client;
+    private String bucketName;
+
+    private final String modelOssEndpoint = "model.oss.internal.endpoint";
+    private final String modelOssAccessKeyId = "model.oss.accessKeyId";
+    private final String modelOssAccessKeySecret = "model.oss.accessKetSecret";
+    private final String modelOssBucketName = "model.oss.bucketName";
+
+    private ModelManager() {
+        // config load
+        Config config = ConfigService.getAppConfig();
+        String endpoint = config.getProperty(modelOssEndpoint, "");
+        String accessKeyId = config.getProperty(modelOssAccessKeyId, "");
+        String accessKetSecret = config.getProperty(modelOssAccessKeySecret, "");
+        // oss client
+        CredentialsProvider credentialsProvider = new DefaultCredentialProvider(accessKeyId, accessKetSecret);
+        this.client = new OSSClientBuilder().build(endpoint, credentialsProvider);
+        this.bucketName = config.getProperty(modelOssBucketName, "");
+
+        config.addChangeListener(changeEvent -> {
+            if (changeEvent.isChanged(modelOssEndpoint)
+                    || changeEvent.isChanged(modelOssAccessKeyId)
+                    || changeEvent.isChanged(modelOssAccessKeySecret)) {
+                String endpointNew = config.getProperty(modelOssEndpoint, "");
+                String accessKeyIdNew = config.getProperty(modelOssAccessKeyId, "");
+                String accessKetSecretNew = config.getProperty(modelOssAccessKeySecret, "");
+                CredentialsProvider credentialsProviderNew = new DefaultCredentialProvider(accessKeyIdNew,
+                        accessKetSecretNew);
+                this.client = new OSSClientBuilder().build(endpointNew, credentialsProviderNew);
+            }
+            if (changeEvent.isChanged(modelOssBucketName)) {
+                this.bucketName = config.getProperty(modelOssBucketName, "");
+            }
+        });
+
+
+        start(SCHEDULE_PERIOD);
+    }
+
+    public static ModelManager getInstance() {
+        if (instance == null) {
+            synchronized (ModelManager.class) {
+                if (instance == null) {
+                    instance = new ModelManager();
+                }
+            }
+        }
+        return instance;
+    }
+
+    /**
+     * 添加一个加载任务到管理器
+     *
+     * @param modelName  Model的名字, 注册到ModelManager的不同model需要不同的名字
+     * @param path       Model在OSS上的全路径
+     * @param modelClass Model的子类型
+     */
+    public void registerModel(String modelName, String path, Class<? extends Model> modelClass) throws ModelRegisterException, IOException {
+        if (modelPathMap.containsKey(modelName)) {
+            // fail fast
+            // throw new RuntimeException(modelName + " already exists");
+            // hard code  广告需要视频模型打分数据，配置要分开
+            return;
+
+//            String oldPath = modelPathMap.get(modelName);
+//            if (path.equals(oldPath)) {
+//                //如果模型的path没有发生改变, 不做任何操作
+//                log.info("Model [{}] and Path [{}] has exist", modelName, path);
+//                return;
+//            } else {
+//                //如果模型的path发生改变, 需要注销掉原有的任务
+//                unRegisterModel(modelName);
+//            }
+        }
+
+        modelPathMap.put(modelName, path);
+        if (loadTasks.containsKey(path)) {
+            ModelLoadTask loadTask = loadTasks.get(path);
+            loadTask.refCount++;
+        } else {
+            ModelLoadTask task = new ModelLoadTask(path, modelClass);
+            task.refCount++;
+            loadTasks.put(path, task);
+            loadModel(task, false, true);
+        }
+    }
+
+    /**
+     * 删除一个加载任务
+     *
+     * @param modelName Model的名字, 需要和registerModel的名字一致
+     */
+    private void unRegisterModel(String modelName) {
+        if (modelPathMap.containsKey(modelName)) {
+            String path = modelPathMap.get(modelName);
+            if (loadTasks.containsKey(path)) {
+                ModelLoadTask task = loadTasks.get(path);
+                task.refCount--;
+                if (task.refCount == 0) {
+                    loadTasks.remove(path);
+                }
+            }
+            modelPathMap.remove(modelName);
+        }
+    }
+
+    /**
+     * @param modelName
+     * @return
+     */
+    public Model getModel(String modelName) {
+        if (modelPathMap.containsKey(modelName) && loadTasks.containsKey(modelPathMap.get(modelName))) {
+            return loadTasks.get(modelPathMap.get(modelName)).model;
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * 开始调度
+     *
+     * @param period
+     */
+    protected void start(long period) {
+        final Runnable task = new Runnable() {
+            public void run() {
+                // 模型更新开关
+                // boolean modelUpdateSwitch = Configuration.getBoolean("recommend-service-framework.model-update-switch", true);
+                boolean modelUpdateSwitch = true;
+                log.info("model update switch [{}]", modelUpdateSwitch);
+                if (modelUpdateSwitch) {
+                    updateModels(false);
+                }
+            }
+        };
+        scheduler.scheduleAtFixedRate(task, 10, period, TimeUnit.MINUTES); // 10分钟
+    }
+
+    /**
+     * 更新模型
+     */
+    public void updateModels(final boolean isForceLoads) {
+        log.info("begin to update: [{}]", loadTasks.keySet().size());
+        for (String modelPath : loadTasks.keySet()) {
+            log.debug("load task model path [{}]", modelPath);
+            ModelLoadTask task = loadTasks.get(modelPath);
+            loadModel(task, isForceLoads, false);
+        }
+    }
+
+    /**
+     * 检查并加载模型
+     * <p>
+     * 从oss加载：
+     * https://help.aliyun.com/zh/oss/developer-reference/streaming-download-7?spm=a2c4g.11186623.0.0.4b527c7dm8LejC
+     *
+     * @param loadTask
+     */
+    private void loadModel(final ModelLoadTask loadTask, final boolean isForceLoads, final boolean isRegister) {
+        if (loadTask.isLoading) {
+            return;
+        }
+        loadTask.isLoading = true;
+        OSSObject ossObj = null;
+        try {
+            ossObj = client.getObject(bucketName, loadTask.path);
+            long timeStamp = ossObj.getObjectMetadata().getLastModified().getTime();
+            if (loadTask.lastModifyTime < timeStamp || isForceLoads) {
+                log.info("model file changed, ready to update, last modify: [{}], current model time: [{}]",
+                        loadTask.lastModifyTime, timeStamp);
+
+                Model model = loadTask.modelClass.newInstance();
+                if (model.loadFromStream(new InputStreamReader(ossObj.getObjectContent()))) {
+                    loadTask.model = model;
+                    loadTask.lastModifyTime = timeStamp;
+                }
+            }
+            ossObj.close();
+        } catch (Exception e) {
+            log.error("update model fail", e);
+        } finally {
+            loadTask.isLoading = false;
+            if (ossObj != null) {
+                try {
+                    ossObj.close();
+                } catch (IOException e) {
+                    log.error("close ossObj fail", e);
+                }
+            }
+        }
+    }
+
+    public class ModelRegisterException extends Exception {
+
+        public ModelRegisterException(String s) {
+            super(s);
+        }
+    }
+
+    /**
+     * 调度的任务单元
+     */
+    private class ModelLoadTask {
+
+        private int refCount;
+        private String path;
+        private long lastModifyTime;
+        private boolean isLoading;
+        private Class<? extends Model> modelClass;
+        private Model model;
+
+        ModelLoadTask(String path, Class<? extends Model> modelClass) {
+            this.refCount = 0;
+            this.path = path;
+            this.lastModifyTime = 0;
+            this.modelClass = modelClass;
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/OssConfig.java

@@ -0,0 +1,15 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+import lombok.Data;
+
+/**
+ * @author dyp
+ */
+@Data
+public class OssConfig {
+
+    private String accessKeyId;
+    private String accessKeySecret;
+    private String endpoint;
+    private String bucketName;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ThompsonSamplingModel.java

@@ -0,0 +1,90 @@
+package com.tzld.piaoquan.recommend.server.framework.model;
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.base.VideoActionFeature;
+import org.apache.commons.math3.distribution.BetaDistribution;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class ThompsonSamplingModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25;  // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(ThompsonSamplingModel.class);
+
+    // key = videoid, value = < push, exp, play, realplay, share, retures >
+    private Map<Long, VideoActionFeature> thompsonSamplingModel;
+
+    private static final int alpha = 20;
+    private static final int beta_returns = 100;
+
+    public ThompsonSamplingModel() {
+        //配置不同环境的hdfs conf
+        this.thompsonSamplingModel = new HashMap<>();
+    }
+
+    public Map<Long, VideoActionFeature> getThompsonSamplingModel() {
+        return this.thompsonSamplingModel;
+    }
+
+
+
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+        Map<Long, VideoActionFeature> initModel = new HashMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 3) {
+                continue;
+            }
+            Long videoId = new BigInteger(items[0].trim()).longValue();
+            VideoActionFeature videoFeature = new VideoActionFeature();
+            videoFeature.setView(Double.valueOf(items[1].trim()));
+            videoFeature.setPlay(Double.valueOf(items[2].trim()));
+            videoFeature.setShare(Double.valueOf(items[3].trim()));
+            videoFeature.setReturns(Double.valueOf(items[5].trim()));
+            initModel.put(videoId, videoFeature);
+        }
+
+        this.thompsonSamplingModel = initModel;
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.thompsonSamplingModel == null)
+            return 0;
+        int sum = this.thompsonSamplingModel.size();
+        return sum;
+    }
+
+    public double score(RankItem rankItem) {
+        double score = 0.0f;
+        VideoActionFeature videoActionFeature = this.thompsonSamplingModel.getOrDefault(rankItem.getVideoId(), new VideoActionFeature());
+
+        int alpha = (int) videoActionFeature.getReturns() + this.alpha;
+        int beta = this.beta_returns + (int) videoActionFeature.getView();
+        score = this.betaSampler(alpha, beta);
+        return score;
+    }
+
+    public double betaSampler(double alpha, double beta) {
+        BetaDistribution betaSample = new BetaDistribution(alpha, beta);
+        return betaSample.sample();
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/AbstractFilter.java

@@ -0,0 +1,33 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller;
+
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class AbstractFilter<T> {
+    protected final static Logger LOGGER = LoggerFactory.getLogger(AbstractFilter.class);
+    protected final FilterConfigInfo filterConfigInfo;
+    protected final RecommendRequest requestContext;
+    protected final User user;
+    protected final int requestIndex;
+
+    public AbstractFilter(FilterConfigInfo filterConfigInfo,
+                          RecommendRequest requestContext,
+                          User user,
+                          Integer requestIndex) {
+
+        this.filterConfigInfo = filterConfigInfo;
+        this.requestContext = requestContext;
+        this.user = user;
+        this.requestIndex = requestIndex;
+    }
+
+    public FilterConfigInfo getFilterConfigInfo() {
+        return filterConfigInfo;
+    }
+
+    public abstract boolean predicate(Candidate candidate, T t);
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/BaseRecaller.java

@@ -0,0 +1,339 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller;
+
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.*;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.recaller.provider.ItemProvider;
+import com.tzld.piaoquan.recommend.server.framework.recaller.provider.QueueProvider;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+
+public class BaseRecaller<InMemoryItem> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(BaseRecaller.class);
+    private static final long DEFAULT_QUEUE_LOAD_TIMEOUT = 150; // ms
+    private static final long DEFAULT_PARALLEL_FILTER_TIMEOUT = 200; // ms
+    private static final ExecutorService filterExecutorService = Executors.newFixedThreadPool(128);
+    private static final ExecutorService fetchQueueExecutorService = Executors.newFixedThreadPool(128);
+
+    protected final ItemProvider<InMemoryItem> itemProvider;
+    private final QueueProvider<InMemoryItem> queueProvider;
+    private final FilterConfig filterConfig ;
+    private final long QUEUE_LOAD_TIMEOUT;
+
+    public BaseRecaller(ItemProvider<InMemoryItem> itemProvider, QueueProvider<InMemoryItem> queueProvider, FilterConfig filterConfig) {
+        this(itemProvider, queueProvider, filterConfig, DEFAULT_QUEUE_LOAD_TIMEOUT);
+    }
+
+    public BaseRecaller(ItemProvider<InMemoryItem> itemProvider, QueueProvider<InMemoryItem> queueProvider,
+                        FilterConfig filterConfig, long queueLoadTimeout) {
+        this.itemProvider = itemProvider;
+        this.queueProvider = queueProvider;
+        this.filterConfig = filterConfig;
+        this.QUEUE_LOAD_TIMEOUT = queueLoadTimeout;
+    }
+
+    public String extractItemId(Entry<InMemoryItem> entry) {
+        return entry.id;
+    }
+
+
+
+    public boolean isValidItem(InMemoryItem item){
+        return item!= null;
+
+    }
+
+    public ItemProvider<InMemoryItem> getItemProvider() {
+        return itemProvider;
+    }
+
+    public QueueProvider<InMemoryItem> getQueueProvider() {
+        return queueProvider;
+    }
+
+    public Map<Candidate, Queue<InMemoryItem>> getQueue(String queueStr) throws Exception {
+        Candidate candidate = new Candidate();
+        candidate.setCandidateKey(queueStr);
+        return loadQueues(Arrays.asList(candidate));
+    }
+
+    public Optional<InMemoryItem> getItem(String itemId) throws Exception {
+        return itemProvider.get(itemId);
+    }
+
+    public long getItemDBSize() {
+        return this.itemProvider.dbSize();
+    }
+
+    public long getIndexDBSize() {
+        return this.queueProvider.dbSize();
+    }
+
+    public long getQueueTTL(String queueNameStr, Map<String, Long> cacheRules) {
+        QueueName name = QueueName.fromString(queueNameStr);
+
+        for (String match : name.getMatches()) {
+            if (cacheRules.containsKey(match)) {
+                return cacheRules.get(match);
+            }
+        }
+        return name.getTTL();
+    }
+
+    private Map<String, Double> listToMap(List<String> list) {
+        Map<String, Double> map = new HashMap<String, Double>();
+        if (list != null) {
+            for (String elem : list) {
+                map.put(elem, 1.0);
+            }
+        }
+        return map;
+    }
+
+    /**
+     * 把queue中的entry 放入RankItem中
+     *
+     * @param entries
+     * @param candidate
+     * @param user
+     * @return
+     */
+    private List<RankItem> toHits(final Iterable<Entry<InMemoryItem>> entries, final Candidate candidate, final User user) {
+
+        List<RankItem> result = new ArrayList<RankItem>();
+        for (Entry entry : entries) {
+            RankItem item = new RankItem();
+            item.setId(extractItemId(entry));
+            item.putRankerScore("L3Score", (Double) entry.scores.get("ordering"));
+            item.setQueue(candidate.getMergeQueueName()); // merge queue
+
+            CandidateInfo candidateInfo = new CandidateInfo();
+            candidateInfo.setCandidateQueueName(candidate.getCandidateKey());
+            candidateInfo.setCandidate(candidate);
+
+            result.add(item);
+        }
+        return result;
+    }
+
+
+    // 读取redis中的数据放入queue中
+    public Map<Candidate, Queue<InMemoryItem>> loadQueues(List<Candidate> candidates) {
+
+        // update queueName
+        // final Map<String, Long> cacheRules = getCacheRulesConfig();
+        Iterable<Candidate> updateCandidates = FluentIterable.from(candidates).transform(new Function<Candidate, Candidate>() {
+            @Override
+            public Candidate apply(Candidate candidate) {
+                try {
+                    long ttl = QueueName.DEFAULT_LOCAL_CACHE_TTL;
+                    candidate.setCandidateQueueName(QueueName.fromString(candidate.getCandidateKey(), ttl));
+                } catch (Exception e) {
+                    candidate.setCandidateQueueName(null);
+                    LOGGER.error("error parse QueueName [{}]", candidate.getCandidateKey());
+                }
+                return candidate;
+            }
+        });
+
+        // parse queues
+        Iterable<QueueName> queueNames = FluentIterable.from(updateCandidates).transform(new Function<Candidate, QueueName>() {
+            @Override
+            public QueueName apply(Candidate candidate) {
+                return candidate.getCandidateQueueName();
+            }
+        });
+
+        // parallel load queues
+        // redis 或者缓存获取index
+        Map<QueueName, Queue<InMemoryItem>> queues = Maps.newConcurrentMap();
+        try {
+            queues = queueProvider.loads(Lists.newArrayList(queueNames), QUEUE_LOAD_TIMEOUT, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            LOGGER.error("load queue occur error [{}]", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        // parse candidate map
+        Map<Candidate, Queue<InMemoryItem>> candidateQueueMap = Maps.newConcurrentMap();
+        for (Candidate candidate : updateCandidates) {
+            QueueName name = candidate.getCandidateQueueName();
+            if (queues.containsKey(name) && queues.get(name) != null) {
+                candidateQueueMap.put(candidate, queues.get(name));
+            }
+        }
+        return candidateQueueMap;
+    }
+
+
+    /**
+     * recall
+     * 1. construct recall filter
+     * 2. Redis并行召回
+     * 3. do filter
+     *
+     * @param requestData
+     * @param user
+     * @param requestIndex
+     * @param recallCandidates
+     * @return
+     */
+    public List<RankItem> recalling(final RecommendRequest requestData, final User user, int requestIndex, List<Candidate> recallCandidates) {
+
+        long startTime = System.currentTimeMillis();
+        final RecallFilter<InMemoryItem> recallFilter = new RecallFilter<InMemoryItem>(this.filterConfig, requestData, user, requestIndex);
+
+        // load queue
+        long queueLoadStartTime = System.currentTimeMillis();
+
+        // load from redis
+        List<Callable<Map<Candidate, Queue<InMemoryItem>>>> fetchQueueCalls = Lists.newArrayList();
+        fetchQueueCalls.add(new Callable<Map<Candidate, Queue<InMemoryItem>>>() {
+            @Override
+            public Map<Candidate, Queue<InMemoryItem>> call() throws Exception {
+                boolean isFromRedis = true;
+                return obtainQueue(recallCandidates, requestData, user, isFromRedis);
+            }
+        });
+
+
+        List<Future<Map<Candidate, Queue<InMemoryItem>>>> fetchQueueFutures = null;
+        try {
+            fetchQueueFutures = fetchQueueExecutorService.invokeAll(fetchQueueCalls, DEFAULT_QUEUE_LOAD_TIMEOUT,
+                    TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("[fetch queue error] inter fail: {}", ExceptionUtils.getStackTrace(e));
+        } catch (Exception e) {
+            LOGGER.error("[fetch queue error] ex", ExceptionUtils.getStackTrace(e));
+        }
+
+        Map<Candidate, Queue<InMemoryItem>> candidateQueueMap = Maps.newHashMap();
+        if (CollectionUtils.isNotEmpty(fetchQueueFutures)) {
+            for (Future<Map<Candidate, Queue<InMemoryItem>>> future : fetchQueueFutures) {
+                if (future.isDone() && !future.isCancelled()) {
+                    Map<Candidate, Queue<InMemoryItem>> result = null;
+                    try {
+                        result = future.get();
+                    } catch (InterruptedException e) {
+                        LOGGER.error("[fetch queue error] InterruptedException {}", ExceptionUtils.getStackTrace(e));
+                    } catch (ExecutionException e) {
+                        LOGGER.error("[fetch queue error] ex {}", ExceptionUtils.getStackTrace(e));
+                    }
+                    if (result != null) {
+                        candidateQueueMap.putAll(result);
+                    }
+                }
+            }
+        }
+
+        // do filter
+        // 执行 recall filter配置文件中的方法
+        long filterStartTime = System.currentTimeMillis();
+
+        List<Map.Entry<Candidate, Queue<InMemoryItem>>> batch = new ArrayList<Map.Entry<Candidate, Queue<InMemoryItem>>>();
+        final List<Callable<List<RankItem>>> callables = new ArrayList<Callable<List<RankItem>>>();
+        int expectedRecallSum = 0;
+        for (final Map.Entry<Candidate, Queue<InMemoryItem>> entry : candidateQueueMap.entrySet()) {
+            callables.add(new Callable<List<RankItem>>() {
+                @Override
+                public List<RankItem> call() throws Exception {
+                    List<RankItem> candidateHits = new ArrayList<RankItem>();
+                    final Candidate candidate = entry.getKey();
+                    try {
+                        // 1. filter
+                        Iterable<Entry<InMemoryItem>> entries = FluentIterable.from(entry.getValue()).filter(new Predicate<Entry<InMemoryItem>>() {
+                            @Override
+                            public boolean apply(Entry<InMemoryItem> entry) {
+                                return isValidItem(entry.item) &&
+                                        recallFilter.predicate(candidate, entry.item);
+                            }
+                        }).limit(candidate.getCandidateNum());
+
+                        // 2. toHits
+                        candidateHits.addAll(toHits(entries, candidate, user));
+
+                        // debug log for tracing
+                        LOGGER.debug("recalled candidate [{}], queue length [{}], expected [{}], hit [{}]",
+                                new Object[]{candidate.getCandidateKey(), entry.getValue().size(), candidate.getCandidateNum(), candidateHits.size()});
+                    } catch (Exception e) {
+                        LOGGER.error("recall filter queue occur error, queue [{}], error: [{}]", candidate.toString(), ExceptionUtils.getFullStackTrace(e));
+                    }
+
+                    return candidateHits;
+                }
+            });
+        }
+
+        Map<String, RankItem> hits = new HashMap<String, RankItem>();
+        try {
+            List<Future<List<RankItem>>> futures = filterExecutorService.invokeAll(callables, DEFAULT_PARALLEL_FILTER_TIMEOUT, TimeUnit.MILLISECONDS);
+
+            for (Future<List<RankItem>> future : futures) {
+                try {
+                    if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                        List<RankItem> part = future.get();
+                        // add to result
+                        for (RankItem item : part) {
+                            // merge candidate Info
+                            if (hits.containsKey(item.getId())) {
+                                hits.get(item.getId()).addToCandidateInfoList(item.getCandidateInfo());
+                            } else {
+                                hits.put(item.getId(), item);
+                            }
+                        }
+                    } else {
+                        LOGGER.error("parallel recall filter Canceled {} ", requestData.getRequestId());
+                    }
+                } catch (Exception e) {
+                    LOGGER.error("parallel recall filter occur error, uid: [{}], Exception [{}]",
+                            requestData.getRequestId(), ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        } catch (Exception e) {
+            LOGGER.error("parallel recall filter occur error, uid: [{}], Exception [{}]",
+                    requestData.getRequestId(), ExceptionUtils.getFullStackTrace(e));
+        }
+
+        List<RankItem> result = new ArrayList<RankItem>(hits.values());
+        return result;
+    }
+
+    private Map<Candidate, Queue<InMemoryItem>> obtainQueue(List<Candidate> refactorCandidates, RecommendRequest requestData, User user, boolean isFromRedis) {
+        if (isFromRedis) {
+            return loadQueues(refactorCandidates);
+        } else {
+            return fetchQueues(refactorCandidates, requestData, user);
+        }
+
+    }
+
+    protected Map<Candidate, Queue<InMemoryItem>> fetchQueues(List<Candidate> candidates, RecommendRequest requestData, User user) {
+
+        return null;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterConfig.java

@@ -0,0 +1,99 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+
+ */
+
+public class FilterConfig {
+    private static Logger LOGGER = LoggerFactory.getLogger(FilterConfig.class);
+    private List<FilterConfigInfo> filterConfigInfoList = new ArrayList<FilterConfigInfo>();
+
+    public FilterConfig(Config config) {
+        this.load(config);
+    }
+
+
+    public FilterConfig(String configFile) {
+        this.load(configFile);
+    }
+
+    public FilterConfig() {
+    }
+
+    public boolean load(String configFile) {
+        Config filterConfig = ConfigFactory.parseResources(configFile);
+        return load(filterConfig);
+    }
+
+    public boolean load(Config config) {
+        Config recallConfig = config.getConfig("recall-config");
+
+        Config filterConf = recallConfig.getConfig("filter-config");
+        try {
+            loadFilters(filterConf);
+            int pos = 0;
+            for (FilterConfigInfo filterConfigInfo : filterConfigInfoList) {
+                LOGGER.info("filter at position [{}], priority [{}] filter name [{}] ",
+                        new Object[]{pos++, filterConfigInfo.getFilterPriority(), filterConfigInfo.getConfigName()});
+            }
+            LOGGER.debug("Load filter config success");
+        } catch (Exception e) {
+            LOGGER.error("Load filter config failed, [{}]", ExceptionUtils.getFullStackTrace(e));
+            return false;
+        }
+
+        return true;
+    }
+
+    public List<FilterConfigInfo> getFilterConfigInfoList() {
+        return filterConfigInfoList;
+    }
+
+    private void loadFilters(Config config) throws Exception {
+        ConfigObject confObj = config.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            Config conf = ((ConfigObject) it.getValue()).toConfig();
+
+            // parse config
+            String configName = it.getKey();
+            String filterName = conf.getString("filter-name");
+            int filterPriority = 0;
+            if (conf.hasPath("filter-priority"))
+                filterPriority = conf.getInt("filter-priority");
+
+            List<Integer> disableExpIds = new ArrayList<Integer>();
+            if (conf.hasPath("disable-expids")) {
+                disableExpIds = conf.getIntList("disable-expids");
+            }
+            FilterConfigInfo filterConfigInfo = new FilterConfigInfo(configName,
+                    filterName, filterPriority, disableExpIds);
+            LOGGER.debug("parse filter config info [{}]", filterConfigInfo);
+
+            addConfigByPriority(filterConfigInfoList, filterConfigInfo);
+        }
+    }
+
+    private void addConfigByPriority(List<FilterConfigInfo> configInfoList, FilterConfigInfo addConfigInfo) {
+
+        int pos = 0;
+        for (; pos < configInfoList.size(); pos++) {
+            if (configInfoList.get(pos).getFilterPriority() <= addConfigInfo.getFilterPriority()) {
+                break;
+            }
+        }
+        configInfoList.add(pos, addConfigInfo);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterConfigInfo.java

@@ -0,0 +1,59 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller;
+
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.FluentIterable;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+
+public class FilterConfigInfo {
+    private Set<Integer> disableExpIds;
+    private String filterName;
+    private Integer filterPriority;
+    private String configName;
+
+    public FilterConfigInfo(String configName,
+                            String filterName,
+                            Integer filterPriority,
+                            List<Integer> expIds) {
+        this.configName = configName;
+        this.filterName = filterName;
+        this.filterPriority = filterPriority;
+        this.disableExpIds = new HashSet<Integer>();
+        this.disableExpIds.addAll(expIds);
+    }
+
+    public Integer getFilterPriority() {
+        return filterPriority;
+    }
+
+    public String getFilterName() {
+        return filterName;
+    }
+
+    public Set<Integer> getDisableExpIds() {
+        return disableExpIds;
+    }
+
+    public String getConfigName() {
+        return configName;
+    }
+
+    @Override
+    public String toString() {
+        return configName + ":" + filterPriority + ":" + filterName + ":" +
+                Joiner.on(",").join(FluentIterable.from(disableExpIds).transform(new Function<Integer, String>() {
+                    @Nullable
+                    @Override
+                    public String apply(Integer integer) {
+                        return integer.toString();
+                    }
+                }));
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/RecallFilter.java

@@ -0,0 +1,68 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller;
+
+
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class RecallFilter<T> {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(RecallFilter.class);
+    public int filterNum;
+    private FilterConfig config;
+    private List<AbstractFilter<T>> filters;
+    private RecommendRequest requestContext;
+    private User user;
+    private int requestIndex;
+
+    public RecallFilter(FilterConfig config,
+                        RecommendRequest requestContext,
+                        User user,
+                        int requestIndex) {
+
+        this.config = config;
+        this.requestContext = requestContext;
+        this.user = user;
+        this.requestIndex = requestIndex;
+        this.filters = new ArrayList<AbstractFilter<T>>();
+        this.constructFilters(config);
+    }
+
+    public List<AbstractFilter<T>> getFilters() {
+        return filters;
+    }
+
+    public void constructFilters(FilterConfig config) {
+        // add filter
+        if (config != null) {
+            for (FilterConfigInfo filterConfigInfo : config.getFilterConfigInfoList()) {
+                try {
+                    AbstractFilter<T> filter = (AbstractFilter) Class.forName(filterConfigInfo.getFilterName())
+                            .getConstructor(FilterConfigInfo.class, RecommendRequest.class, User.class, Integer.class)
+                            .newInstance(filterConfigInfo, this.requestContext, this.user, this.requestIndex);
+
+                    this.filters.add(filter);
+                } catch (Exception e) {
+                    LOGGER.error("Filter config info construct error, [{}] [{}]",
+                            filterConfigInfo, ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public boolean predicate(Candidate candidate, T t) {
+        for (AbstractFilter filter : filters) {
+            if (!filter.predicate(candidate, t)) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/CacheEntry.java

@@ -0,0 +1,24 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+
+import com.google.common.base.Optional;
+import lombok.Data;
+
+@Data
+public class CacheEntry<T> {
+    private String itemId; // itemId
+    private Optional<T> item;
+    private Long expireTime; // 静态特征失效时间
+    private Long dynamicFeatureExpireTime; // 动态特征失效时间
+
+    public CacheEntry(String itemId,
+                      Long expireTime,
+                      Long dynamicFeatureExpireTime,
+                      Optional<T> item) {
+        this.itemId = itemId;
+        this.expireTime = expireTime;
+        this.dynamicFeatureExpireTime = dynamicFeatureExpireTime;
+        this.item = item;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/InMemoryItem.java

@@ -0,0 +1,32 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+
+import com.google.common.base.Joiner;
+import com.tzld.piaoquan.recommend.server.framework.common.ArticleInfo;
+import java.util.Arrays;
+
+
+public class InMemoryItem {
+
+    public String id;
+    public ArticleInfo articleInfo;
+    public Object itemBytes;
+
+    public InMemoryItem(String id) {
+        this.id = id;
+    }
+
+    public InMemoryItem(String id, ArticleInfo articleInfo, Object itemBytes) {
+        this.id = id;
+        this.articleInfo = articleInfo;
+        this.itemBytes = itemBytes;
+    }
+
+    @Override
+    public String toString() {
+        return Joiner.on(":").join(Arrays.asList(
+                "id=" + this.id,
+                "articleinfo=" + this.articleInfo
+        ));
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/Index.java

@@ -0,0 +1,15 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class Index {
+    private String indexName;
+    private List<IndexEntry> indexEntryList;
+
+    public Index(byte[] index){
+
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/IndexEntry.java

@@ -0,0 +1,10 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+import lombok.Data;
+
+@Data
+public class IndexEntry {
+    private String itemId;
+    private Double score;
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/ItemProvider.java

@@ -0,0 +1,301 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import com.tzld.piaoquan.recommend.server.framework.utils.FixedThreadPoolHelper;
+
+import com.tzld.piaoquan.recommend.server.framework.utils.IndexUtils;
+import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang.math.RandomUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Generalized item access to Redis.
+ */
+public class ItemProvider<InMemoryItem> {
+    private static final Logger logger = LoggerFactory.getLogger(ItemProvider.class);
+
+    private static final int REDIS_DEFAULT_TTL = 7 * 24 * 60 * 60;
+    private static final long CACHE_TIMEOUT_MS = 15 * 60 * 1000L;
+    private static final long CACHE_MAXIMUMSIZE = 80 * 10000; // default 80w
+
+    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+    private final ExecutorService itemRefreshExecutorService = new FixedThreadPoolHelper(4, "itemRefresh").getThreadPoolExecutor();
+    // 不能为 static
+    private final BlockingQueue<String> asyncRefreshQueue = new LinkedBlockingQueue<String>(100000);
+    private final RedisSmartClient client;
+    private final String prefix;
+    private final LoadingCache<String, CacheEntry<InMemoryItem>> cache;
+    private final long cacheMaximumSize;
+    private int redisTtl = REDIS_DEFAULT_TTL;
+    private long cacheTimeout = CACHE_TIMEOUT_MS;
+    private long refreshDynamicInterval = CACHE_TIMEOUT_MS;
+
+
+    /**
+     * 构造带有本地 cache 的 item provider, 使用redis作为底层存储,
+     * 本地cache基于 Guava Cache 构建, 配置失效时间与异步刷新时间;
+     *
+     * @param client         redis-cluster client
+     * @param prefix         业务标志
+     * @param cacheTimeOutMs 缓存失效时间
+     */
+    public ItemProvider(RedisSmartClient client,
+                        String prefix,
+                        long cacheTimeOutMs,
+                        long refreshDynamicInterval) {
+        this(client,
+                prefix,
+                cacheTimeOutMs,
+                refreshDynamicInterval,
+                CACHE_MAXIMUMSIZE,
+                REDIS_DEFAULT_TTL,
+                TimeUnit.SECONDS);
+    }
+
+    /**
+     * 构造带有本地 cache 的 item provider, 使用redis作为底层存储,
+     * 本地cache基于 Guava Cache 构建, 配置失效时间与异步刷新时间;
+     *
+     * @param client         redis-cluster client
+     * @param prefix         业务标志
+     * @param cacheTimeOutMs 缓存失效时间
+     * @param redisKeyTtl
+     * @param redisTtlUnit
+     */
+    public ItemProvider(RedisSmartClient client,
+                        String prefix,
+                        long cacheTimeOutMs,
+                        long refreshDynamicInterval,
+                        final long cacheMaximumSize,
+                        long redisKeyTtl,
+                        TimeUnit redisTtlUnit) {
+
+        this.cacheTimeout = cacheTimeOutMs;
+        this.refreshDynamicInterval = refreshDynamicInterval;
+        this.redisTtl = (int) redisTtlUnit.toSeconds(redisKeyTtl);
+        this.cacheMaximumSize = cacheMaximumSize;
+
+        this.client = client;
+        this.prefix = prefix;
+        this.cache = CacheBuilder.newBuilder()
+                .expireAfterWrite((long) (this.cacheTimeout * 1.4), TimeUnit.MILLISECONDS)
+                .maximumSize(cacheMaximumSize)
+                .build(new CacheLoader<String, CacheEntry<InMemoryItem>>() {
+                    @Override
+                    public CacheEntry<InMemoryItem> load(String key) throws Exception {
+                        if (key == null) {
+                            return null;
+                        } else {
+                            return loadItem(key);
+                        }
+                    }
+                });
+
+        // cache refresh
+        for (int i = 0; i < 4; i++) {
+            itemRefreshExecutorService.execute(new AsyncRefreshCacheRunnable());
+        }
+    }
+
+    public ItemProvider(RedisSmartClient client, String prefix) {
+        this(client, prefix, CACHE_TIMEOUT_MS, CACHE_TIMEOUT_MS);
+    }
+
+    public RedisSmartClient getClient() {
+        return client;
+    }
+
+
+    public InMemoryItem deserialize(byte[] bytes){
+        return null;
+    }
+
+
+    /**
+     * 初始时定义 expire time = System.currentTimeMillis()+cacheTimeout
+     * @param id
+     * @return
+     */
+    private CacheEntry<InMemoryItem> loadItem(String id) throws Exception {
+        long start = System.nanoTime();
+        byte[] bytes = client.get(IndexUtils.convertKey(prefix + id));
+        CacheEntry<InMemoryItem> result;
+        if (bytes == null) {
+            result = null;
+        } else {
+            result = makeCacheEntry(id, Optional.of(deserialize(bytes)));
+        }
+
+        return result;
+    }
+
+    public long dbSize() {
+        return this.cache.size();
+    }
+
+    private CacheEntry<InMemoryItem> makeCacheEntry(String itemId, Optional<InMemoryItem> t) {
+        long expireTime = System.currentTimeMillis() + cacheTimeout + RandomUtils.nextInt(10 * 60 * 1000);
+        long dynamicExpireTime = System.currentTimeMillis() + refreshDynamicInterval;
+        return new CacheEntry<InMemoryItem>(itemId, expireTime, dynamicExpireTime, t);
+    }
+
+
+    public CacheEntry<InMemoryItem> getCacheEntry(String id) throws ExecutionException {
+        return cache.get(id);
+
+    }
+
+    public Optional<InMemoryItem> getDirect(String id) throws Exception {
+        CacheEntry<InMemoryItem> cacheEntry = loadItem(id);
+        return cacheEntry.getItem();
+    }
+
+    public Optional<InMemoryItem> get(String id) throws ExecutionException {
+        CacheEntry<InMemoryItem> result = cache.get(id);
+
+        if (result == null || !result.getItem().isPresent()) {
+            cache.invalidate(id);
+            return null;
+        }
+
+        // minimum expire time, update key expire time
+        if (result.getExpireTime() <= 0) {
+            result = makeCacheEntry(id, result.getItem());
+            cache.put(id, result);
+        }
+
+        // check expire and refresh
+        long currentTime = System.currentTimeMillis();
+        if ((result.getDynamicFeatureExpireTime() > 0 && currentTime > result.getDynamicFeatureExpireTime()) ||
+                (result.getExpireTime() > 0 && currentTime > result.getExpireTime())) {
+            boolean ret = asyncRefreshQueue.offer(id);
+            if (logger.isDebugEnabled()) {
+                logger.debug("offer async queue itemid [{}], result [{}]", id, ret);
+            }
+
+            if (asyncRefreshQueue.size() > CACHE_MAXIMUMSIZE) {
+                logger.warn("item backgroud blocking queue length [{}], it is danger", asyncRefreshQueue.size());
+            }
+
+            if (!ret) {
+                logger.error("item backgroud offer blocking queue error, itemid [{}], with queue length [{}]", id, asyncRefreshQueue.size());
+            }
+        }
+
+        return result.getItem();
+    }
+
+    public String setLocalCache(String id, InMemoryItem t) throws ExecutionException {
+        long start = System.nanoTime();
+
+        // put to local cache
+        CacheEntry<InMemoryItem> result = makeCacheEntry(id, Optional.of(t));
+        cache.put(id, result);
+
+        return "";
+    }
+
+    public String set(String id, InMemoryItem t) throws Exception {
+        long start = System.nanoTime();
+
+        // set redis
+        String strKey = prefix + id;
+        String ret = client.setex(IndexUtils.convertKey(strKey), this.redisTtl, t.toString().getBytes());
+
+        // put to cache
+        CacheEntry<InMemoryItem> result = makeCacheEntry(id, Optional.of(t));
+        cache.put(id, result);
+
+        return ret;
+    }
+
+    public long remove(String id) throws Exception {
+        long start = System.nanoTime();
+
+        // del redis
+        long affectRow = client.del(IndexUtils.convertKey(prefix + id));
+
+        // del cache
+        cache.invalidate(id);
+        return affectRow;
+    }
+
+
+    /**
+     * 通过 blocking queue 异步刷新即将到期的item
+     */
+    public class AsyncRefreshCacheRunnable implements Runnable {
+
+        @Override
+        public void run() {
+            while (true) {
+                String itemId = "";
+                try {
+                    itemId = asyncRefreshQueue.take(); // take async item, blocked when no item
+
+                    final long currentTimeMillis = System.currentTimeMillis();
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("async refresh thread get itemid [{}]", itemId); // get itemid
+                    }
+
+                    CacheEntry<InMemoryItem> entry = cache.get(itemId);
+                    // 全量刷新
+                    if (currentTimeMillis > entry.getExpireTime()) {
+                        refreshStaticFeatures(entry);
+                    }
+                    // 刷新动态特征
+                    if (currentTimeMillis > entry.getDynamicFeatureExpireTime()) {
+                        refreshDynamicFeatures(entry);
+                    }
+
+                    long endTime = System.currentTimeMillis();
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("async refresh items spenttime [{}]", endTime - currentTimeMillis);
+                    }
+                } catch (Exception e) {
+                    logger.error("async refresh item error, itemId [{}], [{}]", itemId, ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+
+        public void refreshStaticFeatures(CacheEntry<InMemoryItem> entry) {
+            try {
+                cache.refresh(entry.getItemId());
+                if (logger.isDebugEnabled()) {
+                    logger.debug("async refresh item [{}]", entry.getItemId());
+                }
+            } catch (Exception e) {
+                logger.error("async refresh item error, itemId [{}], exception [{}]", entry.getItemId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+
+        public void refreshDynamicFeatures(CacheEntry<InMemoryItem> entry) {
+            try {
+                // reload dynamic items and update expireTime
+                entry.setItem(Optional.of(entry.getItem().get()));
+                entry.setDynamicFeatureExpireTime(System.currentTimeMillis() + refreshDynamicInterval);
+                if (logger.isDebugEnabled()) {
+                    logger.debug("async refresh items dynamic features [{}]", entry.getItemId());
+                }
+            } catch (Exception e) {
+                logger.error("async refresh item dynamic features error, itemId [{}], exception [{}]",
+                        entry.getItemId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/QueueProvider.java

@@ -0,0 +1,23 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+
+
+
+public interface QueueProvider<T> {
+
+    Queue<T> load(QueueName name) throws Exception;
+
+    long dbSize();
+
+
+    Index get(QueueName name) throws Exception;
+
+    Map<QueueName, Queue<T>> loads(List<QueueName> names) throws Exception;
+
+    Map<QueueName, Queue<T>> loads(List<QueueName> names, long timeout, TimeUnit timeUnit) throws Exception;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/RedisBackedQueueWithoutMeta.java

@@ -0,0 +1,235 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Iterables;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.utils.FixedThreadPoolHelper;
+import com.tzld.piaoquan.recommend.server.framework.utils.IndexUtils;
+import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+
+public class RedisBackedQueueWithoutMeta implements QueueProvider<String> {
+    private static final Logger logger = LoggerFactory.getLogger(RedisBackedQueueWithoutMeta.class);
+
+    private static final int DEFAULT_TTL = 7 * 24 * 60 * 60;
+    private static final long CACHE_MAXIMUMSIZE = 50 * 10000; // default 50w
+    private static final long CACHE_TIMEOUT_MS = 5 * 60 * 1000L;
+
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(64);
+
+    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+    private final ExecutorService queueRefreshExecutorService = new FixedThreadPoolHelper(4, "queueRefresh").getThreadPoolExecutor();
+
+    // 不能为static
+    private final BlockingQueue<QueueName> asyncRefreshQueue = new LinkedBlockingQueue<QueueName>(50000);
+    private final RedisSmartClient client;
+    private final LoadingCache<QueueName, Pair<Long, Queue<String>>> cache;
+
+    /**
+     * 实例化 RedisBackedQueue, 并提供基于Guava Cache的本地缓存管理工作
+     * @param client         redis-cluster 连接池
+     * @param cacheTimeOutMs 缓存有效期, 开启缓存的情况下,默认缓存5min,以写时间为基准
+     */
+    public RedisBackedQueueWithoutMeta(RedisSmartClient client, final long cacheTimeOutMs) {
+        this.client = client;
+
+        cache = CacheBuilder.newBuilder()
+                .expireAfterWrite((long) (cacheTimeOutMs * 1.3), TimeUnit.MILLISECONDS)
+                .maximumSize(CACHE_MAXIMUMSIZE)
+                .build(new RedisBackedQueueWithoutMetaCacheLoader(client));
+
+        // protect key
+        for (int i = 0; i < 4; i++) {
+            queueRefreshExecutorService.execute(new CacheRefreshRunnable());
+        }
+    }
+
+    public RedisBackedQueueWithoutMeta(RedisSmartClient client) {
+        this(client, CACHE_TIMEOUT_MS);
+    }
+
+    public RedisSmartClient getClient() {
+        return client;
+    }
+
+    public Index get(QueueName name) throws Exception {
+        byte[] bytes = client.get(IndexUtils.convertKey(name, IndexUtils.queuePrefixV1));
+        Index index = new Index(bytes);
+        return index;
+    }
+
+    public Index getDirect(QueueName name) throws Exception {
+        return get(name);
+    }
+
+    public long dbSize() {
+        return this.cache.size();
+    }
+
+    public long updateIndex(QueueName name, Index index) throws Exception {
+        return updateIndex(name.toString(), index);
+    }
+
+    public long updateIndex(String name, Index index) throws Exception {
+        Preconditions.checkArgument(StringUtils.equals(name, index.getIndexName()), "queue name not equal indexName, invalid Param");
+        long start = System.nanoTime();
+
+        // TODO fix redis key value details
+        String result = client.setex(IndexUtils.convertKey(name, IndexUtils.queuePrefixV1), this.DEFAULT_TTL, index.toString().getBytes());
+        logger.debug("updateIndexResult= {}, queue= {}, indexSize= {}", new Object[]{result, name, index.getIndexEntryList().size()});
+
+        return CollectionUtils.isNotEmpty(index.getIndexEntryList()) ? index.getIndexEntryList().size() : 0l;
+    }
+
+    /**
+     * 加载指定索引, 并指定索引本地缓存的TTL
+     *
+     * @param name QueueName
+     * @return
+     * @throws ExecutionException
+     */
+    public Queue<String> load(QueueName name) throws ExecutionException {
+
+        Pair<Long, Queue<String>> cachedQueue = cache.get(name);
+        if (cachedQueue == null) {
+            // 清理本地缓存中当前key的数据
+            cache.invalidate(name);
+            logger.debug("invalidate queue [{}]", name);
+            return new Queue<String>(name.toString());
+        }
+        // update key refresh time
+        if (cachedQueue.getKey() == Long.MIN_VALUE) {
+            cache.put(name, Pair.of(System.currentTimeMillis() + name.getTTL(), cachedQueue.getValue()));
+        }
+
+        if (name.getTTL() == 0) {
+            cache.invalidate(name);
+            logger.debug("invalidate queue [{}]", name);
+        }
+
+        // check and refresh
+        if (name.getTTL() > 0 && System.currentTimeMillis() > cachedQueue.getKey()) {
+            boolean isok = asyncRefreshQueue.offer(name);
+            if (logger.isDebugEnabled()) {
+                logger.debug("offer async queue name [{}], result [{}]", name, isok);
+            }
+            if (asyncRefreshQueue.size() > CACHE_MAXIMUMSIZE) {
+                logger.warn("index backgroud blocking queue length [{}], it is danger", asyncRefreshQueue.size());
+            }
+            if (!isok) {
+                logger.error("index backgroud offer blocking queue error, queuename [{}], with queue lenght [{}]", name, asyncRefreshQueue.size());
+            }
+        }
+
+        return cachedQueue.getValue();
+    }
+
+    public Map<QueueName, Queue<String>> loads(List<QueueName> names) throws ExecutionException {
+        return this.loads(names, 200, TimeUnit.MILLISECONDS);
+    }
+
+    public Map<QueueName, Queue<String>> loads(List<QueueName> names, long timeout, TimeUnit timeUnit) throws ExecutionException {
+
+        long startTime = System.currentTimeMillis();
+        final Iterable<List<QueueName>> namesIterable = Iterables.partition(names, 1);
+        final List<Callable<Map<QueueName, Queue<String>>>> callables = new ArrayList<Callable<Map<QueueName, Queue<String>>>>();
+        for (final List<QueueName> queueNames : namesIterable) {
+            callables.add(new Callable<Map<QueueName, Queue<String>>>() {
+                @Override
+                public Map<QueueName, Queue<String>> call() {
+
+                    Map<QueueName, Queue<String>> result = new HashMap<QueueName, Queue<String>>();
+                    for (final QueueName name : queueNames) {
+                        try {
+                            Queue queue = load(name);
+                            result.put(name, queue);
+                        } catch (Exception e) {
+                            logger.error("load queue error [{}] [{}]", name, ExceptionUtils.getFullStackTrace(e));
+                        }
+                    }
+                    return result;
+                }
+            });
+        }
+
+        Map<QueueName, Queue<String>> loadQueue = new ConcurrentHashMap<QueueName, Queue<String>>();
+        try {
+            List<Future<Map<QueueName, Queue<String>>>> futures = executorService.invokeAll(callables);
+            for (Future<Map<QueueName, Queue<String>>> future : futures) {
+                if (future.isDone() && !future.isCancelled()) {
+                    try {
+                        Map<QueueName, Queue<String>> ret = future.get();
+                        loadQueue.putAll(ret);
+                    } catch (ExecutionException ee) {
+                        logger.error("Failed to execute load a queue partition for {}", ExceptionUtils.getFullStackTrace(ee));
+                    }
+                }
+            }
+        } catch (InterruptedException ie) {
+            logger.error("Interrupted when waiting for parallel queue load finish for {}", ExceptionUtils.getFullStackTrace(ie));
+        }
+
+
+        return loadQueue;
+    }
+
+    public long len(QueueName name) throws Exception {
+        byte[] bytes = client.get(IndexUtils.convertKey(name, IndexUtils.queuePrefixV1));
+        // 检索结果
+        Index index = new Index(bytes);
+
+        return CollectionUtils.isNotEmpty(index.getIndexEntryList()) ? index.getIndexEntryList().size() : 0L;
+    }
+
+    public void delete(QueueName name) throws Exception {
+        client.del(IndexUtils.convertKey(name, IndexUtils.queuePrefixV1));
+    }
+
+
+    // 使用Guava Cache 的异步刷新接口, 异步刷新到期的key
+    public class CacheRefreshRunnable implements Runnable {
+
+        @Override
+        public void run() {
+            while (true) {
+                QueueName queueName = null;
+                try {
+                    queueName = asyncRefreshQueue.take(); // take async item, blocked when no item
+                    final long startTime = System.currentTimeMillis();
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("async refresh thread get queuename [{}]", queueName); // get itemid
+                    }
+
+                    Pair<Long, Queue<String>> entry = cache.get(queueName);
+                    // check refresh
+                    if (startTime > entry.getKey()) {
+                        cache.refresh(queueName);
+                    }
+
+                    long endTime = System.currentTimeMillis();
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("parallel refresh queue [{}] spenttime [{}]", queueName, endTime - startTime);
+                    }
+                } catch (Exception e) {
+                    logger.error("schedule refresh queue error [{}], exception [{}]", queueName, ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/RedisBackedQueueWithoutMetaCacheLoader.java

@@ -0,0 +1,57 @@
+package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
+
+import com.google.common.base.Function;
+import com.google.common.cache.CacheLoader;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Entry;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.utils.IndexUtils;
+import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+
+public class RedisBackedQueueWithoutMetaCacheLoader extends CacheLoader<QueueName, Pair<Long, Queue<String>>> {
+
+    private static final Logger logger = LoggerFactory.getLogger(RedisBackedQueueWithoutMetaCacheLoader.class);
+    private final RedisSmartClient client;
+
+    public RedisBackedQueueWithoutMetaCacheLoader(RedisSmartClient client) {
+        this.client = client;
+    }
+
+    @Override
+    public Pair<Long, Queue<String>> load(QueueName name) throws Exception {
+        long start = System.nanoTime();
+
+        byte[] bytes = client.get(IndexUtils.convertKey(name, IndexUtils.queuePrefixV1));
+        Index index = new Index(bytes);
+
+        if (null == index || CollectionUtils.isEmpty(index.getIndexEntryList())) {
+            logger.error("empty_index_fetch");
+        }
+
+        Queue<String> queue = new Queue<String>(name.toString());
+
+        Iterable<Entry<String>> iterable = Iterables.transform(index.getIndexEntryList(), new Function<IndexEntry, Entry<String>>() {
+            @Nullable
+            @Override
+            public Entry<String> apply(IndexEntry indexEntry) {
+                Entry<String> entry = new Entry<String>("", indexEntry.getItemId());
+                entry.addScore("ordering", indexEntry.getScore());
+                return entry;
+            }
+        });
+
+        queue.addAll(Lists.newArrayList(iterable));
+
+        long end = System.nanoTime();
+        return Pair.of(Long.MIN_VALUE, queue);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/AbstractScorer.java

@@ -0,0 +1,71 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.Model;
+import com.tzld.piaoquan.recommend.server.service.score.model.ModelManager;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+
+public abstract class AbstractScorer {
+    public static Logger LOGGER = LoggerFactory.getLogger(AbstractScorer.class);
+    protected ScorerConfigInfo scorerConfigInfo;
+    protected ModelManager modelManager = ModelManager.getInstance();
+
+    public AbstractScorer(ScorerConfigInfo scorerConfigInfo) {
+        this.scorerConfigInfo = scorerConfigInfo;
+    }
+
+    public void loadModel() {
+    }
+
+    public boolean isEnable() {
+        return !getScorerConfigInfo().getDisableSwitch();
+    }
+
+
+    public void doLoadModel(Class<? extends Model> modelClass) {
+
+        String modelPath = scorerConfigInfo.getModelPath();
+        if (StringUtils.isNotBlank(modelPath)) {
+            try {
+                // 使用 modelPath 作为 modelName 注册
+                modelManager.registerModel(modelPath, modelPath, modelClass);
+                LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
+            } catch (ModelManager.ModelRegisterException e) {
+                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+            } catch (IOException e) {
+                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+            }
+        } else {
+            LOGGER.error("modelpath is null, for model class [{}]", modelClass);
+        }
+    }
+
+    public Model getModel() {
+        if (StringUtils.isBlank(scorerConfigInfo.getModelPath())) {
+            return null;
+        }
+        return modelManager.getModel(scorerConfigInfo.getModelPath());
+    }
+
+    public ScorerConfigInfo getScorerConfigInfo() {
+        return scorerConfigInfo;
+    }
+
+    public abstract List<RankItem> scoring(final ScoreParam param,
+                                           final UserFeature userFeature,
+                                           final List<RankItem> rankItems);
+
+    public abstract List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                           final Map<String, String> userFeatureMap,
+                                           final List<RankItem> rankItems);
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseGBDTModelScorer.java

@@ -0,0 +1,19 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.tzld.piaoquan.recommend.server.service.score.model.GBDTModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class BaseGBDTModelScorer extends AbstractScorer {
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseGBDTModelScorer.class);
+
+    public BaseGBDTModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public synchronized void loadModel() {
+        doLoadModel(GBDTModel.class);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseLRModelScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseLRModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRModelScorer.class);
+
+    public BaseLRModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(LRModel.class);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseThompsonSamplingScorer.java

@@ -0,0 +1,32 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.ThompsonSamplingModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+
+public abstract class BaseThompsonSamplingScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseThompsonSamplingScorer.class);
+
+    public BaseThompsonSamplingScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(ThompsonSamplingModel.class);
+    }
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                           final Map<String, String> userFeatureMap,
+                                           final List<RankItem> rankItems){
+
+        return rankItems;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScoreParam.java

@@ -0,0 +1,13 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
+import lombok.Data;
+
+/**
+ * @author dyp
+ */
+@Data
+public class ScoreParam {
+    private String mid;
+    private RequestContext requestContext;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerConfig.java

@@ -0,0 +1,140 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+public class ScorerConfig {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
+    private List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
+
+    public ScorerConfig(Config config) {
+        this.load(config);
+    }
+
+    public ScorerConfig(String configFile) {
+        this.load(configFile);
+    }
+
+    public ScorerConfig() {
+    }
+
+    public static Config mergeConfig(Config baselineConfig, List<Config> expConfigs) {
+        if (expConfigs == null || expConfigs.size() == 0) {
+            return baselineConfig;
+        }
+        Config resultConfig = expConfigs.get(0);
+        for (Config config : expConfigs) {
+            resultConfig = resultConfig.withFallback(config);
+        }
+        resultConfig = resultConfig.withFallback(baselineConfig);
+        return resultConfig;
+    }
+
+    public boolean load(String configFile) {
+        Config config = ConfigFactory.parseResources(configFile);
+        return load(config);
+    }
+
+    public boolean load(Config baselineConfig, List<Config> expConfigs) {
+        Config config = mergeConfig(baselineConfig, expConfigs);
+        return load(config);
+    }
+
+    public boolean load(Config config) {
+        Config scorerConfig = config.getConfig("scorer-config");
+
+        try {
+            loadScorers(scorerConfig);
+            int pos = 0;
+            for (ScorerConfigInfo scorerConfigInfo : configInfoList) {
+                LOGGER.debug("scorer at position [{}], priority [{}], scorer name [{}]",
+                        new Object[]{pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName()});
+            }
+            LOGGER.debug("Load scorer config success");
+        } catch (Exception e) {
+            LOGGER.error("Load scorer config failed, [{}]", ExceptionUtils.getFullStackTrace(e));
+            return false;
+        }
+
+        return true;
+    }
+
+    public List<ScorerConfigInfo> getConfigInfoList() {
+        return configInfoList;
+    }
+
+
+    public String loadOptionStringConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getString(path) : null;
+    }
+
+    private Config loadOptionConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
+    }
+
+    private void loadScorers(Config config) throws Exception {
+
+        ConfigObject confObj = config.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            Config conf = ((ConfigObject) it.getValue()).toConfig();
+            // parse config
+            String configName = it.getKey();
+            String scorerName = conf.getString("scorer-name");
+            int scorerPriority = 0;
+            if (conf.hasPath("scorer-priority"))
+                scorerPriority = conf.getInt("scorer-priority");
+            Boolean disableSwitch = false;
+            if (conf.hasPath("disable-switch")) {
+                disableSwitch = conf.getBoolean("disable-switch");
+            }
+            Config paramConfig = loadOptionConfig(conf, "param-config");
+            // model path
+            String modelPath = loadOptionStringConfig(conf, "model-path");
+            if (modelPath == null) {
+                modelPath = loadOptionStringConfig(conf, "default-model-path");
+                LOGGER.debug("model-path is not exists in config file, use default-model-path instead, modelPath={}", modelPath);
+            }
+            // enable queues
+            Set<String> enableQueues = new HashSet<String>();
+            if (conf.hasPath("enable-queues")) {
+                enableQueues.addAll(conf.getStringList("enable-queues"));
+            }
+            ScorerConfigInfo configInfo = new ScorerConfigInfo(configName,
+                    scorerName,
+                    scorerPriority,
+                    disableSwitch,
+                    enableQueues,
+                    modelPath,
+                    paramConfig
+            );
+            LOGGER.debug("parse scorer config info [{}]", configInfo);
+            // add to ConfigInfoList
+            addConfigByPriority(configInfoList, configInfo);
+        }
+    }
+
+    private void addConfigByPriority(List<ScorerConfigInfo> configInfoList, ScorerConfigInfo addConfigInfo) {
+
+        int pos = 0;
+        for (; pos < configInfoList.size(); pos++) {
+            if (configInfoList.get(pos).getScorerPriority() <= addConfigInfo.getScorerPriority()) {
+                break;
+            }
+        }
+
+        configInfoList.add(pos, addConfigInfo);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerConfigInfo.java

@@ -0,0 +1,78 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.google.gson.Gson;
+import com.typesafe.config.Config;
+
+import java.util.Set;
+
+
+public class ScorerConfigInfo {
+
+    private String configName;
+    private Integer scorerPriority;
+    private Boolean disableSwitch;
+    private String scorerName;
+    private Set<String> enableQueues;
+    private String modelPath;
+    private Config paramConfig; // param config
+
+    public ScorerConfigInfo(String configName,
+                            String scorerName,
+                            Integer scorerPriority,
+                            Boolean disableSwitch,
+                            Set<String> enableQueues,
+                            String modelPath,
+                            Config paramConfig) {
+
+        this.configName = configName;
+        this.scorerName = scorerName;
+        this.scorerPriority = scorerPriority;
+        this.disableSwitch = disableSwitch;
+        this.enableQueues = enableQueues;
+        this.modelPath = modelPath;
+        this.paramConfig = paramConfig;
+    }
+
+    public Config getParamConfig() {
+        return paramConfig;
+    }
+
+    public Set<String> getEnableQueues() {
+        return enableQueues;
+    }
+
+    public void setEnableQueues(Set<String> enableQueues) {
+        this.enableQueues = enableQueues;
+    }
+
+    public Integer getScorerPriority() {
+        return scorerPriority;
+    }
+
+    public String getScorerName() {
+        return scorerName;
+    }
+
+    public String getConfigName() {
+        return configName;
+    }
+
+    public boolean isQueueEnable(String queueName) {
+        return this.enableQueues == null ||
+                this.enableQueues.isEmpty() ||
+                this.enableQueues.contains(queueName);
+    }
+
+    public String getModelPath() {
+        return modelPath;
+    }
+
+    public Boolean getDisableSwitch() {
+        return disableSwitch;
+    }
+
+    @Override
+    public String toString() {
+        return new Gson().toJson(this);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerPipeline.java

@@ -0,0 +1,187 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+
+@Slf4j
+public class ScorerPipeline {
+    public static final int corePoolSize = 128;
+    public static final int SCORE_TIME_OUT = 400;
+    public static final Logger LOGGER = LoggerFactory.getLogger(ScorerPipeline.class);
+    public static final ExecutorService executorService = Executors.newFixedThreadPool(corePoolSize);
+
+    public List<AbstractScorer> scorers;
+
+    public ScorerPipeline(List<AbstractScorer> scorers) {
+        this.scorers = scorers;
+    }
+
+    /**
+     * scoring
+     *
+     * @return
+     */
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+
+        if (CollectionUtils.isEmpty(scorers)) {
+            log.error("scorers is empty");
+            return rankItems;
+        }
+        List<RankItem> items = rankItems;
+
+        for (final AbstractScorer scorer : scorers) {
+            if (!scorer.isEnable()) {
+                continue;
+            }
+
+            final int beforeSize = rankItems.size();
+            final long startTime = System.currentTimeMillis();
+
+            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
+            String[] scorerNames = fullScorerName.split("\\.");
+            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
+
+            final List<RankItem> scoreRankerItems = items;
+            Callable<List<RankItem>> callable = () -> scorer.scoring(param, userFeature, scoreRankerItems);
+
+            // execute score use thread to protected score worst time
+            List<RankItem> scoredItems = new ArrayList<RankItem>();
+            try {
+                List<Future<List<RankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
+                for (Future<List<RankItem>> future : futures) {
+                    try {
+                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                            scoredItems.addAll(future.get());
+                        } else {
+                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
+                                    new Object[]{scorerName, scoreRankerItems.size()});
+                        }
+                    } catch (Exception e) {
+                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
+                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+                    }
+                }
+            } catch (Exception e) {
+                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
+                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+            }
+
+            //  变更item
+            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
+                items = scoreRankerItems;
+            } else {
+                items = new ArrayList<>(items);
+            }
+
+            int position = 0;
+            for (RankItem item : items) {
+                item.getRankerIndex().put(scorerName, position++);
+                item.getRankerScore().put(scorerName, item.getScore());
+            }
+
+            //
+            long spentTime = System.currentTimeMillis() - startTime;
+            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
+                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
+        }
+
+        int position = 0;
+        for (RankItem item : items) {
+            item.getRankerIndex().put("finalScore", position++);
+            item.getRankerScore().put("finalScore", item.getScore());
+        }
+
+        return items;
+    }
+
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                  final Map<String, String> userFeatureMap,
+                                  final List<RankItem> rankItems) {
+
+        if (CollectionUtils.isEmpty(scorers)) {
+            log.error("scorers is empty");
+            return rankItems;
+        }
+        List<RankItem> items = rankItems;
+
+        for (final AbstractScorer scorer : scorers) {
+            if (!scorer.isEnable()) {
+                continue;
+            }
+
+            final int beforeSize = rankItems.size();
+            final long startTime = System.currentTimeMillis();
+
+            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
+            String[] scorerNames = fullScorerName.split("\\.");
+            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
+
+            final List<RankItem> scoreRankerItems = items;
+            Callable<List<RankItem>> callable = () -> scorer.scoring(sceneFeatureMap, userFeatureMap, scoreRankerItems);
+
+            // execute score use thread to protected score worst time
+            List<RankItem> scoredItems = new ArrayList<RankItem>();
+            try {
+                List<Future<List<RankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
+                for (Future<List<RankItem>> future : futures) {
+                    try {
+                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                            scoredItems.addAll(future.get());
+                        } else {
+                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
+                                    new Object[]{scorerName, scoreRankerItems.size()});
+                        }
+                    } catch (Exception e) {
+                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
+                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+                    }
+                }
+            } catch (Exception e) {
+                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
+                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+            }
+
+            //  变更item
+            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
+                items = scoreRankerItems;
+            } else {
+                items = new ArrayList<>(items);
+            }
+
+            int position = 0;
+            for (RankItem item : items) {
+                item.getRankerIndex().put(scorerName, position++);
+                if(scorerName.equals("finalScore")) {
+                    item.getRankerScore().put(scorerName, item.getScore());
+                }
+            }
+
+            //
+            long spentTime = System.currentTimeMillis() - startTime;
+            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
+                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
+        }
+
+        int position = 0;
+        for (RankItem item : items) {
+            item.getRankerIndex().put("finalScore", position++);
+            item.getRankerScore().put("finalScore", item.getScore());
+        }
+
+        return items;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerUtils.java

@@ -0,0 +1,153 @@
+package com.tzld.piaoquan.recommend.server.framework.score;
+
+
+import com.typesafe.config.Config;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+@Slf4j
+public final class ScorerUtils {
+    private static Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
+
+    private static Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
+
+    public static String BASE_CONF = "feeds_score_config_baseline.conf";
+    public static String VIDEO_SCORE_CONF_FOR_AD = "video_score_config_for_ad.conf";
+
+    public static String FLOWPOOL_CONF = "feeds_score_config_thompson.conf";
+
+
+    public static void warmUp() {
+        log.info("scorer warm up ");
+        ScorerUtils.init(BASE_CONF);
+        ScorerUtils.init(FLOWPOOL_CONF);
+        ScorerUtils.init(VIDEO_SCORE_CONF_FOR_AD);
+    }
+
+    private ScorerUtils() {
+        // init(BASE_CONF);
+    }
+
+    public static void init(String configFile) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(configFile);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+
+        initLoadModel(scorers);
+        scorerPipelineCache.put(configFile, new ScorerPipeline(scorers));
+    }
+
+    /**
+     * init load model
+     *
+     * @param scorers
+     */
+    public static void initLoadModel(List<AbstractScorer> scorers) {
+        for (AbstractScorer scorer : scorers) {
+            if (scorer.isEnable()) {
+                scorer.loadModel();
+            }
+        }
+    }
+
+    public static void initLoadModel(String configFile) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(configFile);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        initLoadModel(scorers);
+    }
+
+
+    public static void initLoadModel(Config config) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(config);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        initLoadModel(scorers);
+    }
+
+    public static ScorerPipeline getScorerPipeline(String configFile) {
+        // 不需要保证严格意义的单例
+        if (scorerPipelineCache.containsKey(configFile)) {
+            return scorerPipelineCache.get(configFile);
+        }
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(configFile);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        ScorerPipeline pipeline = new ScorerPipeline(scorers);
+        scorerPipelineCache.put(configFile, pipeline);
+        return pipeline;
+    }
+
+    public static ScorerPipeline getScorerPipeline(Config mergeConfig) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(mergeConfig);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        return new ScorerPipeline(scorers);
+    }
+
+    public static ScorerPipeline getScorerPipeline(Config baselineConfig, List<Config> configList) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(baselineConfig, configList);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        return new ScorerPipeline(scorers);
+    }
+
+    /**
+     * construct scorers
+     *
+     * @param scorerConfig
+     * @return
+     */
+    public static List<AbstractScorer> constructScorers(ScorerConfig scorerConfig) {
+
+        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
+        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
+            if (!configInfo.getDisableSwitch()) {
+                try {
+                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
+                            .getConstructor(ScorerConfigInfo.class)
+                            .newInstance(configInfo);
+                    scorers.add(scorer);
+                    LOGGER.debug("construct score [{}]", configInfo.getScorerName());
+                } catch (Exception e) {
+                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
+                }
+
+            }
+        }
+        return scorers;
+    }
+
+    /**
+     * construct scorers
+     *
+     * @param scorerConfig
+     * @return
+     */
+    public static List<AbstractScorer> constructQueueScorers(ScorerConfig scorerConfig, String queueName) {
+
+        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
+        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
+            if (!configInfo.getDisableSwitch() && configInfo.isQueueEnable(queueName)) {
+                try {
+                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
+                            .getConstructor(ScorerConfigInfo.class)
+                            .newInstance(configInfo);
+                    scorers.add(scorer);
+                    LOGGER.debug("construct queue scorer [{}] [{}]", queueName, configInfo.getScorerName());
+                } catch (Exception e) {
+                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+        return scorers;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/AbstractUserAttentionExtractor.java

@@ -0,0 +1,28 @@
+package com.tzld.piaoquan.recommend.server.framework.userattention;
+
+
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+public abstract class AbstractUserAttentionExtractor {
+    protected UserAttentionExtractorConfig extractorConfig;
+
+    public AbstractUserAttentionExtractor(UserAttentionExtractorConfig extractorConfig) {
+        this.extractorConfig = extractorConfig;
+    }
+
+    public String getExtractorName() {
+        String[] arr = this.extractorConfig.getName().split(".");
+        return arr.length > 0 ? arr[arr.length - 1] : "UnKnownExtractor";
+    }
+
+    public UserAttentionExtractorConfig getExtractorConfig() {
+        return extractorConfig;
+    }
+
+
+    public abstract void extractAttention(final RecommendRequest requestData, final User user);
+
+    public void loadModel() {
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorConfig.java

@@ -0,0 +1,44 @@
+package com.tzld.piaoquan.recommend.server.framework.userattention;
+
+import com.typesafe.config.Config;
+
+
+
+public class UserAttentionExtractorConfig {
+    private final String name;  // attention extractor name
+    private final Integer priority;
+    private final String modelPath;
+    private final Config paramConfig; // param config
+
+    public UserAttentionExtractorConfig(String name,
+                                        Config paramConfig,
+                                        Integer priority,
+                                        String modelPath) {
+        this.name = name;
+        this.paramConfig = paramConfig;
+        this.priority = priority;
+        this.modelPath = modelPath;
+    }
+
+    public Integer getPriority() {
+        return priority;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Config getParamConfig() {
+        return paramConfig;
+    }
+
+    @Override
+    public String toString() {
+        return "AttentionExtractorConfig{" +
+                "name='" + name + '\'' +
+                ", priority=" + priority +
+                ", modelPath='" + modelPath + '\'' +
+                ", paramConfig=" + paramConfig +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorPipeline.java

@@ -0,0 +1,36 @@
+package com.tzld.piaoquan.recommend.server.framework.userattention;
+
+
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.common.UserAttention;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class UserAttentionExtractorPipeline {
+    private List<AbstractUserAttentionExtractor> extractorList = new ArrayList<AbstractUserAttentionExtractor>();
+
+    public UserAttentionExtractorPipeline(List<AbstractUserAttentionExtractor> extractorList) {
+        this.extractorList = extractorList;
+    }
+
+    public void extractAttention(final RecommendRequest requestData, final User user) {
+        if (user.getUserAttention() == null) {
+            UserAttention userAttention = new UserAttention();
+            user.setUserAttention(userAttention);
+        }
+
+        long startTime = System.currentTimeMillis();
+
+        for (AbstractUserAttentionExtractor extractor : extractorList) {
+            try {
+                long currStartTime = System.currentTimeMillis();
+                extractor.extractAttention(requestData, user);
+                // log
+            } catch (Exception e) {
+                // log
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorUtils.java

@@ -0,0 +1,136 @@
+package com.tzld.piaoquan.recommend.server.framework.userattention;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class UserAttentionExtractorUtils {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(UserAttentionExtractorUtils.class);
+
+    public static final String BASE_CONF = "";
+
+    /**
+     * 主调用函数
+     *
+     * @param config
+     */
+    public static void initLoadModel(Config config) {
+        initLoadModel(constructExtractorList(constructExtractorConfigList(config)));
+    }
+
+
+    public static UserAttentionExtractorPipeline getAtttentionPipeline(String configFile) {
+
+        Config attentionConfig = ConfigFactory.parseResources(configFile);
+        List<AbstractUserAttentionExtractor> attentionExtractors = constructExtractorList(constructExtractorConfigList(attentionConfig));
+        UserAttentionExtractorPipeline pipeline = new UserAttentionExtractorPipeline(attentionExtractors);
+
+        return pipeline;
+    }
+
+
+    public static List<UserAttentionExtractorConfig> constructExtractorConfigList(Config config) {
+        long startTime = System.currentTimeMillis();
+        List<UserAttentionExtractorConfig> configList = new ArrayList<UserAttentionExtractorConfig>();
+        try {
+            Config mergedConfig = config.getConfig("attention-extractor-config");
+            for (ConfigObject.Entry<String, ConfigValue> it : mergedConfig.root().entrySet()) {
+                LOGGER.debug("deal with base config, attention key: [{}]", it.getKey());
+                Config entryConf = ((ConfigObject) it.getValue()).toConfig();
+                UserAttentionExtractorConfig extractorConfig = parseExtractorConfig(entryConf);
+                addConfigByPriority(configList, extractorConfig);
+            }
+            // debug log
+            if (LOGGER.isDebugEnabled()) {
+                for (UserAttentionExtractorConfig extractorConfig : configList) {
+                    LOGGER.debug("attention extractor config [{}] [{}]", extractorConfig.getName(), extractorConfig.toString());
+                }
+            }
+        } catch (Exception e) {
+            LOGGER.error("construct extractor config {} failed {}", config.toString(), ExceptionUtils.getFullStackTrace(e));
+        }
+        return configList;
+    }
+
+
+    private static List<AbstractUserAttentionExtractor> constructExtractorList(List<UserAttentionExtractorConfig> configList) {
+        long startTime = System.currentTimeMillis();
+        List<AbstractUserAttentionExtractor> extractorList = new ArrayList<AbstractUserAttentionExtractor>();
+        for (UserAttentionExtractorConfig config : configList) {
+            try {
+                AbstractUserAttentionExtractor extractor = (AbstractUserAttentionExtractor) Class.forName(config.getName())
+                        .getConstructor(UserAttentionExtractorConfig.class)
+                        .newInstance(config);
+                extractorList.add(extractor);
+            } catch (Exception e) {
+                LOGGER.error("instance extractor {} failed {}", config.getName(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        return extractorList;
+    }
+
+    private static String loadOptionStringConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getString(path) : null;
+    }
+
+    private static Config loadOptionConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
+    }
+
+    private static int loadOptionIntConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getInt(path) : 0;
+    }
+
+    /**
+     * attention层的配置化参数
+     */
+    private static UserAttentionExtractorConfig parseExtractorConfig(Config extractorConfig) {
+        String attentionExtractorName = extractorConfig.getString("name");
+        Integer priority = loadOptionIntConfig(extractorConfig, "attention-priority");
+        Config paramConfig = loadOptionConfig(extractorConfig, "param-config");
+        String modelPath = loadOptionStringConfig(extractorConfig, "model-path");
+        return new UserAttentionExtractorConfig(attentionExtractorName,
+                paramConfig,
+                priority,
+                modelPath);
+    }
+
+
+    /**
+     * attention支持模型获取
+     *
+     * @param attentionExtractors
+     */
+    private static void initLoadModel(List<AbstractUserAttentionExtractor> attentionExtractors) {
+        for (AbstractUserAttentionExtractor attentionExtractor : attentionExtractors) {
+            attentionExtractor.loadModel();
+        }
+    }
+
+    /**
+     * 按优先级处理逻辑
+     *
+     * @param configInfoList
+     * @param extractorConfig
+     */
+    private static void addConfigByPriority(List<UserAttentionExtractorConfig> configInfoList, UserAttentionExtractorConfig extractorConfig) {
+        int pos = 0;
+        for (; pos < configInfoList.size(); pos++) {
+            if (configInfoList.get(pos).getPriority() <= extractorConfig.getPriority()) {
+                break;
+            }
+        }
+        configInfoList.add(pos, extractorConfig);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/FixedThreadPoolHelper.java

@@ -0,0 +1,11 @@
+package com.tzld.piaoquan.recommend.server.framework.utils;
+
+public class FixedThreadPoolHelper extends ThreadPoolHelper {
+    public FixedThreadPoolHelper(int nThread) {
+        super(ThreadPoolUtils.newFixedThreadPool(nThread));
+    }
+
+    public FixedThreadPoolHelper(int nThread, String name) {
+        super(ThreadPoolUtils.newFixedThreadPool(nThread, name), "FixedPool_" + name);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/IndexUtils.java

@@ -0,0 +1,37 @@
+package com.tzld.piaoquan.recommend.server.framework.utils;
+
+
+import com.google.common.base.Charsets;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+
+/**
+ * 正排倒排的key相关的utils
+ */
+public class IndexUtils {
+    /**
+     * 倒排索引专用的前缀, 用于区分同一个queue的多个存储版本, 读写要一致
+     **/
+    public static final String queuePrefixV1 = "a:";
+
+    public static byte[] convertKey(String itemId) {
+        return itemId.getBytes(Charsets.UTF_8);//BaseSafeEncoder.SAFE_CHARSET
+    }
+
+    /**
+     * 用于转换倒排索引queue, 写入和读取的前缀要一致
+     **/
+    public static byte[] convertKey(QueueName queueName, String queuePrefix) throws Exception {
+        return convertKey(queueName.toString(), queuePrefix);
+    }
+
+    /**
+     * 用于转换倒排索引queue, 写入和读取的前缀要一致
+     **/
+    public static byte[] convertKey(String queueName, String queuePrefix) throws Exception {
+        return convertKey(queuePrefix + queueName);
+    }
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/RedisSmartClient.java

@@ -0,0 +1,92 @@
+package com.tzld.piaoquan.recommend.server.framework.utils;
+
+
+import com.google.common.base.Function;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import redis.clients.jedis.JedisCluster;
+import redis.clients.jedis.Tuple;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class RedisSmartClient {
+
+    private final static ExecutorService executorService = Executors.newCachedThreadPool();
+    @Qualifier("redisTemplate")
+    protected RedisTemplate<String, String> redisTemplate;
+
+
+    public RedisSmartClient() {
+    }
+
+
+
+
+    public String set(final byte[] key, final byte[] value) {
+        return "";
+
+    }
+
+    public byte[] get(final byte[] key) {
+        return "".getBytes();
+    }
+
+
+    public String setex(final byte[] key, final int seconds, final byte[] value) {
+        return "";
+
+    }
+
+
+
+    public Long del(final byte[] key) {
+        return 1L;
+
+    }
+
+
+
+    public Long hset(final byte[] key, final byte[] field, final byte[] value) {
+        return 1L;
+    }
+
+    public Long hlen(final byte[] key) {
+        return 1L;
+
+    }
+
+
+
+
+
+    public String set(final String key, final String value) {
+        return "";
+
+
+    }
+
+    public String get(final String key) {
+        return "";
+
+
+    }
+
+
+    public String hget(final String key, final String field) {
+        return "";
+
+    }
+
+    public String hmset(final String key, final Map<String, String> hash) {
+        return "";
+
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/ThreadPoolHelper.java

@@ -0,0 +1,64 @@
+package com.tzld.piaoquan.recommend.server.framework.utils;
+
+
+import lombok.Data;
+
+import java.util.Timer;
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * 线程池Helper
+ *   可以命名线程池; 保存ThreadPoolExecutor实例, 方便监控线程池信息
+ */
+@Data
+public class ThreadPoolHelper {
+    private String name;
+    private ThreadPoolExecutor threadPoolExecutor;
+    private Timer counterTimer;
+
+    ThreadPoolHelper(ThreadPoolExecutor threadPoolExecutor) {
+        this.threadPoolExecutor = threadPoolExecutor;
+        this.name = "";
+    }
+
+    ThreadPoolHelper(ThreadPoolExecutor threadPoolExecutor, String name) {
+        this.threadPoolExecutor = threadPoolExecutor;
+        this.name = name;
+    }
+
+
+    public long getTaskCount() {
+        return this.threadPoolExecutor.getTaskCount();
+    }
+
+    public long getActiveCount() {
+        return this.threadPoolExecutor.getActiveCount();
+    }
+
+    public long getCorePoolSize() {
+        return this.threadPoolExecutor.getCorePoolSize();
+    }
+
+    public long getPoolSize() {
+        return this.threadPoolExecutor.getPoolSize();
+    }
+
+    public long getMaximumPoolSize() {
+        return this.threadPoolExecutor.getMaximumPoolSize();
+    }
+
+    public long getLargestPoolSize() {
+        return this.threadPoolExecutor.getLargestPoolSize();
+    }
+
+    public long getTaskQueueSize() {
+        return this.threadPoolExecutor.getQueue().size();
+    }
+
+    public void cancelCounter() {
+        if (counterTimer != null) {
+            counterTimer.cancel();
+            counterTimer.purge();
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/ThreadPoolUtils.java

@@ -0,0 +1,46 @@
+package com.tzld.piaoquan.recommend.server.framework.utils;
+
+
+import java.util.concurrent.*;
+
+/**
+ * 命名线程池工场类
+ * 最后创建的线程名称:
+ *   name_{counter}
+ */
+class NamedThreadFactory implements ThreadFactory {
+    private int counter = 0;
+    private String name;
+    NamedThreadFactory(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public Thread newThread(Runnable r) {
+        counter += 1;
+        return new Thread(r, name + "_" + counter);
+    }
+}
+
+
+public class ThreadPoolUtils {
+    public static ThreadPoolExecutor newFixedThreadPool(int nThreads, String poolName) {
+        return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
+                new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("Fixed_" + poolName));
+    }
+    public static ThreadPoolExecutor newFixedThreadPool(int nThreads) {
+        return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
+                new LinkedBlockingQueue<Runnable>());
+    }
+
+    public static ThreadPoolExecutor newCachedThreadPool(String poolName) {
+        return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
+                new SynchronousQueue<Runnable>(), new NamedThreadFactory("Cached_" + poolName));
+    }
+
+    public static ThreadPoolExecutor newCachedThreadPool() {
+        return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
+                new SynchronousQueue<Runnable>());
+    }
+}
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/attention/SimpleAttentionExtractor.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.implement.attention;
+
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.userattention.AbstractUserAttentionExtractor;
+import com.tzld.piaoquan.recommend.server.framework.userattention.UserAttentionExtractorConfig;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+public class SimpleAttentionExtractor extends AbstractUserAttentionExtractor {
+
+    public SimpleAttentionExtractor(UserAttentionExtractorConfig config) {
+        super(config);
+    }
+
+    @Override
+    public void extractAttention(RecommendRequest recommendRequestData, User user) {
+        //TODO
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/SimpleHotCandidate.java

@@ -0,0 +1,43 @@
+package com.tzld.piaoquan.recommend.server.implement.candidate;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.IndexCandidateQueue;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class SimpleHotCandidate extends IndexCandidateQueue {
+
+    public static final String ItemType = "vlog";
+
+    public SimpleHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int currRecall = 0;
+        String region = user.getRegion();
+        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
+
+        QueueName queueName24HRegion = new QueueName(ItemType, "rov")
+                .addMatch("type", "region24h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName24HRegion, 1, getStrategyQueueInfo().getQueueName());
+
+        QueueName queueName1HRegion = new QueueName(ItemType, "rov")
+                .addMatch("type", "region1h")
+                .addMatch("region", region);
+
+        currRecall += addCandidateKey(simplifiedCandidates, queueName1HRegion, 1, getStrategyQueueInfo().getQueueName());
+
+        candidates.putAll(simplifiedCandidates);
+        return currRecall;
+
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recaller/HistoryLongPeriodFilter.java

@@ -0,0 +1,35 @@
+package com.tzld.piaoquan.recommend.server.implement.recaller;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
+import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
+import com.tzld.piaoquan.recommend.server.framework.recaller.provider.InMemoryItem;
+import com.google.common.collect.Sets;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.util.Set;
+
+
+public class HistoryLongPeriodFilter extends AbstractFilter<InMemoryItem> {
+
+    protected Set<String> historySet = Sets.newHashSet();
+
+    public HistoryLongPeriodFilter(FilterConfigInfo filterConfigInfo,
+                         RecommendRequest recommendRequest,
+                         User user,
+                         Integer requestIndex) {
+        super(filterConfigInfo, recommendRequest, user, requestIndex);
+        // 用户近1个月播放行为历史历史
+        historySet = Sets.newHashSet(user.getLast1monthUserAction().getFeedsPlayed());
+        if (historySet == null || historySet.size() == 0) {
+            historySet = Sets.newHashSet();
+        }
+    }
+
+    public boolean predicate(Candidate candidate, InMemoryItem t) {
+        return !this.historySet.contains(t.id);
+    }
+
+
+}

recommend-server-service/src/main/resources/attention_config.conf

@@ -0,0 +1,6 @@
+attention-extractor-config = {
+  categories-by-ctr-extractor = {
+    name = "com.tzld.piaoquan.recommend.server.implement.attention.SimpleAttentionExtractor"
+  }
+
+}

recommend-server-service/src/main/resources/filter_config.conf

@@ -0,0 +1,7 @@
+recall-config = {
+  filter-config = {
+    history-filter-config = {
+      filter-name = "com.tzld.piaoquan.recommend.server.implement.recaller.HistoryLongPeriodFilter"
+    }
+  }
+}

recommend-server-service/src/main/resources/merge_config.conf

@@ -0,0 +1,85 @@
+queue-config = {
+  top-queue = {
+    class = "com.tzld.piaoquan.recommend.server.framework.SimpleMergeQueue"
+    children = {
+      hot-queue = {
+        class = "com.tzld.piaoquan.recommend.server.framework.SimpleMergeQueue"
+        children = {
+          hot-index = {
+            class = "com.tzld.piaoquan.recommend.server.implement.candidate.HotCandidateQueue"
+          }
+        }
+      }
+      explore-queue = {
+        class = "com.tzld.piaoquan.recommend.server.framework.SimpleMergeQueue"
+        children = {
+          category-explore-index = {
+            class = "com.tzld.piaoquan.recommend.server.implement.candidate.HotCandidateQueue"
+          }
+        }
+      }
+      exploit-queue = {
+        class = "com.tzld.piaoquan.recommend.server.framework.SimpleMergeQueue"
+        children = {
+          user-group-index = {
+            class = "com.tzld.piaoquan.recommend.server.implement.candidate.HotCandidateQueue"
+          }
+        }
+      }
+    }
+  }
+}
+
+
+
+rule-config = {
+  // 顶层队列
+  top-queue = {
+    merge-rule = {
+      hot-queue = {
+        recall-percentage = 0.1
+        min-merge-num = 1
+        priority = 1
+      }
+      explore-queue = {
+        recall-percentage = 0.2
+        min-merge-num = 1
+        priority = 5
+      }
+      exploit-queue = {
+        recall-percentage = 0.7
+        min-merge-num = 1
+        max-merge-num = 8
+      }
+    }
+  }
+
+  // 精选队列
+  hot-queue = {
+    merge-rule = {
+      hot-index = {
+        recall-percentage = 0.5
+        min-merge-num = 1
+      }
+    }
+  }
+
+  // 探索队列
+  explore-queue = {
+    merge-rule = {
+      category-explore-index = {
+        recall-percentage = 0.75
+      }
+    }
+  }
+
+  // 兴趣利用队列
+  exploit-queue = {
+    merge-rule = {
+      user-group-index = {
+        recall-percentage = 0.1
+        min-merge-num = 1
+      }
+    }
+  }
+}