1129

recommend-server-service/pom.xml

@@ -28,6 +28,16 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-pool2</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.4</version>
+        </dependency>
+        <dependency>
+            <groupId>com.typesafe</groupId>
+            <artifactId>config</artifactId>
+            <version>1.2.1</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>
@@ -132,6 +142,12 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
+        <dependency>
+            <groupId>cn.jpush.api</groupId>
+            <artifactId>jpush-client</artifactId>
+            <version>3.3.2</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
     <build>
         <finalName>recommend-server-service</finalName>

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/biz/UserBiz.java

@@ -0,0 +1,73 @@
+package com.tzld.piaoquan.recommend.framework.biz;
+
+
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+public class UserBiz  {
+
+    /**
+     * 获取用户数据
+     * @param requestData
+     * @return
+     */
+    public User getUser(final RecommendRequest requestData) {
+        // get uid or mid
+        User user = new User();
+        user.setUid("");
+        // user.setFeedsProfile(getRealTimeUserFeedsProfile(apps, GetType.BATCH, imei));
+        return user;
+    }
+
+    /**
+     * 并行获取 user profile 信息
+     * @param uid
+     *
+     **/
+
+    public User getUserProfileParallel(final String uid ) {
+        User user = new User();
+        return user;
+    }
+
+
+
+
+    /**
+     * 异步的添加 session item data
+     *
+     * @param
+
+     */
+    public void addSessionItemDataAsync() {
+    }
+
+
+    public void addExposeHistoryById() {
+
+    }
+
+    public void addExposeHistory() {
+
+    }
+
+    public void addExposeHistoryById(final String uid, final List<String> itemIds, int keepNum) {
+
+    }
+
+
+    public Set<String> getUserHistory()
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/common/Ordering.java

@@ -0,0 +1,17 @@
+package com.tzld.piaoquan.recommend.framework.common;
+
+public enum Ordering {
+
+    publishtime,  // 发布时间
+    updatetime, // 更新时间
+    createtime, // 创建时间
+
+    ctr,
+    quality,
+    position,
+    sim,
+    chisquare,
+    click,
+    play,
+    expectedduration
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/common/item/Item.java

@@ -0,0 +1,64 @@
+package com.tzld.piaoquan.recommend.framework.common.item;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.Map;
+import java.util.Objects;
+
+
+/**
+ * @author sunmingze
+ * @date 2023-11-27
+ * attention 序列中的item信息
+ */
+
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+public class Item {
+    /**
+     * poi/deal 等item 类型
+     */
+    private String itemType;
+    /**
+     * poi/deal 等item 的id
+     */
+    private Long itemId;
+    /**
+     * 用户attention序列中，该item 的行为时间
+     */
+    private Long timeStamp;
+
+    /**
+     * 扩展字段，存放不同触发策略特有的 item 信息
+     */
+    private Map<String, Object> extInfo;
+
+    public Item(String itemType, Long itemId) {
+        this.itemType = itemType;
+        this.itemId = itemId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof Item)) {
+            return false;
+        }
+        Item dtype = (Item) o;
+        return Objects.equals(getItemType(), dtype.getItemType()) &&
+                Objects.equals(getItemId(), dtype.getItemId());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getItemType(), getItemId());
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/common/item/ItemInfo.java

@@ -0,0 +1,26 @@
+package com.tzld.piaoquan.recommend.framework.common.item;
+
+
+import java.util.Map;
+import java.util.ArrayList;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.io.Serializable;
+
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+public class ItemInfo {
+    String id;      //item的id
+    String title;
+    long add_time;
+    Map<String, Double> category;
+    Map<String, Double> keywords;
+    Map<String, Double> extract_words; //提取的关键词
+    ArrayList<String> vectors; // 向量化信息
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/common/item/ItemProvider.java

@@ -0,0 +1,332 @@
+package com.tzld.piaoquan.recommend.framework.common.item;
+
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang.math.RandomUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Generalized item access to Redis.
+ */
+public abstract class ItemProvider<T> {
+    private static final Logger logger = LoggerFactory.getLogger(ItemProvider.class);
+
+    private static final int REDIS_DEFAULT_TTL = 7 * 24 * 60 * 60;
+    private static final long CACHE_TIMEOUT_MS = 15 * 60 * 1000L;
+    private static final long CACHE_MAXIMUMSIZE = 80 * 10000; // default 80w
+
+    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+    private final ExecutorService itemRefreshExecutorService = new FixedThreadPoolHelper(4, "itemRefresh").getThreadPoolExecutor();
+    // 不能为 static
+    private final BlockingQueue<String> asyncRefreshQueue = new LinkedBlockingQueue<String>(100000);
+    private final RedisSmartClient client;
+    private final String prefix;
+    private final LoadingCache<String, CacheEntry<T>> cache;
+    private final long cacheMaximumSize;
+    private int redisTtl = REDIS_DEFAULT_TTL;
+    private long cacheTimeout = CACHE_TIMEOUT_MS;
+    private long refreshDynamicInterval = CACHE_TIMEOUT_MS;
+
+    /**
+     * 构造带有本地 cache 的 item provider, 使用redis作为底层存储,
+     * 本地cache基于 Guava Cache 构建, 配置失效时间与异步刷新时间;
+     *
+     * @param client         redis-cluster client
+     * @param prefix         业务标志
+     * @param cacheTimeOutMs 缓存失效时间
+     */
+    public ItemProvider(RedisSmartClient client,
+                        String prefix,
+                        long cacheTimeOutMs,
+                        long refreshDynamicInterval) {
+        this(client,
+                prefix,
+                cacheTimeOutMs,
+                refreshDynamicInterval,
+                CACHE_MAXIMUMSIZE,
+                REDIS_DEFAULT_TTL,
+                TimeUnit.SECONDS);
+    }
+
+    /**
+     * 构造带有本地 cache 的 item provider, 使用redis作为底层存储,
+     * 本地cache基于 Guava Cache 构建, 配置失效时间与异步刷新时间;
+     *
+     * @param client         redis-cluster client
+     * @param prefix         业务标志
+     * @param cacheTimeOutMs 缓存失效时间
+     * @param redisKeyTtl
+     * @param redisTtlUnit
+     */
+    public ItemProvider(RedisSmartClient client,
+                        String prefix,
+                        long cacheTimeOutMs,
+                        long refreshDynamicInterval,
+                        final long cacheMaximumSize,
+                        long redisKeyTtl,
+                        TimeUnit redisTtlUnit) {
+
+        this.cacheTimeout = cacheTimeOutMs;
+        this.refreshDynamicInterval = refreshDynamicInterval;
+        this.redisTtl = (int) redisTtlUnit.toSeconds(redisKeyTtl);
+        this.cacheMaximumSize = cacheMaximumSize;
+
+        this.client = client;
+        this.prefix = prefix;
+        this.cache = CacheBuilder.newBuilder()
+                .expireAfterWrite((long) (this.cacheTimeout * 1.4), TimeUnit.MILLISECONDS)
+                .maximumSize(cacheMaximumSize)
+                .build(new CacheLoader<String, CacheEntry<T>>() {
+                    @Override
+                    public CacheEntry<T> load(String key) throws Exception {
+                        if (key == null) {
+                            return null;
+                        } else {
+                            return loadItem(key);
+                        }
+                    }
+                });
+
+        // PerfCounter
+        scheduler.scheduleAtFixedRate(new PerfCounterRunnable(), 60, 60, TimeUnit.SECONDS);
+
+        // cache refresh
+        for (int i = 0; i < 4; i++) {
+            itemRefreshExecutorService.execute(new AsyncRefreshCacheRunnable());
+        }
+    }
+
+    public ItemProvider(RedisSmartClient client, String prefix) {
+        this(client, prefix, CACHE_TIMEOUT_MS, CACHE_TIMEOUT_MS);
+    }
+
+    public RedisSmartClient getClient() {
+        return client;
+    }
+
+    protected abstract T deserialize(byte[] bytes);
+
+    protected abstract T reloadDynamicFeatures(T t);
+
+    protected abstract byte[] serialize(T t);
+
+    /**
+     * 初始时定义 expire time = System.currentTimeMillis()+cacheTimeout
+     *
+     * @param id
+     * @return
+     */
+    private CacheEntry<T> loadItem(String id) throws Exception {
+        long start = System.nanoTime();
+        byte[] bytes = client.get(IndexUtils.convertKey(prefix + id));
+        CacheEntry<T> result;
+        if (bytes == null) {
+            result = null;
+        } else {
+            result = makeCacheEntry(id, Optional.of(deserialize(bytes)));
+        }
+        //
+        return result;
+    }
+
+    public long dbSize() {
+        return this.cache.size();
+    }
+
+    private CacheEntry<T> makeCacheEntry(String itemId, Optional<T> t) {
+        long expireTime = System.currentTimeMillis() + cacheTimeout + RandomUtils.nextInt(10 * 60 * 1000);
+        long dynamicExpireTime = System.currentTimeMillis() + refreshDynamicInterval;
+        return new CacheEntry<T>(itemId, expireTime, dynamicExpireTime, t);
+    }
+
+    public long ttl(String id) throws Exception {
+
+        return client.ttl(IndexUtils.convertKey(id));
+    }
+
+    public long ttl(byte[] id) throws Exception {
+
+        return client.ttl(id);
+    }
+
+    public CacheEntry<T> getCacheEntry(String id) throws ExecutionException {
+        return cache.get(id);
+
+    }
+
+    public Optional<T> getDirect(String id) throws Exception {
+        CacheEntry<T> cacheEntry = loadItem(id);
+        return cacheEntry.getItem();
+    }
+
+    public Optional<T> get(String id) throws ExecutionException {
+        CacheEntry<T> result = cache.get(id);
+
+        if (result == null || !result.getItem().isPresent()) {
+            cache.invalidate(id);
+            return null;
+        }
+
+        // minimum expire time, update key expire time
+        if (result.getExpireTime() <= 0) {
+            result = makeCacheEntry(id, result.getItem());
+            cache.put(id, result);
+        }
+
+        // check expire and refresh
+        long currentTime = System.currentTimeMillis();
+        if ((result.getDynamicFeatureExpireTime() > 0 && currentTime > result.getDynamicFeatureExpireTime()) ||
+                (result.getExpireTime() > 0 && currentTime > result.getExpireTime())) {
+            boolean ret = asyncRefreshQueue.offer(id);
+            if (logger.isDebugEnabled()) {
+                logger.debug("offer async queue itemid [{}], result [{}]", id, ret);
+            }
+
+            if (asyncRefreshQueue.size() > CACHE_MAXIMUMSIZE) {
+                logger.warn("item backgroud blocking queue length [{}], it is danger", asyncRefreshQueue.size());
+            }
+
+            if (!ret) {
+                PerfCounter.setCounterCount(PerfCounterUtils.ASYNC_QUEUE_ITEM_OFFER_FAILED_COUNT, 1);
+                logger.error("item backgroud offer blocking queue error, itemid [{}], with queue length [{}]", id, asyncRefreshQueue.size());
+            }
+        }
+
+        return result.getItem();
+    }
+
+    public String setLocalCache(String id, T t) throws ExecutionException {
+        long start = System.nanoTime();
+
+        // put to local cache
+        CacheEntry<T> result = makeCacheEntry(id, Optional.of(t));
+        cache.put(id, result);
+
+        PerfCounter.countDuration(PerfCounterUtils.ITEM_SAVE_DURATION, System.nanoTime() - start);
+        return "";
+    }
+
+    public String set(String id, T t) throws Exception {
+        long start = System.nanoTime();
+
+        // set redis
+        String strKey = prefix + id;
+        String ret = client.setex(IndexUtils.convertKey(strKey), this.redisTtl, serialize(t));
+
+        // put to cache
+        CacheEntry<T> result = makeCacheEntry(id, Optional.of(t));
+        cache.put(id, result);
+
+        PerfCounter.countDuration(PerfCounterUtils.ITEM_SAVE_DURATION, System.nanoTime() - start);
+        return ret;
+    }
+
+    public long remove(String id) throws Exception {
+        long start = System.nanoTime();
+
+        // del redis
+        long affectRow = client.del(IndexUtils.convertKey(prefix + id));
+
+        // del cache
+        cache.invalidate(id);
+        PerfCounter.countDuration(PerfCounterUtils.ITEM_DEL_DURATION, System.nanoTime() - start);
+        return affectRow;
+    }
+
+    // 周期性上报cache的统计信息
+    public class PerfCounterRunnable implements Runnable {
+        @Override
+        public void run() {
+            PerfCounter.setGaugeValue(PerfCounterUtils.ITEM_STATS_SIZE, cache.size());
+            PerfCounter.setGaugeValue(PerfCounterUtils.ITEM_STATS_HIT_RATE, cache.stats().hitRate());
+            PerfCounter.setGaugeValue(PerfCounterUtils.ITEM_STATS_AVG_LOAD_PENALTY, cache.stats().averageLoadPenalty());
+            PerfCounter.setGaugeValue(PerfCounterUtils.ITEM_STATS_EVICTION_COUNT, cache.stats().evictionCount());
+
+            // async queue size
+            PerfCounter.setGaugeValue(PerfCounterUtils.ASYNC_QUEUE_STATS_ITEM_QUEUE_SIZE, asyncRefreshQueue.size());
+        }
+    }
+
+
+    /**
+     * 通过 blocking queue 异步刷新即将到期的item
+     */
+    public class AsyncRefreshCacheRunnable implements Runnable {
+
+        @Override
+        public void run() {
+            while (true) {
+                String itemId = "";
+                try {
+                    itemId = asyncRefreshQueue.take(); // take async item, blocked when no item
+
+                    final long currentTimeMillis = System.currentTimeMillis();
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("async refresh thread get itemid [{}]", itemId); // get itemid
+                    }
+
+                    CacheEntry<T> entry = cache.get(itemId);
+                    // 全量刷新
+                    if (currentTimeMillis > entry.getExpireTime()) {
+                        refreshStaticFeatures(entry);
+                        PerfCounter.setCounterCount(PerfCounterUtils.ITEM_REFRESH_COUNT, 1);
+                    }
+                    // 刷新动态特征
+                    if (currentTimeMillis > entry.getDynamicFeatureExpireTime()) {
+                        refreshDynamicFeatures(entry);
+                        PerfCounter.setCounterCount(PerfCounterUtils.ITEM_DYNAMIC_REFRESH_COUNT, 1);
+                    }
+
+                    long endTime = System.currentTimeMillis();
+                    PerfCounter.setHistogramValue(PerfCounterUtils.ITEM_REFRESH_DURATION, endTime - currentTimeMillis);
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("async refresh items spenttime [{}]", endTime - currentTimeMillis);
+                    }
+                } catch (Exception e) {
+                    logger.error("async refresh item error, itemId [{}], [{}]", itemId, ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+
+        public void refreshStaticFeatures(CacheEntry<T> entry) {
+            try {
+                cache.refresh(entry.getItemId());
+                if (logger.isDebugEnabled()) {
+                    logger.debug("async refresh item [{}]", entry.getItemId());
+                }
+            } catch (Exception e) {
+                PerfCounter.setCounterCount(PerfCounterUtils.ITEM_REFRESH_FAILED_COUNT, 1);
+                logger.error("async refresh item error, itemId [{}], exception [{}]", entry.getItemId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+
+        public void refreshDynamicFeatures(CacheEntry<T> entry) {
+            try {
+                // reload dynamic items and update expireTime
+                entry.setItem(Optional.of(reloadDynamicFeatures(entry.getItem().get())));
+                entry.setDynamicFeatureExpireTime(System.currentTimeMillis() + refreshDynamicInterval);
+                if (logger.isDebugEnabled()) {
+                    logger.debug("async refresh items dynamic features [{}]", entry.getItemId());
+                }
+            } catch (Exception e) {
+
+                logger.error("async refresh item dynamic features error, itemId [{}], exception [{}]",
+                        entry.getItemId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/common/item/RankerItem.java

@@ -0,0 +1,547 @@
+package com.tzld.piaoquan.recommend.framework.common.item;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.CandidateInfo;
+
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+public class RankerItem implements Comparable<RankerItem> {
+    private String id = "";
+    private String originDocId = ""; // 原始docid
+    private double score = 0;   // 最终的score
+    private double l3score = -1;  // 索引中的score值
+    private double recScore;  // 精排ctr模型 打分
+    private double dwelltimeScore;  // 停留时长打分
+    private double completenessScore;
+    private double pCtrBScore;  // 粗排ctr模型 打分
+    private int l3ScoreIndex;
+    private String reason;
+    private String queue;
+    private CandidateInfo candidateInfo;
+    private List<CandidateInfo> candidateInfoList; // 兼容多个召回key命中
+    private Object itemInfo;
+    private Object itemInfoBytes;
+    private Map<String, String> flags;
+    private Map<String, Double> rankerScore;
+    private Map<String, Integer> rankerIndex;
+    private Map<String, String> itemInfoDebug;
+    // item 信息
+    private long exposeCount;
+    private long clickCount;
+    // merger 信息
+    private List<String> mergeQueuePath;
+    private List<String> mergeDecisionLabels;
+    // reason
+    private String reasonByClickItemId = "";
+    private String reasonBySearchTags = "";
+    private String reasonByExploitTags = "";
+    private String reasonByExploreTags = "";
+    // 排序因子
+    private Map<String, Double> rankItemCategories = Maps.newHashMap();
+    private Map<String, Double> rankItemSubCategories = Maps.newHashMap();
+    private Map<String, Double> rankItemUserTags = Maps.newHashMap();
+    private Map<String, Double> rankItemTitleTags = Maps.newHashMap();
+    private String source;
+    // for L3Score
+    private int index;// 无意义
+    // for feature debug
+    private Map<String, String> ctrFeaWeight;
+    private Map<String, String> dwellTimeFeaWeight;
+    private Map<String, String> dnnCtrFeaWeight;
+    private Map<String, String> completenessFeaWeight;
+
+    private List<Long> w2vTopics;
+    private List<Long> LDATopics;
+    private Map<String, Double> rankedTags;
+
+    public RankerItem() {
+        this.rankerIndex = new HashMap<String, Integer>();
+        this.rankerScore = new HashMap<String, Double>();
+        this.itemInfoDebug = new HashMap<String, String>();
+        this.mergeQueuePath = new LinkedList<String>();
+        this.mergeDecisionLabels = new LinkedList<String>();
+        this.ctrFeaWeight = new HashMap<String, String>();
+        this.dwellTimeFeaWeight = new HashMap<String, String>();
+        this.dnnCtrFeaWeight = new HashMap<String, String>();
+        this.candidateInfoList = new ArrayList<CandidateInfo>();
+        this.completenessFeaWeight = new HashMap<String, String>();
+        this.w2vTopics = new ArrayList<>();
+        this.LDATopics = new ArrayList<>();
+        this.rankedTags = new HashMap<>();
+    }
+
+    public RankerItem(RankerItem other) {
+
+        // 涉及到变更的字段
+        this.id = other.getId();
+        this.originDocId = other.getOriginDocId();
+        this.score = other.getScore();
+        this.l3score = other.getL3score();
+        this.recScore = other.getRecScore();
+        this.pCtrBScore = other.getPCtrBScore();
+        this.dwelltimeScore = other.getDwelltimeScore();
+        this.l3ScoreIndex = other.getL3ScoreIndex();
+
+        this.reason = other.getReason();
+        this.queue = other.getQueue();
+
+        // item 信息
+        this.exposeCount = other.exposeCount;
+        this.clickCount = other.clickCount;
+
+        // merger 信息
+        this.mergeQueuePath = Lists.newArrayList();
+        this.mergeDecisionLabels = Lists.newArrayList();
+        if (other.getMergeQueuePath() != null)
+            this.mergeQueuePath.addAll(other.getMergeQueuePath());
+        if (other.getMergeDecisionLabels() != null)
+            this.mergeDecisionLabels.addAll(other.getMergeDecisionLabels());
+
+        this.rankedTags = new HashMap<>();
+        if (other.getRankedTags() != null) {
+            this.rankedTags.putAll(other.getRankedTags());
+        }
+
+        this.reasonByClickItemId = other.getReasonByClickItemId();
+        this.reasonBySearchTags = other.getReasonBySearchTags();
+        this.reasonByExploitTags = other.getReasonByExploitTags();
+        this.reasonByExploreTags = other.getReasonByExploreTags();
+
+        // candidateinfo
+        this.candidateInfo = other.getCandidateInfo() != null ? other.getCandidateInfo().deepcopy() : null;
+        this.candidateInfoList = Lists.newArrayList();
+        if (other.getCandidateInfoList() != null) {
+            for (CandidateInfo tmpCandidateInfo : other.getCandidateInfoList()) {
+                this.candidateInfoList.add(tmpCandidateInfo.deepcopy());
+            }
+        }
+
+        // 只读字段
+        this.itemInfo = other.getItemInfo();
+        this.itemInfoBytes = other.getItemInfoBytes();
+        this.candidateInfoList = other.getCandidateInfoList(); // 兼容多个召回key命中
+
+        this.rankItemCategories = other.getRankItemCategories();
+        this.rankItemSubCategories = other.getRankItemSubCategories();
+        this.rankItemUserTags = other.getRankItemUserTags();
+        this.rankItemTitleTags = other.getRankItemTitleTags();
+        this.source = other.getSource();
+
+        this.ctrFeaWeight = other.getCtrFeaWeight();
+        this.dwellTimeFeaWeight = other.getDwellTimeFeaWeight();
+        this.completenessFeaWeight = other.getCompletenessFeaWeight();
+        this.dnnCtrFeaWeight = other.getDnnCtrFeaWeight();
+
+        this.rankerScore = other.getRankerScore();
+        this.rankerIndex = other.getRankerIndex();
+        this.itemInfoDebug = other.getItemInfoDebug();
+    }
+
+    public double getPCtrBScore() {
+        return pCtrBScore;
+    }
+
+    public void setPCtrBScore(double pCtrBScore) {
+        this.pCtrBScore = pCtrBScore;
+    }
+
+    public RankerItem deepcopy() {
+        return new RankerItem(this);
+    }
+
+    public List<CandidateInfo> getCandidateInfoList() {
+        return candidateInfoList;
+    }
+
+    public void setCandidateInfoList(List<CandidateInfo> candidateInfoList) {
+        this.candidateInfoList = candidateInfoList;
+    }
+
+    public void addToCandidateInfoList(CandidateInfo candidateInfo) {
+        this.candidateInfoList.add(candidateInfo);
+    }
+
+    public void putCtrFeaWeight(String featureName, double weight) {
+        this.ctrFeaWeight.put(featureName, Double.toString(weight));
+    }
+
+    public void putAllCtrFeaWeight(Map<String, Double> featureMap) {
+        for (Map.Entry<String, Double> entry : featureMap.entrySet()) {
+            this.ctrFeaWeight.put(entry.getKey(), Double.toString(entry.getValue()));
+        }
+    }
+
+    public void putDwellTimeFeaWeight(String featureName, double weight) {
+        this.dwellTimeFeaWeight.put(featureName, Double.toString(weight));
+    }
+
+
+    public void putAllDwellTimeFeaWeight(Map<String, Double> featureMap) {
+        for (Map.Entry<String, Double> entry : featureMap.entrySet()) {
+            this.dwellTimeFeaWeight.put(entry.getKey(), Double.toString(entry.getValue()));
+        }
+    }
+
+    public void putCompletenessFeaWeight(String featureName, double weight) {
+        this.completenessFeaWeight.put(featureName, Double.toString(weight));
+    }
+
+    public void putAllCompletenessFeaWeight(Map<String, Double> featureMap) {
+        for (Map.Entry<String, Double> entry : featureMap.entrySet()) {
+            this.completenessFeaWeight.put(entry.getKey(), Double.toString(entry.getValue()));
+        }
+    }
+
+    public void putDnnCtrFeaWeight(Map<String, Map<String, String>> featureMap) {
+        for (Map.Entry<String, Map<String, String>> feaWeight : featureMap.entrySet()) {
+            Map<String, String> feaWeightMap = feaWeight.getValue();
+            this.dnnCtrFeaWeight = feaWeightMap;
+        }
+    }
+
+    public Map<String, String> getCtrFeaWeight() {
+        return ctrFeaWeight;
+    }
+
+    public Map<String, String> getDwellTimeFeaWeight() {
+        return dwellTimeFeaWeight;
+    }
+
+    public Map<String, String> getCompletenessFeaWeight() {
+        return completenessFeaWeight;
+    }
+
+    public Map<String, String> getDnnCtrFeaWeight() {
+        return dnnCtrFeaWeight;
+    }
+
+
+    public Map<String, Double> getRankItemUserTags() {
+        return rankItemUserTags;
+    }
+
+    public void setRankItemUserTags(Map<String, Double> rankItemUserTags) {
+        this.rankItemUserTags = rankItemUserTags;
+    }
+
+    public Map<String, Double> getRankItemTitleTags() {
+        return rankItemTitleTags;
+    }
+
+    public void setRankItemTitleTags(Map<String, Double> rankItemTitleTags) {
+        this.rankItemTitleTags = rankItemTitleTags;
+    }
+
+    public Map<String, Double> getRankItemCategories() {
+        return rankItemCategories;
+    }
+
+    public void setRankItemCategories(Map<String, Double> rankItemCategories) {
+        this.rankItemCategories = rankItemCategories;
+    }
+
+    public Map<String, Double> getRankItemSubCategories() {
+        return rankItemSubCategories;
+    }
+
+    public void setRankItemSubCategories(Map<String, Double> rankItemSubCategories) {
+        this.rankItemSubCategories = rankItemSubCategories;
+    }
+
+    public int getIndex() {
+        return index;
+    }
+
+    public void setIndex(int index) {
+        this.index = index;
+    }
+
+    public double getL3score() {
+        return l3score;
+    }
+
+    public void setL3score(double l3score) {
+        this.l3score = l3score;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public double getScore() {
+        return score;
+    }
+
+    public void setScore(double score) {
+        this.score = score;
+    }
+
+    public double getDwelltimeScore() {
+        return dwelltimeScore;
+    }
+
+    public void setDwelltimeScore(double dwelltimeScore) {
+        this.dwelltimeScore = dwelltimeScore;
+    }
+
+    public double getCompletenessScore() {
+        return completenessScore;
+    }
+
+    public void setCompletenessScore(double completenessScore) {
+        this.completenessScore = completenessScore;
+    }
+
+    public CandidateInfo getCandidateInfo() {
+        return candidateInfo;
+    }
+
+    public void setCandidateInfo(CandidateInfo candidateInfo) {
+        this.candidateInfo = candidateInfo;
+    }
+    
+    public Map<String, String> getFlags() {
+        return flags;
+    }
+
+    public void setFlags(Map<String, String> flags) {
+        this.flags = flags;
+    }
+
+    public String getReason() {
+        return reason;
+    }
+
+    public void setReason(String reason) {
+        this.reason = reason;
+    }
+
+    public String getQueue() {
+        return queue;
+    }
+
+    public void setQueue(String queue) {
+        this.queue = queue;
+    }
+
+    public String getSource() {
+        return source;
+    }
+
+    public void setSource(String source) {
+        this.source = source;
+    }
+
+    public Object getItemInfo() {
+        return itemInfo;
+    }
+
+    public void setItemInfo(Object itemInfo) {
+        this.itemInfo = itemInfo;
+    }
+
+    public Object getItemInfoBytes() {
+        return itemInfoBytes;
+    }
+
+    public void setItemInfoBytes(Object itemInfoBytes) {
+        this.itemInfoBytes = itemInfoBytes;
+    }
+
+    public double getRecScore() {
+        return recScore;
+    }
+
+    public void setRecScore(double recScore) {
+        this.recScore = recScore;
+    }
+
+    public int getL3ScoreIndex() {
+        return l3ScoreIndex;
+    }
+
+    public void setL3ScoreIndex(int l3ScoreIndex) {
+        this.l3ScoreIndex = l3ScoreIndex;
+    }
+
+    public Map<String, Integer> getRankerIndex() {
+        return rankerIndex;
+    }
+
+    public void setRankerIndex(Map<String, Integer> rankerIndex) {
+        this.rankerIndex = rankerIndex;
+    }
+
+    public void putRankerIndex(String rankerName, Integer rankerIndex) {
+        this.rankerIndex.put(rankerName, rankerIndex);
+    }
+
+    public Map<String, Double> getRankerScore() {
+        return rankerScore;
+    }
+
+    public void setRankerScore(Map<String, Double> rankerScore) {
+        this.rankerScore = rankerScore;
+    }
+
+    public void putRankerScore(String rankerName, Double rankerScore) {
+        this.rankerScore.put(rankerName, rankerScore);
+    }
+
+    public Map<String, String> getItemInfoDebug() {
+        return this.itemInfoDebug;
+    }
+
+    public void setItemInfoDebug(Map<String, String> itemInfoDebug) {
+        this.itemInfoDebug = itemInfoDebug;
+    }
+
+    public void addMergeQueuePath(String mergeQueueName) {
+        this.mergeQueuePath.add(mergeQueueName);
+    }
+
+    public List<String> getMergeQueuePath() {
+        return this.mergeQueuePath;
+    }
+
+    public String getLastMergeQueueName() {
+        int pathSize = this.mergeQueuePath.size();
+        if (pathSize > 0) {
+            return this.mergeQueuePath.get(pathSize - 1);
+        } else {
+            return getCandidateInfo().getCandidate_queue();
+        }
+    }
+
+    public void addMergeDecisionLabel(String label) {
+        this.mergeDecisionLabels.add(label);
+    }
+
+    public List<String> getMergeDecisionLabels() {
+        return this.mergeDecisionLabels;
+    }
+
+    public String getReasonByClickItemId() {
+        return reasonByClickItemId;
+    }
+
+    public void setReasonByClickItemId(String reasonByClickItemId) {
+        this.reasonByClickItemId = reasonByClickItemId;
+    }
+
+    public String getReasonBySearchTags() {
+        return reasonBySearchTags;
+    }
+
+    public void setReasonBySearchTags(String reasonBySearchTags) {
+        this.reasonBySearchTags = reasonBySearchTags;
+    }
+
+    public String getReasonByExploitTags() {
+        return reasonByExploitTags;
+    }
+
+    public void setReasonByExploitTags(String reasonByExploitTags) {
+        this.reasonByExploitTags = reasonByExploitTags;
+    }
+
+    public String getReasonByExploreTags() {
+        return reasonByExploreTags;
+    }
+
+    public void setReasonByExploreTags(String reasonByExploreTags) {
+        this.reasonByExploreTags = reasonByExploreTags;
+    }
+
+    public String getOriginDocId() {
+        return originDocId;
+    }
+
+    public void setOriginDocId(String originDocId) {
+        this.originDocId = originDocId;
+    }
+
+    public long getExposeCount() {
+        return exposeCount;
+    }
+
+    public void setExposeCount(long exposeCount) {
+        this.exposeCount = exposeCount;
+    }
+
+    public long getClickCount() {
+        return clickCount;
+    }
+
+    public void setClickCount(long clickCount) {
+        this.clickCount = clickCount;
+    }
+
+
+
+
+    public Map<String, Double> getRankedTags() {
+        return rankedTags;
+    }
+
+    public void setRankedTags(Map<String, Double> rankedTags) {
+        this.rankedTags = rankedTags;
+    }
+
+
+    public void setLDATopics(List<Long> LDATopics) {
+        this.LDATopics = LDATopics;
+    }
+
+    /**
+     * 转化为RecommendItem
+     *
+     * @return
+     */
+
+
+
+    @Override
+    public String toString() {
+        StringBuffer buf = new StringBuffer();
+        buf.append("id: " + id);
+        buf.append(",");
+        buf.append("score: " + score);
+        buf.append(",");
+        buf.append("queue: " + queue);
+        return buf.toString();
+    }
+
+    @Override
+    public int compareTo(RankerItem o) {
+        if (o == null) {
+            return -1;
+        }
+        if (score > o.score) {
+            return -1;
+        } else if (score < o.score) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+
+}
+
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/common/user/User.java

@@ -0,0 +1,26 @@
+package com.tzld.piaoquan.recommend.framework.common.user;
+
+
+
+import com.tzld.piaoquan.recommend.framework.common.user.UserAttention;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+
+/**
+ * @author sunmingze
+ * @date 2019/8/30
+ */
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+public class User {
+
+    private String mid;
+    private String uid;
+    private UserAttention userAttention;
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/common/user/UserAttention.java

@@ -0,0 +1,43 @@
+package com.tzld.piaoquan.recommend.framework.common.user;
+
+
+
+import com.tzld.piaoquan.recommend.framework.common.item.Item;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.List;
+
+/**
+ * @author sunmingze
+ * @date 2023/11/27
+ */
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+public class UserAttention {
+    /*
+     * attention 抽取子策略，得到用户近期行为list
+     */
+
+    /**
+     * 1h 内的数据
+     */
+    private List<Item> hour1ViewItems;
+    private List<Item> hour1ClickItems;
+    private List<Item> hour1RealPlayItems;
+    private List<Item> hour1ShareItems;
+
+    /**
+     * 24h 内的数据
+     */
+    private List<Item> hour24ViewItems;
+    private List<Item> hour24ClickItems;
+    private List<Item> hour24RealPlayItems;
+    private List<Item> hour24ShareItems;
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/common/user/UserInfo.java

@@ -0,0 +1,27 @@
+package com.tzld.piaoquan.recommend.framework.common.user;
+
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+
+/**
+ * @author sunmingze
+ * @date 2019/8/30
+ */
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+public class UserInfo {
+    String id;
+    String ip;
+    String age;
+    String sex;
+    String region;
+    String province;
+    String graph_emb;
+    String graph_cluser;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/merger/IndexCandidateQueue.java

@@ -0,0 +1,76 @@
+package com.tzld.piaoquan.recommend.framework.merger;
+
+import com.tzld.piaoquan.recommend.framework.common.item.RankerItem;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.QueueName;
+
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public abstract class IndexCandidateQueue extends StrategyQueue {
+    protected Map<String, Candidate> myCandidates = new HashMap<String, Candidate>();
+
+    public IndexCandidateQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    public abstract int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId);
+
+    @Override
+    public int doMerge(final Map<String, Pair<MergeRule, List<RankerItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        // TODO: sublist by every queue
+        return MergeUtils.simpleMergeWithProtection(items, rankerItemsListMap, recNum, user, requestData, requestIndex, expId);
+    }
+
+    @Override
+    public int candidate(final Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        myCandidates.clear();
+
+        int n = addCandidateKey(candidateMap, recallNum, user, requestData, requestIndex, expId);
+        // log
+        candidateMap.putAll(myCandidates);
+
+        return n;
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, String key, int num, String queue) {
+        if (candidates.containsKey(key)) {
+            return 0;
+        } else {
+            Candidate candidate = new Candidate();
+            candidate.setCandidateKey(key);
+            candidate.setCandidateNum(num);
+            candidate.setQueue(queue);
+            candidates.put(key, candidate);
+            return num;
+        }
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue) {
+        return this.addCandidateKey(candidates, queueName, num, queue, true, true, true);
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue, boolean needScore, boolean needPornRank) {
+        return this.addCandidateKey(candidates, queueName, num, queue, needScore, needPornRank, true);
+    }
+
+    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue, boolean needScore, boolean needPornRank, boolean needExposeHistoryFilter) {
+        if (candidates.containsKey(queueName.toString())) {
+            return 0;
+        } else {
+            Candidate candidate = new Candidate();
+            candidate.setCandidateKey(queueName.toString());
+            candidate.setCandidateNum(num);
+            candidate.setQueue(queue);
+            candidates.put(queueName.toString(), candidate);
+            return num;
+        }
+    }
+}
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/merger/MergeRule.java

@@ -0,0 +1,42 @@
+package com.tzld.piaoquan.recommend.framework.merger;
+
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+
+public class MergeRule {
+    public String queueName;
+
+    public Set<Integer> enableExpIdSet = null;
+    public Set<Integer> disableExpIdSet = null;
+
+    public double recallPercentage = 0.1; // 召回百分比, 默认占 10%
+    public int minMergeNum = 0; // 合并保护规则: 最小合并条数
+    public int maxMergeNum = 100; // 合并保护规则: 最大合并条数
+
+    private Map<String, String> properties;
+
+    public MergeRule() {
+        this.properties = new HashMap<String, String>();
+    }
+
+    public boolean isDisabled(final int expId) {
+        if (enableExpIdSet != null && !enableExpIdSet.contains(expId)) {
+            return true;
+        }
+        if (disableExpIdSet != null && disableExpIdSet.contains(expId)) {
+            return true;
+        }
+        return false;
+    }
+
+    public void putProperty(String key, String value) {
+        this.properties.put(key, value);
+    }
+
+    public Map<String, String> getProperties() {
+        return properties;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/merger/MergeUtils.java

@@ -0,0 +1,319 @@
+package com.tzld.piaoquan.recommend.framework.merger;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+import com.typesafe.config.Config;
+import com.tzld.piaoquan.recommend.framework.utils.Configuration;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.CandidateInfo;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.common.item.RankerItem;
+
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+
+
+public class MergeUtils {
+    public static final Logger LOGGER = LoggerFactory.getLogger(MergeUtils.class);
+
+
+    public static StrategyQueue createTopQueue(String mergeConfFile, Config configDefault, String topQueueName) {
+        StrategyQueueConfig strategyQueueConfig = new StrategyQueueConfig();
+        Config mergeQueueConf = Configuration.getConfigByFileName(mergeConfFile, configDefault);
+
+        if (strategyQueueConfig.load(mergeQueueConf)) {
+            LOGGER.debug("Merger config init succ");
+        } else {
+            LOGGER.error("Merge config init failed: init MergeBiz using queue config {}", mergeConfFile);
+
+            if (!strategyQueueConfig.load(configDefault)) {
+                LOGGER.error("Merge config init using static queue config failed: {}", configDefault);
+            }
+        }
+
+        StrategyQueue topQueue = null;
+        try {
+            topQueue = MergeUtils.constructStrategyQueue(topQueueName, strategyQueueConfig);
+        } catch (InstantiationException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (IllegalAccessException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (ClassNotFoundException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (NoSuchMethodException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        } catch (InvocationTargetException e) {
+            LOGGER.error("construct top StrategyQueue: [{}]", e);
+        }
+        return topQueue;
+    }
+
+    public static StrategyQueue constructStrategyQueue(String queueName, StrategyQueueConfig strategyQueueConfig) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
+        if (!strategyQueueConfig.getStrategyQueueInfoMap().containsKey(queueName)) {
+            return null;
+        }
+
+        StrategyQueueInfo strategyQueueInfo = strategyQueueConfig.getStrategyQueueInfoMap().get(queueName);
+        StrategyQueue strategyQueue = (StrategyQueue) Class.forName(strategyQueueInfo.getQueueClass())
+                .getConstructor(StrategyQueueInfo.class, strategyQueueConfig.getClass())
+                .newInstance(strategyQueueInfo, strategyQueueConfig);
+
+        return strategyQueue;
+    }
+
+    /**
+     * 分发items到策略树中
+     * 分发的过程中不要破坏相对顺序
+     *
+     * @param strategyQueue
+     * @param items
+     */
+    public static void distributeItemsToMultiQueues(StrategyQueue strategyQueue, List<RankerItem> items) {
+        List<StrategyQueue> strategyQueueList = strategyQueue.getAllQueues();
+
+        // TODO: perf tunning
+        Multimap<String, RankerItem> mergeQueuesItems = ArrayListMultimap.create();
+        for (RankerItem item : items) {
+            for (CandidateInfo candidateInfo : item.getCandidateInfoList()) {
+                String mergeQueue = candidateInfo.getCandidate().getQueue();
+                RankerItem currentItem = item.deepcopy();
+
+                // set candidate info
+                currentItem.setQueue(mergeQueue);
+                currentItem.setCandidateInfo(candidateInfo);
+
+                mergeQueuesItems.put(mergeQueue, currentItem);
+            }
+        }
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (Map.Entry<String, RankerItem> entry : mergeQueuesItems.entries()) {
+                LOGGER.debug("after distribute item queue info: [{}] [{}] [{}]",
+                        new Object[]{entry.getKey(), entry.getValue().getId(), entry.getValue().getQueue()});
+            }
+        }
+
+        for (StrategyQueue queue : strategyQueueList) {
+            String mergeQueueName = queue.getStrategyQueueInfo().getQueueName();
+            if (mergeQueuesItems.containsKey(mergeQueueName)) {
+                List<RankerItem> currMergeQueueItems = new ArrayList<RankerItem>(mergeQueuesItems.get(mergeQueueName));
+                queue.setItems(currMergeQueueItems);
+            }
+        }
+    }
+
+    /**
+     * 基于 score 字段融合: score 较大的 Item 优先
+     *
+     * @param resultRankerItems
+     * @param rankerItemsListMap
+     * @param freeRecNum
+     * @param expId
+     */
+    public static void simpleMergeByScore(List<RankerItem> resultRankerItems, final Map<String, Pair<MergeRule, List<RankerItem>>> rankerItemsListMap, int freeRecNum, int expId) {
+        PriorityQueue<Pair<String, Integer>> mergePriorityQueue = new PriorityQueue<Pair<String, Integer>>(freeRecNum, new Comparator<Pair<String, Integer>>() {
+            @Override
+            public int compare(Pair<String, Integer> o1, Pair<String, Integer> o2) {
+                return rankerItemsListMap.get(o1.getLeft()).getRight().get(o1.getRight()).compareTo(rankerItemsListMap.get(o2.getLeft()).getRight().get(o2.getRight()));
+            }
+        });
+        for (Pair<MergeRule, List<RankerItem>> entry : rankerItemsListMap.values()) {
+            if (entry.getLeft().isDisabled(expId) || entry.getRight() == null || entry.getRight().isEmpty()) {
+                continue;
+            }
+
+            MergeRule myRule = entry.getLeft();
+            if (myRule.minMergeNum < myRule.maxMergeNum && entry.getRight().size() > myRule.minMergeNum) {
+                Collections.sort(entry.getRight().subList(myRule.minMergeNum, entry.getRight().size()));
+                mergePriorityQueue.add(Pair.of(myRule.queueName, myRule.minMergeNum));
+            }
+        }
+
+        while (freeRecNum > 0 && !mergePriorityQueue.isEmpty()) {
+            Pair<String, Integer> item = mergePriorityQueue.poll();
+            String myName = item.getLeft();
+            int myIndex = item.getRight();
+
+            resultRankerItems.add(rankerItemsListMap.get(myName).getRight().get(myIndex));
+            resultRankerItems.get(resultRankerItems.size() - 1).addMergeDecisionLabel("score:" + myName);
+            freeRecNum -= 1;
+            myIndex += 1;
+
+            if (freeRecNum > 0 && rankerItemsListMap.get(myName).getRight().size() > myIndex && myIndex < rankerItemsListMap.get(myName).getLeft().maxMergeNum) {
+                mergePriorityQueue.add(Pair.of(myName, myIndex));
+            }
+        }
+    }
+
+    /**
+     * 队列融合:
+     * 1. 各队列基于排序不变;
+     * 2. 各队列保证最小合并数量 (minMergeNum) 条目;
+     * 3. 各队列限制最大合并数量 (maxMergeNum) 条目;
+     * 4. 从各队列取 Score 最高条目.
+     *
+     * @param rankerItemsListMap
+     * @param recNum
+     * @param user
+     * @param requestData
+     * @param requestIndex
+     * @param expId
+     * @return
+     */
+    public static int simpleMergeWithProtection(List<RankerItem> resultRankerItems, final Map<String, Pair<MergeRule, List<RankerItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        if (resultRankerItems == null) {
+            resultRankerItems = new LinkedList<RankerItem>();
+        }
+        // 保证各队列最小合并条目后剩余可自由调配条目数量
+        int freeRecNum = recNum;
+        for (Pair<MergeRule, List<RankerItem>> entry : rankerItemsListMap.values()) {
+
+            if (entry.getLeft().isDisabled(expId) || entry.getRight() == null || entry.getRight().isEmpty()) {
+                continue;
+            }
+
+            MergeRule myRule = entry.getLeft();
+            int mySize = entry.getRight().size();
+            int myMinMergeNum = Math.min(myRule.minMergeNum, mySize);
+            freeRecNum -= myMinMergeNum;
+
+            int prevSize = resultRankerItems.size();
+            resultRankerItems.addAll(entry.getRight().subList(0, myMinMergeNum));
+            int afterSize = resultRankerItems.size();
+            for (int ix = prevSize; ix < afterSize; ++ix) {
+                resultRankerItems.get(ix).addMergeDecisionLabel("min_protect:" + myRule.queueName);
+            }
+        }
+
+        if (freeRecNum > 0) {
+            simpleMergeByScore(resultRankerItems, rankerItemsListMap, freeRecNum, expId);
+        }
+
+        return resultRankerItems.size();
+    }
+
+    /**
+     * 按优先级队列融合:
+     * 1. 各队列基于 priority 依次融合: 各队列保证最小合并数量 (minMergeNum) 条目;
+     * 2. 若没有凑足 recNum, 各队列限制最大合并数量 (maxMergeNum) 条目条件下按 score 优先选择;
+     *
+     * @param rankerItemsListMap
+     * @param recNum
+     * @param user
+     * @param requestData
+     * @param requestIndex
+     * @param expId
+     * @return
+     */
+    public static int simpleMergeByPriority(List<RankerItem> resultRankerItems, final Map<String, Pair<MergeRule, List<RankerItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        if (resultRankerItems == null) {
+            resultRankerItems = new LinkedList<RankerItem>();
+        }
+
+        final List<Map.Entry<String, Pair<MergeRule, List<RankerItem>>>> rankerItemsList = new LinkedList<Map.Entry<String, Pair<MergeRule, List<RankerItem>>>>(rankerItemsListMap.entrySet());
+
+        // 保证各队列最小合并条目后剩余可自由调配条目数量
+        Collections.sort(rankerItemsList, new Comparator<Map.Entry<String, Pair<MergeRule, List<RankerItem>>>>() {
+            private final int DEFAULT_PRIORITY = 0;
+
+            @Override
+            public int compare(Map.Entry<String, Pair<MergeRule, List<RankerItem>>> o1, Map.Entry<String, Pair<MergeRule, List<RankerItem>>> o2) {
+                int p1 = getPriority(o1.getValue().getLeft()), p2 = getPriority(o2.getValue().getLeft());
+                return p1 - p2;
+            }
+
+            private int getPriority(MergeRule rule) {
+                if (rule.getProperties().containsKey("priority")) {
+                    return Integer.valueOf(rule.getProperties().get("priority"));
+                }
+                return DEFAULT_PRIORITY;
+            }
+        });
+
+        int freeRecNum = recNum;
+        for (int i = 0; i < rankerItemsList.size(); ++i) {
+            if (rankerItemsList.get(i).getValue().getLeft().isDisabled(expId) || rankerItemsList.get(i).getValue().getRight() == null || rankerItemsList.get(i).getValue().getRight().isEmpty()) {
+                continue;
+            }
+
+            MergeRule myRule = rankerItemsList.get(i).getValue().getLeft();
+            int mySize = rankerItemsList.get(i).getValue().getRight().size();
+            int myMinMergeNum = Math.min(myRule.minMergeNum, mySize);
+            freeRecNum -= myMinMergeNum;
+
+            int prevSize = resultRankerItems.size();
+            resultRankerItems.addAll(rankerItemsList.get(i).getValue().getRight().subList(0, myMinMergeNum));
+            if (mySize > myMinMergeNum) {
+                Collections.sort(rankerItemsList.get(i).getValue().getRight().subList(myMinMergeNum, mySize));
+            }
+            int afterSize = resultRankerItems.size();
+            for (int ix = prevSize; ix < afterSize; ++ix) {
+                resultRankerItems.get(ix).addMergeDecisionLabel("min_protect:" + myRule.queueName);
+            }
+        }
+
+        if (freeRecNum > 0) {
+            simpleMergeByScore(resultRankerItems, rankerItemsListMap, freeRecNum, expId);
+        }
+
+        return resultRankerItems.size();
+    }
+
+    /**
+     * 基于 tag 和 category 重排序, 保证具有相同的分类或标签的 Item 在 windowSize 窗口内只有 similarLimit 条
+     *
+     * @param items
+     * @param expectedRecommendItemsCount
+     * @param windowSize
+     * @param similarLimit
+     */
+    public static void diversityRerank(List<RankerItem> items, Function<Pair<RankerItem, RankerItem>, Boolean> similarFunc, int expectedRecommendItemsCount, final int windowSize, final int similarLimit) {
+        List<RankerItem> rerankedItems = new LinkedList<RankerItem>();
+        expectedRecommendItemsCount = Math.min(expectedRecommendItemsCount, items.size());
+        for (int ix = 0; ix < expectedRecommendItemsCount; ++ix) {
+            int windowStartPosition = Math.max(ix - windowSize + 1, 0);
+            int nextId = 0;
+            while (nextId < items.size()) {
+                int similarCount = 0;
+                for (int jx = windowStartPosition; jx < ix && similarCount < similarLimit; ++jx) {
+                    if (similarFunc.apply(Pair.of(items.get(nextId), rerankedItems.get(jx)))) {
+                        similarCount += 1;
+                    }
+                }
+                if (similarCount < similarLimit) {
+                    break;
+                }
+                nextId += 1;
+            }
+            if (nextId >= items.size()) {
+                nextId = 0;
+            }
+
+            RankerItem nextItem = items.get(nextId);
+            nextItem.addMergeDecisionLabel("sim_rerank:" + windowStartPosition + "_" + windowSize);
+            items.remove(nextId);
+            rerankedItems.add(nextItem);
+        }
+
+        if (items.size() > 0) {
+            rerankedItems.addAll(items);
+        }
+
+        items.clear();
+        items.addAll(rerankedItems);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/merger/SimilarityUtils.java

@@ -0,0 +1,115 @@
+package com.tzld.piaoquan.recommend.framework.merger;
+
+
+import com.google.common.collect.Sets;
+import com.google.common.base.Function;
+import com.tzld.piaoquan.recommend.framework.common.item.RankerItem;
+import org.apache.commons.lang3.tuple.Pair;
+
+import javax.annotation.Nullable;
+
+public class SimilarityUtils {
+
+    public static Function<Pair<RankerItem, RankerItem>, Boolean> getIsSameUserTagOrCategoryFunc() {
+        return new Function<Pair<RankerItem, RankerItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankerItem, RankerItem> itemPair) {
+                return isSameCategory(itemPair.getLeft(), itemPair.getRight()) || isSameUserTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankerItem, RankerItem>, Boolean> getIsSameTitleTagFunc() {
+        return new Function<Pair<RankerItem, RankerItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankerItem, RankerItem> itemPair) {
+                return isSameTitleTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankerItem, RankerItem>, Boolean> getIsSameUserTagFunc() {
+        return new Function<Pair<RankerItem, RankerItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankerItem, RankerItem> itemPair) {
+                return isSameUserTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankerItem, RankerItem>, Boolean> getIsSameCategoryFunc() {
+        return new Function<Pair<RankerItem, RankerItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankerItem, RankerItem> itemPair) {
+                return isSameCategory(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static Function<Pair<RankerItem, RankerItem>, Boolean> getIsUserTagFunc() {
+        return new Function<Pair<RankerItem, RankerItem>, Boolean>() {
+            @Nullable
+            @Override
+            public Boolean apply(@Nullable Pair<RankerItem, RankerItem> itemPair) {
+                return isSameUserTag(itemPair.getLeft(), itemPair.getRight());
+            }
+        };
+    }
+
+    public static boolean isSameCategory(RankerItem rankerItem1, RankerItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getRankItemCategories() == null ||
+                rankerItem2.getRankItemCategories() == null) {
+            return false;
+        }
+
+        return !Sets.intersection(rankerItem1.getRankItemCategories().keySet(),
+                        rankerItem2.getRankItemCategories().keySet())
+                .isEmpty();
+    }
+
+    public static boolean isSameTitleTag(RankerItem rankerItem1, RankerItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getRankItemTitleTags() == null ||
+                rankerItem2.getRankItemTitleTags() == null) {
+            return false;
+        }
+        return !Sets.intersection(rankerItem1.getRankItemTitleTags().keySet(),
+                        rankerItem2.getRankItemTitleTags().keySet())
+                .isEmpty();
+    }
+
+    public static boolean isSameUserTag(RankerItem rankerItem1, RankerItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getRankItemUserTags() == null ||
+                rankerItem2.getRankItemUserTags() == null) {
+            return false;
+        }
+        return !Sets.intersection(rankerItem1.getRankItemUserTags().keySet(),
+                        rankerItem2.getRankItemUserTags().keySet())
+                .isEmpty();
+    }
+
+    public static boolean isSameQueue(RankerItem rankerItem1, RankerItem rankerItem2) {
+        if (rankerItem1 == null || rankerItem2 == null ||
+                rankerItem1.getQueue() == null) {
+            return false;
+        }
+        return rankerItem1.getQueue().equals(rankerItem2.getQueue());
+    }
+
+    public static boolean isSameSubCategory(RankerItem item1, RankerItem item2) {
+        if (item1 == null || item2 == null || item1.getRankItemSubCategories() == null
+                || item2.getRankItemSubCategories() == null) {
+            return false;
+        }
+
+        return !Sets.intersection(item1.getRankItemSubCategories().keySet(),
+                        item2.getRankItemSubCategories().keySet())
+                .isEmpty();
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/merger/SimpleMergeQueue.java

@@ -0,0 +1,29 @@
+package com.tzld.piaoquan.recommend.framework.merger;
+
+
+
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.CandidateInfo;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.common.item.RankerItem;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.*;
+
+
+public class SimpleMergeQueue extends StrategyQueue {
+    public SimpleMergeQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int doMerge(final Map<String, Pair<MergeRule, List<RankerItem>>> rankerItemsListMap, final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        return MergeUtils.simpleMergeWithProtection(items, rankerItemsListMap, recNum, user, requestData, requestIndex, expId);
+    }
+
+    @Override
+    public int candidate(Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        return simpleCandidateByPercentage(candidateMap, recallNum, user, requestData, requestIndex, expId);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/merger/StrategyQueue.java

@@ -0,0 +1,205 @@
+package com.tzld.piaoquan.recommend.framework.merger;
+
+import com.tzld.piaoquan.recommend.framework.common.item.RankerItem;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+public abstract class StrategyQueue {
+    protected final Map<String, Pair<MergeRule, List<RankerItem>>> rankerItemsList = new HashMap<String, Pair<MergeRule, List<RankerItem>>>();
+    private final Logger LOGGER = LoggerFactory.getLogger(StrategyQueue.class);
+    protected Map<String, StrategyQueue> children;
+
+    protected List<RankerItem> items;
+    private StrategyQueueInfo strategyQueueInfo;
+    private List<StrategyQueue> allQueues = null;
+
+    public StrategyQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        this.strategyQueueInfo = strategyQueueInfo;
+
+        items = new LinkedList<RankerItem>();
+        children = new HashMap<String, StrategyQueue>();
+
+        for (String childName : strategyQueueInfo.getChildren()) {
+            if (!strategyQueueConfig.getStrategyQueueInfoMap().containsKey(childName)) {
+                LOGGER.error("No config for Child queue [{}]", childName);
+                continue;
+            }
+            StrategyQueueInfo childQueueInfo = strategyQueueConfig.getStrategyQueueInfoMap().get(childName);
+
+            StrategyQueue childQueue = null;
+            String childQueueClass = childQueueInfo.getQueueClass();
+            try {
+                childQueue = (StrategyQueue) Class.forName(childQueueClass).getConstructor(childQueueInfo.getClass(), strategyQueueConfig.getClass()).newInstance(childQueueInfo, strategyQueueConfig);
+            } catch (InstantiationException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (IllegalAccessException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (ClassNotFoundException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (NoSuchMethodException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            } catch (InvocationTargetException e) {
+                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
+            }
+            if (childQueue != null) {
+                putChild(childName, childQueue);
+            } else {
+                LOGGER.error("construct child queue [{}] failed", childName);
+            }
+        }
+
+//        getAllQueues();
+    }
+
+    public StrategyQueueInfo getStrategyQueueInfo() {
+        return strategyQueueInfo;
+    }
+
+    public List<StrategyQueue> getAllQueues() {
+        if (allQueues == null) {
+            allQueues = new LinkedList<StrategyQueue>();
+
+            if (!strategyQueueInfo.isLeaf()) {
+                for (Map.Entry<String, StrategyQueue> child : children.entrySet()) {
+                    allQueues.addAll(child.getValue().getAllQueues());
+                }
+            }
+
+            allQueues.add(this);
+        }
+        return allQueues;
+    }
+
+    private void putChild(String queueName, StrategyQueue child) {
+        children.put(queueName, child);
+    }
+
+    public List<RankerItem> getItems() {
+        return items;
+    }
+
+    public void setItems(List<RankerItem> items) {
+        this.items = items;
+    }
+
+    public void clearItems() {
+        items.clear();
+    }
+
+    public void addItem(RankerItem item) {
+        this.items.add(item);
+    }
+
+    public abstract int doMerge(final Map<String, Pair<MergeRule, List<RankerItem>>> rankerItemsListMap, final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId);
+
+    public abstract int candidate(Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId);
+
+    public final int merge(final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+        this.beforeMerge(recNum, user, requestData, requestIndex, expId);
+        this.doMerge(rankerItemsList, recNum, user, requestData, requestIndex, expId);
+        this.doDeDup();
+        return this.afterMerge();
+    }
+
+    public void beforeMerge(final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
+
+        if (strategyQueueInfo.isLeaf()) {
+            LOGGER.debug("leaf queue [{}] with [{}] items to merge", getStrategyQueueInfo().getQueueName(), items.size());
+        } else {
+            rankerItemsList.clear();
+
+            // XXX: what if a queue recall items by itself?
+            clearItems();
+
+            int queueCount = 0, itemCount = 0;
+            for (MergeRule rule : strategyQueueInfo.getRulesList()) {
+                if (rule.isDisabled(expId)) {
+                    continue;
+                }
+
+                int myRecNum = Math.min(recNum, rule.maxMergeNum);
+                int actualRecNum = children.get(rule.queueName).merge(myRecNum, user, requestData, requestIndex, expId);
+
+                queueCount += 1;
+                itemCount += actualRecNum;
+
+                rankerItemsList.put(rule.queueName, Pair.of(rule, children.get(rule.queueName).getItems()));
+            }
+
+            LOGGER.debug(getStrategyQueueInfo().getQueueName() + " merge info: [{}] queues with [{}] items to merge", queueCount, itemCount);
+        }
+    }
+
+    /**
+     * id dedup
+     *
+     * @return
+     */
+    public final int doDeDup() {
+        List<RankerItem> pureItems = new ArrayList<RankerItem>();
+        Set<String> deDupItems = new HashSet<String>();
+        int deDupCount = 0;
+        for (RankerItem item : items) {
+            if (deDupItems.contains(item.getId())) {
+                if (LOGGER.isDebugEnabled()) {
+                    LOGGER.debug("merge queue dedup item, queue [{}] itemid [{}]",
+                            strategyQueueInfo.getQueueName(), item.getId());
+                }
+                deDupCount++;
+                continue;
+            }
+
+            deDupItems.add(item.getId());
+            pureItems.add(item);
+        }
+        items = pureItems;
+        LOGGER.debug("dedup items, queue [{}], dedup number [{}], current number [{}]",
+                new Object[]{getStrategyQueueInfo().getQueueName(), deDupCount, items.size()});
+        return items.size();
+    }
+
+    public int afterMerge() {
+        int n = items.size();
+        for (RankerItem item : items) {
+            item.addMergeQueuePath(getStrategyQueueInfo().getQueueName());
+        }
+        LOGGER.debug("number of items after [{}] merge: [{}]", getStrategyQueueInfo().getQueueName(), n);
+        return n;
+    }
+
+    protected int simpleCandidateByPercentage(Map<String, Candidate> candidateMap, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int n = 0;
+
+        for (MergeRule rule : strategyQueueInfo.getRulesList()) {
+            if (rule.isDisabled(expId)) {
+                continue;
+            }
+
+            int myRecallNum = (int) (recallNum * rule.recallPercentage);
+            if (!children.containsKey(rule.queueName)) {
+                LOGGER.error("Rule for not exist child queue [{}]", rule.queueName);
+            }
+
+            Map<String, Candidate> myCandidates = new HashMap<String, Candidate>();
+            n += children.get(rule.queueName).candidate(myCandidates, myRecallNum, user, requestData, requestIndex, expId);
+            candidateMap.putAll(myCandidates);
+        }
+
+        return n;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/merger/StrategyQueueConfig.java

@@ -0,0 +1,235 @@
+package com.tzld.piaoquan.recommend.framework.merger;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.*;
+
+/**
+ * typesafe config for merger tree
+ */
+public class StrategyQueueConfig {
+    private final Logger LOGGER = LoggerFactory.getLogger(StrategyQueueConfig.class);
+
+    private static final String CHILDREN_STRING = "children";
+    private static final String STRATEGY_QUEUE_CLASS_CONF_STRING = "class";
+
+    public static final String DEFAULT_STRATEGY_CLASS_NAME = "com.xiaomi.data.recommend.merger.SimpleMergeQueue";
+
+    private static final String MERGE_RULE_STRING = "merge-rule";
+    private static final String QUEUE_NAME_STRING = "queue-name";
+    private static final String RECALL_PERCENTAGE_STRING = "recall-percentage";
+    private static final String ENABLE_EXPERIMENT_LIST_STRING = "enable-exp";
+    private static final String DISABLE_EXPERIMENT_LIST_STRING = "disable-exp";
+    private static final String MIN_MERGE_NUM_STRING = "min-merge-num";
+    private static final String MAX_MERGE_NUM_STRING = "max-merge-num";
+
+    private static final Set<String> RESERVED_PROPERTIES = new HashSet<String>(Arrays.asList(QUEUE_NAME_STRING, RECALL_PERCENTAGE_STRING, ENABLE_EXPERIMENT_LIST_STRING, DISABLE_EXPERIMENT_LIST_STRING, MIN_MERGE_NUM_STRING, MAX_MERGE_NUM_STRING));
+
+    Map<String, StrategyQueueInfo> strategyQueueInfoMap;
+
+    public Map<String, StrategyQueueInfo> getStrategyQueueInfoMap() {
+        return strategyQueueInfoMap;
+    }
+
+    public boolean load(Config config) {
+        config.checkValid(config, "queue-config");
+        Config queueConf = config.getConfig("queue-config");
+        if (loadMergeQueues(queueConf)) {
+            LOGGER.debug("Load queues config success");
+        } else {
+            LOGGER.error("Load queues config failed");
+            return false;
+        }
+
+        config.checkValid(config, "rule-config");
+        Config rulesConf = config.getConfig("rule-config");
+        if (loadMergeRules(rulesConf)) {
+            LOGGER.debug("Load rules config success");
+        } else {
+            LOGGER.error("Load rules config failed");
+            return false;
+        }
+        // TODO: validate queue structure & rules
+        for (String queueName : strategyQueueInfoMap.keySet()) {
+            StrategyQueue queue = null;
+            try {
+                queue = MergeUtils.constructStrategyQueue(queueName, this);
+            } catch (ClassNotFoundException e) {
+                e.printStackTrace();
+            } catch (NoSuchMethodException e) {
+                e.printStackTrace();
+            } catch (IllegalAccessException e) {
+                e.printStackTrace();
+            } catch (InvocationTargetException e) {
+                e.printStackTrace();
+            } catch (InstantiationException e) {
+                e.printStackTrace();
+            }
+
+            if (queue == null) {
+                LOGGER.error("Queue config info & rules error");
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    public boolean load(String configFile) {
+        Config queueConfig = ConfigFactory.parseResources(configFile);
+        return load(queueConfig);
+    }
+
+    private boolean loadMergeQueues(Config queueConf) {
+        strategyQueueInfoMap = new HashMap<String, StrategyQueueInfo>();
+
+        ConfigObject confObj = queueConf.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            String myName = it.getKey();
+            Config myConf = ((ConfigObject) it.getValue()).toConfig();
+            if (loadMergeQueueInfo(myName, myConf)) {
+                LOGGER.debug("Load queue [{}] info success", myName);
+            } else {
+                LOGGER.error("Load queue [{}] info failed", myName);
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean loadMergeQueueInfo(String name, Config conf) {
+        LOGGER.debug("Start load queue [{}]", name);
+        if (!strategyQueueInfoMap.containsKey(name)) {
+            strategyQueueInfoMap.put(name, new StrategyQueueInfo(name));
+        }
+        StrategyQueueInfo mergeQueueInfo = strategyQueueInfoMap.get(name);
+
+        if (conf.hasPath(STRATEGY_QUEUE_CLASS_CONF_STRING)) {
+            mergeQueueInfo.setQueueClass(conf.getString(STRATEGY_QUEUE_CLASS_CONF_STRING));
+        } else {
+            LOGGER.info("No Class config was found for queue [{}], use default class [{}]", name, DEFAULT_STRATEGY_CLASS_NAME);
+            mergeQueueInfo.setQueueClass(DEFAULT_STRATEGY_CLASS_NAME);
+        }
+
+        if (conf.hasPath(CHILDREN_STRING)) {
+            ConfigObject childrenConf = conf.getObject(CHILDREN_STRING);
+            for (ConfigObject.Entry<String, ConfigValue> it : childrenConf.entrySet()) {
+                String key = it.getKey();
+                Config childConf = ((ConfigObject) it.getValue()).toConfig();
+                String childName = key;
+
+                boolean isLoadSuccess = loadMergeQueueInfo(childName, childConf);
+                if (isLoadSuccess) {
+                    LOGGER.debug("Load queue [{}] SUCCESS", childName);
+                } else {
+                    LOGGER.error("Load queue [{}] FAILED: [{}]", childName, it.getValue());
+                    return false;
+                }
+
+                mergeQueueInfo.addChild(childName);
+            }
+        } else {
+            mergeQueueInfo.setLeaf(true);
+        }
+
+        LOGGER.debug("Load queue [{}] success", name);
+        return true;
+    }
+
+    private boolean loadMergeRules(Config conf) {
+        ConfigObject confObj = conf.root();
+
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            String myName = it.getKey();
+            Config subTreeConf = ((ConfigObject) it.getValue()).toConfig();
+            boolean isParseSuccess = parseMergeRule(myName, subTreeConf);
+            if (isParseSuccess) {
+                LOGGER.debug("Parse rules for queue [{}] SUCCESS", myName);
+            } else {
+                LOGGER.error("Parse rules for queue [{}] FAILED: [{}]", myName, it.getValue());
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private boolean parseMergeRule(String name, Config ruleConf) {
+        if (!strategyQueueInfoMap.containsKey(name)) {
+            LOGGER.error("skip rules for not exist queue [{}]", name);
+            return true;
+        } else {
+            LOGGER.debug("parse rules for queue [{}]", name);
+        }
+
+        if (!ruleConf.hasPath(MERGE_RULE_STRING)) {
+            LOGGER.debug("no merge rules for queue [{}]", name);
+            return true;
+        }
+
+        ConfigObject ruleConfObj = ruleConf.getObject(MERGE_RULE_STRING);
+
+        for (ConfigObject.Entry<String, ConfigValue> it : ruleConfObj.entrySet()) {
+            String ruleName = it.getKey();
+            MergeRule rule = new MergeRule();
+
+            Config myConf = ((ConfigObject) it.getValue()).toConfig();
+
+            rule.recallPercentage = getOrDefault(myConf, RECALL_PERCENTAGE_STRING, 0.1);
+            rule.minMergeNum = getOrDefault(myConf, MIN_MERGE_NUM_STRING, 0);
+            rule.maxMergeNum = getOrDefault(myConf, MAX_MERGE_NUM_STRING, 500);
+
+            // 子队列名规则: 用 queue-name 显式指定, 默认为对应规则名
+            rule.queueName = getOrDefault(myConf, QUEUE_NAME_STRING, ruleName);
+            if (myConf.hasPath(ENABLE_EXPERIMENT_LIST_STRING)) {
+                List<Integer> expIds = myConf.getIntList(ENABLE_EXPERIMENT_LIST_STRING);
+                rule.enableExpIdSet = new HashSet<Integer>(expIds);
+            }
+            if (myConf.hasPath(DISABLE_EXPERIMENT_LIST_STRING)) {
+                List<Integer> expIds = myConf.getIntList(DISABLE_EXPERIMENT_LIST_STRING);
+                rule.disableExpIdSet = new HashSet<Integer>(expIds);
+            }
+
+            for (ConfigObject.Entry<String, ConfigValue> properties : myConf.entrySet()) {
+                if (!RESERVED_PROPERTIES.contains(properties.getKey())) {
+                    rule.putProperty(properties.getKey(), myConf.getString(properties.getKey()));
+                }
+            }
+
+            if (strategyQueueInfoMap.get(name).getChildren().contains(rule.queueName)) { // TODO: perf: duplicate get
+                strategyQueueInfoMap.get(name).addRule(rule);
+            } else {
+                LOGGER.info("Skip rule={} for not exist queue={}", ruleName, rule.queueName);
+            }
+        }
+        return true;
+    }
+
+    private int getOrDefault(Config conf, String path, int defaultValue) {
+        if (conf.hasPath(path)) {
+            return conf.getInt(path);
+        }
+        return defaultValue;
+    }
+
+    private String getOrDefault(Config conf, String path, String defaultValue) {
+        if (conf.hasPath(path)) {
+            return conf.getString(path);
+        }
+        return defaultValue;
+    }
+
+    private double getOrDefault(Config conf, String path, double defaultValue) {
+        if (conf.hasPath(path)) {
+            return conf.getDouble(path);
+        }
+        return defaultValue;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/merger/StrategyQueueInfo.java

@@ -0,0 +1,64 @@
+package com.tzld.piaoquan.recommend.framework.merger;
+
+
+import java.util.*;
+
+
+public class StrategyQueueInfo {
+    private String queueName;
+
+    private boolean isLeaf = false;
+
+    private String queueClass; // Full path of StrategyQueue Class
+
+    private List<MergeRule> rulesList;
+
+    private Set<String> children;
+
+    public StrategyQueueInfo(String name) {
+        this.queueName = name;
+        rulesList = new LinkedList<MergeRule>();
+        children = new HashSet<String>();
+    }
+
+    public String getQueueName() {
+        return queueName;
+    }
+
+    public boolean isLeaf() {
+        return isLeaf;
+    }
+
+    public void setLeaf(boolean leaf) {
+        isLeaf = leaf;
+    }
+
+
+    public String getQueueClass() {
+        return queueClass;
+    }
+
+    public void setQueueClass(String queueClass) {
+        this.queueClass = queueClass;
+    }
+
+    public void addRule(MergeRule rule) {
+        this.rulesList.add(rule);
+    }
+
+    public List<MergeRule> getRulesList() {
+        return rulesList;
+    }
+
+    public void addChild(String childName) {
+        this.children.add(childName);
+    }
+
+    public Set<String> getChildren() {
+        return children;
+    }
+}
+
+
+
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/ranker/AbstractRanker.java

@@ -0,0 +1,47 @@
+package com.tzld.piaoquan.recommend.framework.ranker;
+
+
+import com.tzld.piaoquan.recommend.framework.common.item.RankerItem;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.IndexDescription;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public abstract class AbstractRanker {
+    protected static Logger LOGGER = LoggerFactory.getLogger(AbstractRanker.class);
+
+    public static final String ADJUST_POSITION_STRING = "position";
+    public static final String QUEUE_IDENTIFY_STRING = "queue-identify";
+    public static final String ADJUST_LIMIT_STRING = "adjust-limit";
+    public static final String ADJUST_RATIO_STRING = "adjust-ratio";
+    public static final String RERANK_COUNT_CONF_STRING = "rerank-count";
+    public static final String WINDOW_SIZE_CONF_STRING = "window-size";
+    public static final String SIMILARITY_LIMIT_CONF_STRING = "similar-limit";
+
+    protected RankerConfig rankConfig;
+
+    public AbstractRanker(RankerConfig config) {
+        this.rankConfig = config;
+    }
+
+    protected void afterRank() {
+        // TODO: add decision label
+    }
+
+    public void rank(final RecommendRequest requestContext, final User user, final int requestIndex, List<RankerItem> rankerItems) {
+        if (!rankConfig.isEnable()) {
+            return;
+        }
+        doRank(requestContext, user, requestIndex, rankerItems);
+        afterRank();
+    }
+
+    public abstract void doRank(final RecommendRequest requestContext, final User user, final int requestIndex, List<RankerItem> rankerItems);
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/ranker/RankPipeline.java

@@ -0,0 +1,84 @@
+package com.tzld.piaoquan.recommend.framework.ranker;
+
+
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.typesafe.config.Config;
+import com.tzld.piaoquan.recommend.framework.common.item.RankerItem;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang.exception.ExceptionUtils;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public class RankPipeline {
+    private List<AbstractRanker> rankers = Lists.newArrayList();
+    private Map<String, Integer> boostMap;
+
+    public RankPipeline(List<AbstractRanker> rankers) {
+        this.rankers = rankers;
+    }
+
+    public void setBoostMap(Map<String, Integer> boostMap) {
+        this.boostMap = boostMap;
+    }
+
+    public static void boostQueues(List<RankerItem> rankerItems, Map<String, Integer> boostMap) {
+        if (boostMap == null || boostMap.isEmpty()) {
+            return ;
+        }
+
+        List<RankerItem> boostItems = Lists.newArrayList();
+        Map<String, Integer> boostCounts = Maps.newConcurrentMap();
+        boostCounts.putAll(boostMap);
+        for (Iterator<RankerItem> itemIterator = rankerItems.iterator(); itemIterator.hasNext(); ) {
+            RankerItem currentItem = itemIterator.next();
+            if (boostCounts.containsKey(currentItem.getQueue())) {
+                itemIterator.remove();
+                currentItem.setScore((currentItem.getScore() + 1.0) * 100.0); // FIXME: boost priority
+                boostItems.add(currentItem);
+                int myQueueBoostCount = boostCounts.get(currentItem.getQueue()) - 1;
+                if (myQueueBoostCount > 0) {
+                    boostCounts.put(currentItem.getQueue(), myQueueBoostCount);
+                } else {
+                    boostCounts.remove(currentItem.getQueue());
+                }
+            }
+        }
+        rankerItems.addAll(0, boostItems);
+    }
+
+    public List<RankerItem> doRank(final RecommendRequest requestContext, final User user, final int requestIndex, List<RankerItem> rankerItems) {
+        // initial rank by final score
+        Collections.sort(rankerItems);
+
+        for (AbstractRanker ranker : rankers) {
+            ranker.rank(requestContext, user, requestIndex, rankerItems);
+        }
+
+        // boost
+        if (boostMap != null && !boostMap.isEmpty()) {
+            boostQueues(rankerItems, boostMap);
+        }
+
+        return rankerItems;
+    }
+
+    public static List<AbstractRanker> constructRankers(Config rankersConfig) {
+        List<AbstractRanker> rankers = Lists.newArrayList();
+        for (RankerConfig config : RankerConfig.loadRankerConfig(rankersConfig)) {
+            try {
+                AbstractRanker ranker = (AbstractRanker) Class.forName(config.getRankerClass()).getConstructor(config.getClass()).newInstance(config);
+                rankers.add(ranker);
+            } catch (Exception e) {
+                //
+            }
+        }
+
+        return rankers;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/ranker/RankerConfig.java

@@ -0,0 +1,97 @@
+package com.tzld.piaoquan.recommend.framework.ranker;
+
+
+import com.google.common.collect.Lists;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import com.tzld.piaoquan.recommend.framework.utils.ConfigUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+
+import java.util.Collections;
+import java.util.List;
+
+public class RankerConfig implements Comparable<RankerConfig> {
+
+    private String name;
+    private String rankerClass;
+
+    private int rankPriority;
+    private boolean enableSwitch;
+
+    private String experimentKey;
+
+    private Config paramConfig;
+
+    public RankerConfig(String name, String rankerClass, int rankPriority, boolean enableSwitch, String experimentKey, Config paramConfig) {
+        this.name = name;
+        this.rankerClass = rankerClass;
+        this.rankPriority = rankPriority;
+        this.enableSwitch = enableSwitch;
+        this.experimentKey = experimentKey;
+        this.paramConfig = paramConfig;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getRankerClass() {
+        return rankerClass;
+    }
+
+    public int getPriority() {
+        return rankPriority;
+    }
+
+    public boolean isEnable() {
+        return enableSwitch;
+    }
+
+    public String getExperimentKey() {
+        return experimentKey;
+    }
+
+    public Config getParamConfig() {
+        return paramConfig;
+    }
+
+
+    public static List<RankerConfig> loadRankerConfig(Config config) {
+        List<RankerConfig> rankerConfigList = Lists.newArrayList();
+        try {
+            for (ConfigObject.Entry<String, ConfigValue> it : config.getConfig("ranker-config").root().entrySet()) {
+                String name = it.getKey();
+                Config entryConf = ((ConfigObject)it.getValue()).toConfig();
+                int priority = ConfigUtils.loadOrDefault(entryConf, "ranker-priority", 100);
+                boolean enableSwitch = ConfigUtils.loadOrDefault(entryConf, "enable-switch", true);
+                String experimentKey = ConfigUtils.loadOrDefault(entryConf, "experiment-key", "");
+                Config paramConfig = ConfigUtils.loadOrDefault(entryConf, "param-config", (Config) null);
+                RankerConfig rankerConfig = new RankerConfig(name, entryConf.getString("ranker"), priority, enableSwitch, experimentKey, paramConfig);
+                rankerConfigList.add(rankerConfig);
+                //按照priority倒叙排序
+                Collections.sort(rankerConfigList);
+            }
+        } catch (Exception e) {
+            // log
+        }
+
+        return rankerConfigList;
+    }
+
+    /**
+     * 降序排列
+     * @param o
+     * @return
+     */
+    @Override
+    public int compareTo(RankerConfig o) {
+        if (o == null) {
+            return -1;
+        }
+        if (this.rankPriority == o.rankPriority) {
+            return 0;
+        }
+        return this.rankPriority < o.rankPriority ? 1 : -1;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/recaller/AbstractFilter.java

@@ -0,0 +1,37 @@
+package com.tzld.piaoquan.recommend.framework.recaller;
+
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.IndexDescription;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class AbstractFilter<T> {
+    protected final static Logger LOGGER = LoggerFactory.getLogger(AbstractFilter.class);
+    protected final FilterConfigInfo filterConfigInfo;
+    protected final RecommendRequest requestContext;
+    protected final User user;
+    protected final int requestIndex;
+
+    public AbstractFilter(FilterConfigInfo filterConfigInfo,
+                          RecommendRequest requestContext,
+                          User user,
+                          Integer requestIndex) {
+
+        this.filterConfigInfo = filterConfigInfo;
+        this.requestContext = requestContext;
+        this.user = user;
+        this.requestIndex = requestIndex;
+    }
+
+    public FilterConfigInfo getFilterConfigInfo() {
+        return filterConfigInfo;
+    }
+
+    public abstract boolean predicate(Candidate candidate, T t);
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/recaller/AbstractRecaller.java

@@ -0,0 +1,353 @@
+package com.tzld.piaoquan.recommend.framework.recaller;
+
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import com.tzld.piaoquan.recommend.framework.common.item.ItemProvider;
+import com.tzld.piaoquan.recommend.framework.common.item.RankerItem;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.*;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+
+public abstract class AbstractRecaller<T> {
+
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(AbstractRecaller.class);
+    private static final long DEFAULT_QUEUE_LOAD_TIMEOUT = 150; // ms
+    private static final long DEFAULT_PARALLEL_FILTER_TIMEOUT = 200; // ms
+    private static final ExecutorService filterExecutorService = Executors.newFixedThreadPool(128);
+    private static final ExecutorService fetchQueueExecutorService = Executors.newFixedThreadPool(128);
+
+    protected final ItemProvider<T> itemProvider;
+    private final QueueProvider<T> queueProvider;
+    private final FilterConfig filterConfig;
+    private final long QUEUE_LOAD_TIMEOUT;
+
+    public AbstractRecaller(ItemProvider<T> itemProvider, QueueProvider<T> queueProvider, FilterConfig filterConfig) {
+        this(itemProvider, queueProvider, filterConfig, DEFAULT_QUEUE_LOAD_TIMEOUT);
+    }
+
+    public AbstractRecaller(ItemProvider<T> itemProvider, QueueProvider<T> queueProvider,
+                            FilterConfig filterConfig, long queueLoadTimeout) {
+        this.itemProvider = itemProvider;
+        this.queueProvider = queueProvider;
+        this.filterConfig = filterConfig;
+        this.QUEUE_LOAD_TIMEOUT = queueLoadTimeout;
+    }
+
+    public abstract Map<String, Long> getCacheRulesConfig();
+
+    public abstract String extractItemId(Entry<T> entry);
+
+    public abstract Object extractItemBytes(Entry<T> entry);
+    public abstract boolean isValidItem(T t);
+
+    public abstract List<Candidate> filterCandidates(RecommendRequest requestData, User user, int requestIndex, List<Candidate> candidates);
+
+    public ItemProvider<T> getItemProvider() {
+        return itemProvider;
+    }
+
+    public QueueProvider<T> getQueueProvider() {
+        return queueProvider;
+    }
+
+    public Map<Candidate, Queue<T>> getQueue(String queueStr) throws Exception {
+        Candidate candidate = new Candidate();
+        candidate.setCandidateKey(queueStr);
+        return loadQueues(Arrays.asList(candidate));
+    }
+
+    public Optional<T> getItem(String itemId) throws Exception {
+        return itemProvider.get(itemId);
+    }
+
+    public long getItemDBSize() {
+        return this.itemProvider.dbSize();
+    }
+
+    public long getIndexDBSize() {
+        return this.queueProvider.dbSize();
+    }
+
+    public long getQueueTTL(String queueNameStr, Map<String, Long> cacheRules) {
+
+//        LOGGER.info("[recall server] queue name from string={}", queueNameStr);
+        QueueName name = QueueName.fromString(queueNameStr);
+
+        for (String match : name.getMatches()) {
+            if (cacheRules.containsKey(match)) {
+                return cacheRules.get(match);
+            }
+        }
+        return name.getTTL();
+    }
+
+    private Map<String, Double> listToMap(List<String> list) {
+        Map<String, Double> map = new HashMap<String, Double>();
+        if (list != null) {
+            for (String elem : list) {
+                map.put(elem, 1.0);
+            }
+        }
+        return map;
+    }
+
+    /**
+     * @param entries
+     * @param candidate
+     * @param user
+     * @return
+     */
+    private List<RankerItem> toHits(final Iterable<Entry<T>> entries, final Candidate candidate, final User user) {
+
+        List<RankerItem> result = new ArrayList<RankerItem>();
+        for (Entry entry : entries) {
+            RankerItem item = new RankerItem();
+            item.setId(extractItemId(entry));
+
+            CandidateInfo candidateInfo = new CandidateInfo();
+            candidateInfo.setCandidate_queue(candidate.getCandidateKey());
+            candidateInfo.setCandidate(candidate);
+            item.setCandidateInfo(candidateInfo);
+            item.addToCandidateInfoList(candidateInfo);
+
+            result.add(item);
+        }
+        return result;
+    }
+
+    public Map<Candidate, Queue<T>> loadQueues(List<Candidate> candidates) {
+
+        // update queueName
+        final Map<String, Long> cacheRules = getCacheRulesConfig();
+        Iterable<Candidate> updateCandidates = FluentIterable.from(candidates).transform(new Function<Candidate, Candidate>() {
+            @Override
+            public Candidate apply(Candidate candidate) {
+                try {
+                    long ttl = getQueueTTL(candidate.getCandidateKey(), cacheRules);
+                    candidate.setCandidateQueueName(QueueName.fromString(candidate.getCandidateKey(), ttl));
+                } catch (Exception e) {
+                    candidate.setCandidateQueueName(null);
+                    LOGGER.error("error parse QueueName [{}]", candidate.getCandidateKey());
+                }
+                return candidate;
+            }
+        }).filter(new Predicate<Candidate>() {
+            @Override
+            public boolean apply(Candidate candidate) {
+                return candidate.getCandidateQueueName() != null && !StringUtils.equals("dssm-exploit-index", candidate.getQueue())
+                        && !StringUtils.equals("dnnytb-exploit-index", candidate.getQueue());
+            }
+        });
+
+        // parse queues
+        Iterable<QueueName> queueNames = FluentIterable.from(updateCandidates).transform(new Function<Candidate, QueueName>() {
+            @Override
+            public QueueName apply(Candidate candidate) {
+                return candidate.getCandidateQueueName();
+            }
+        });
+
+        // parallel load queues
+        Map<QueueName, Queue<T>> queues = Maps.newConcurrentMap();
+        try {
+            queues = queueProvider.loads(Lists.newArrayList(queueNames), QUEUE_LOAD_TIMEOUT, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            LOGGER.error("load queue occur error [{}]", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        // parse candidate map
+        Map<Candidate, Queue<T>> candidateQueueMap = Maps.newConcurrentMap();
+        for (Candidate candidate : updateCandidates) {
+            QueueName name = candidate.getCandidateQueueName();
+            if (queues.containsKey(name) && queues.get(name) != null) {
+                candidateQueueMap.put(candidate, queues.get(name));
+            }
+        }
+        return candidateQueueMap;
+    }
+
+
+    /**
+     * recall
+     * @param requestData
+     * @param user
+     * @param requestIndex
+     * @param recallCandidates
+     * @return
+     */
+    public List<RankerItem> recalling(final RecommendRequest requestData, final User user, int requestIndex, List<Candidate> recallCandidates) {
+
+        long startTime = System.currentTimeMillis();
+        final RecallFilter<T> recallFilter = new RecallFilter<T>(this.filterConfig, requestData, user, requestIndex);
+
+        // filter candidates
+        final List<Candidate> refactorCandidates = filterCandidates(requestData, user, requestIndex, recallCandidates);
+
+        // load queue
+        List<Callable<Map<Candidate, Queue<T>>>> fetchQueueCalls = Lists.newArrayList();
+
+        fetchQueueCalls.add(new Callable<Map<Candidate, Queue<T>>>() {
+            @Override
+            public Map<Candidate, Queue<T>> call() throws Exception {
+                boolean isFromRedis = true;
+                return obtainQueue(refactorCandidates, requestData, user, isFromRedis);
+            }
+        });
+
+        fetchQueueCalls.add(new Callable<Map<Candidate, Queue<T>>>() {
+            @Override
+            public Map<Candidate, Queue<T>> call() throws Exception {
+                boolean isFromRedis = false;
+                return obtainQueue(refactorCandidates, requestData, user, isFromRedis);
+            }
+        });
+
+        List<Future<Map<Candidate, Queue<T>>>> fetchQueueFutures = null;
+
+        try {
+            fetchQueueFutures = fetchQueueExecutorService.invokeAll(fetchQueueCalls, DEFAULT_QUEUE_LOAD_TIMEOUT,
+                    TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("[fetch queue error] inter fail: {}", ExceptionUtils.getStackTrace(e));
+        } catch (Exception e) {
+            LOGGER.error("[fetch queue error] ex", ExceptionUtils.getStackTrace(e));
+        }
+
+        Map<Candidate, Queue<T>> candidateQueueMap = Maps.newHashMap();
+        if (CollectionUtils.isNotEmpty(fetchQueueFutures)) {
+
+            for (Future<Map<Candidate, Queue<T>>> future : fetchQueueFutures) {
+
+                if (future.isDone() && !future.isCancelled()) {
+
+                    Map<Candidate, Queue<T>> result = null;
+                    try {
+                        result = future.get();
+                    } catch (InterruptedException e) {
+                        LOGGER.error("[fetch queue error] InterruptedException {}", ExceptionUtils.getStackTrace(e));
+                    } catch (ExecutionException e) {
+                        LOGGER.error("[fetch queue error] ex {}", ExceptionUtils.getStackTrace(e));
+                    }
+
+                    if (result != null) {
+                        candidateQueueMap.putAll(result);
+                    }
+                }
+            }
+
+        }
+
+        // do filter
+        long filterStartTime = System.currentTimeMillis();
+
+        List<Map.Entry<Candidate, Queue<T>>> batch = new ArrayList<Map.Entry<Candidate, Queue<T>>>();
+        final List<Callable<List<RankerItem>>> callables = new ArrayList<Callable<List<RankerItem>>>();
+        int expectedRecallSum = 0;
+        for(final Map.Entry<Candidate, Queue<T>> entry : candidateQueueMap.entrySet()) {
+            callables.add(new Callable<List<RankerItem>>() {
+                @Override
+                public List<RankerItem> call() throws Exception {
+                    List<RankerItem> candidateHits = new ArrayList<RankerItem>();
+                    final Candidate candidate = entry.getKey();
+                    try {
+                        // 1. filter
+                        Iterable<Entry<T>> entries = FluentIterable.from(entry.getValue()).filter(new Predicate<Entry<T>>() {
+                            @Override
+                            public boolean apply(Entry<T> entry) {
+                                return isValidItem(entry.item) &&
+                                        recallFilter.predicate(candidate, entry.item);
+                            }
+                        }).limit(candidate.getCandidateNum());
+
+                        // 2. toHits
+                        candidateHits.addAll(toHits(entries, candidate, user));
+
+                        // debug log for tracing
+                        LOGGER.debug("recalled candidate [{}], queue length [{}], expected [{}], hit [{}]",
+                                new Object[]{candidate.getCandidateKey(), entry.getValue().size(), candidate.getCandidateNum(), candidateHits.size()});
+                    } catch (Exception e) {
+                        LOGGER.error("recall filter queue occur error, queue [{}], error: [{}]", candidate.toString(), ExceptionUtils.getFullStackTrace(e));
+                    }
+
+                    return candidateHits;
+                }
+            });
+        }
+
+        Map<String, RankerItem> hits = new HashMap<String, RankerItem>();
+        try {
+            List<Future<List<RankerItem>>> futures = filterExecutorService.invokeAll(callables, DEFAULT_PARALLEL_FILTER_TIMEOUT, TimeUnit.MILLISECONDS);
+
+            for (Future<List<RankerItem>> future : futures) {
+                try {
+                    if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                        List<RankerItem> part = future.get();
+                        // add to result
+                        for (RankerItem item : part) {
+                            // merge candidate Info
+                            if (hits.containsKey(item.getId())) {
+                                hits.get(item.getId()).addToCandidateInfoList(item.getCandidateInfo());
+                            } else {
+                                hits.put(item.getId(), item);
+                            }
+                        }
+                        // log
+                    } else {
+
+                    }
+                } catch (Exception e) {
+                  //
+                }
+            }
+        } catch (Exception e) {
+            //
+        }
+
+        List<RankerItem> result = new ArrayList<RankerItem>(hits.values());
+        return result;
+    }
+
+    private Map<Candidate, Queue<T>> obtainQueue(List<Candidate> refactorCandidates, RecommendRequest requestData, User user, boolean isFromRedis) {
+
+        if (isFromRedis) {
+            return loadQueues(refactorCandidates);
+        } else {
+            return fetchQueues(refactorCandidates, requestData, user);
+        }
+
+    }
+
+    protected Map<Candidate, Queue<T>> fetchQueues(List<Candidate> candidates, RecommendRequest requestData, User user) {
+
+        return null;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/recaller/FilterConfig.java

@@ -0,0 +1,95 @@
+package com.tzld.piaoquan.recommend.framework.recaller;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class FilterConfig {
+    private static Logger LOGGER = LoggerFactory.getLogger(FilterConfig.class);
+    private List<FilterConfigInfo> filterConfigInfoList = new ArrayList<FilterConfigInfo>();
+
+    public FilterConfig(Config config) {
+        this.load(config);
+    }
+
+
+    public FilterConfig(String configFile) {
+        this.load(configFile);
+    }
+
+    public FilterConfig() {
+    }
+
+    public boolean load(String configFile) {
+        Config filterConfig = ConfigFactory.parseResources(configFile);
+        return load(filterConfig);
+    }
+
+    public boolean load(Config config) {
+        Config recallConfig = config.getConfig("recall-config");
+
+        Config filterConf = recallConfig.getConfig("filter-config");
+        try {
+            loadFilters(filterConf);
+            int pos = 0;
+            for (FilterConfigInfo filterConfigInfo : filterConfigInfoList) {
+                LOGGER.info("filter at position [{}], priority [{}] filter name [{}] ",
+                        new Object[]{pos++, filterConfigInfo.getFilterPriority(), filterConfigInfo.getConfigName()});
+            }
+            LOGGER.debug("Load filter config success");
+        } catch (Exception e) {
+            LOGGER.error("Load filter config failed, [{}]", ExceptionUtils.getFullStackTrace(e));
+            return false;
+        }
+
+        return true;
+    }
+
+    public List<FilterConfigInfo> getFilterConfigInfoList() {
+        return filterConfigInfoList;
+    }
+
+    private void loadFilters(Config config) throws Exception {
+        ConfigObject confObj = config.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            Config conf = ((ConfigObject) it.getValue()).toConfig();
+
+            // parse config
+            String configName = it.getKey();
+            String filterName = conf.getString("filter-name");
+            int filterPriority = 0;
+            if (conf.hasPath("filter-priority"))
+                filterPriority = conf.getInt("filter-priority");
+
+            List<Integer> disableExpIds = new ArrayList<Integer>();
+            if (conf.hasPath("disable-expids")) {
+                disableExpIds = conf.getIntList("disable-expids");
+            }
+            FilterConfigInfo filterConfigInfo = new FilterConfigInfo(configName,
+                    filterName, filterPriority, disableExpIds);
+            LOGGER.debug("parse filter config info [{}]", filterConfigInfo);
+
+            addConfigByPriority(filterConfigInfoList, filterConfigInfo);
+        }
+    }
+
+    private void addConfigByPriority(List<FilterConfigInfo> configInfoList, FilterConfigInfo addConfigInfo) {
+
+        int pos = 0;
+        for (; pos < configInfoList.size(); pos++) {
+            if (configInfoList.get(pos).getFilterPriority() <= addConfigInfo.getFilterPriority()) {
+                break;
+            }
+        }
+        configInfoList.add(pos, addConfigInfo);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/recaller/FilterConfigInfo.java

@@ -0,0 +1,58 @@
+package com.tzld.piaoquan.recommend.framework.recaller;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.FluentIterable;
+
+
+
+public class FilterConfigInfo {
+    private Set<Integer> disableExpIds;
+    private String filterName;
+    private Integer filterPriority;
+    private String configName;
+
+    public FilterConfigInfo(String configName,
+                            String filterName,
+                            Integer filterPriority,
+                            List<Integer> expIds) {
+        this.configName = configName;
+        this.filterName = filterName;
+        this.filterPriority = filterPriority;
+        this.disableExpIds = new HashSet<Integer>();
+        this.disableExpIds.addAll(expIds);
+    }
+
+    public Integer getFilterPriority() {
+        return filterPriority;
+    }
+
+    public String getFilterName() {
+        return filterName;
+    }
+
+    public Set<Integer> getDisableExpIds() {
+        return disableExpIds;
+    }
+
+    public String getConfigName() {
+        return configName;
+    }
+
+    @Override
+    public String toString() {
+        return configName + ":" + filterPriority + ":" + filterName + ":" +
+                Joiner.on(",").join(FluentIterable.from(disableExpIds).transform(new Function<Integer, String>() {
+                    @Nullable
+                    @Override
+                    public String apply(Integer integer) {
+                        return integer.toString();
+                    }
+                }));
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/recaller/RecallFilter.java

@@ -0,0 +1,70 @@
+package com.tzld.piaoquan.recommend.framework.recaller;
+
+
+
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class RecallFilter<T> {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(RecallFilter.class);
+    public int filterNum;
+    private FilterConfig config;
+    private List<AbstractFilter<T>> filters;
+    private RecommendRequest requestContext;
+    private User user;
+    private int requestIndex;
+
+    public RecallFilter(FilterConfig config,
+                        RecommendRequest requestContext,
+                        User user,
+                        int requestIndex) {
+
+        this.config = config;
+        this.requestContext = requestContext;
+        this.user = user;
+        this.requestIndex = requestIndex;
+        this.filters = new ArrayList<AbstractFilter<T>>();
+        this.constructFilters(config);
+    }
+
+    public List<AbstractFilter<T>> getFilters() {
+        return filters;
+    }
+
+    public void constructFilters(FilterConfig config) {
+        // add filter
+        if (config != null) {
+            for (FilterConfigInfo filterConfigInfo : config.getFilterConfigInfoList()) {
+                try {
+                    AbstractFilter<T> filter = (AbstractFilter) Class.forName(filterConfigInfo.getFilterName())
+                            .getConstructor(FilterConfigInfo.class, RecommendRequest.class, User.class, Integer.class)
+                            .newInstance(filterConfigInfo, this.requestContext, this.user, this.requestIndex);
+
+                    this.filters.add(filter);
+                } catch (Exception e) {
+                    LOGGER.error("Filter config info construct error, [{}] [{}]",
+                            filterConfigInfo, ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public boolean predicate(Candidate candidate, T t) {
+        for (AbstractFilter filter : filters) {
+            if (!filter.predicate(candidate, t)) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/AbstractRetriever.java

@@ -0,0 +1,27 @@
+package com.tzld.piaoquan.recommend.framework.retrieve;
+
+
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.IndexDescription;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.Map;
+
+public abstract class AbstractRetriever {
+
+    
+    public abstract int doRetrieve(Map<String, Candidate> candidates,
+                                   final RecommendRequest requestData,
+                                   final User user,
+                                   final String channelName,
+                                   final RetrieveConfig config);
+
+    protected void addCandidateKey(Map<String, Candidate> candidateMap, String queue, IndexDescription index) {
+        Candidate candidate = IndexDescription.convertToCandidate(queue, index);
+        String mapKey = queue + "##" + candidate.getCandidateQueueName();
+        candidateMap.put(mapKey, candidate);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/RetrieveChannel.java

@@ -0,0 +1,44 @@
+package com.tzld.piaoquan.recommend.framework.retrieve;
+
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.IndexDescription;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+
+public class RetrieveChannel {
+    private String name;
+    private AbstractRetriever retriever;
+    private RetrieveConfig config;
+
+    public RetrieveChannel(String name) {
+        this.name = name;
+    }
+
+    public void setConfig(RetrieveConfig config) {
+        this.config = config;
+    }
+
+    public void setRetrieverClass(String retrieverClass) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
+        this.retriever = (AbstractRetriever) Class.forName(retrieverClass).getConstructor().newInstance();
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public AbstractRetriever getRetriever() {
+        return retriever;
+    }
+
+    public RetrieveConfig getConfig() {
+        return config;
+    }
+
+    public int doRetrieve(Map<String, Candidate> candidates, final RecommendRequest requestData, final User user) {
+        return getRetriever().doRetrieve(candidates, requestData, user, getName(), config);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/RetrieveConfig.java

@@ -0,0 +1,142 @@
+package com.tzld.piaoquan.recommend.framework.retrieve;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.IndexDescription;
+import com.tzld.piaoquan.recommend.framework.utils.ParamUtils;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import java.util.HashMap;
+import java.util.Map;
+
+public class RetrieveConfig {
+    private final int priority;
+    private final int recallWeight;
+    private final int bsWeight;
+    private final int asProtectCount;
+    private final int boostCount;
+
+    private final boolean enable;
+    private final String enableKey;
+    private final Map<String, IndexDescription> childIndics;
+    private final Config paramConfig;
+
+    public RetrieveConfig(int priority,
+                          int recallWeight, int bsWeight, int asProtectCount, int boostCount,
+                          boolean enable, String enableKey,
+                          Config indexDescConfig, Config paramConfig) {
+        this.priority = priority;
+        this.recallWeight = recallWeight;
+        this.bsWeight = bsWeight;
+        this.asProtectCount = asProtectCount;
+        this.boostCount = boostCount;
+        this.enable = enable;
+        this.enableKey = enableKey;
+        this.childIndics = parseChildIndics(indexDescConfig);
+        this.paramConfig = paramConfig;
+    }
+
+    public int getRecallWeight() {
+        return recallWeight;
+    }
+
+    public int getBsWeight() {
+        return bsWeight;
+    }
+
+    public int getAsProtectCount() {
+        return asProtectCount;
+    }
+
+    public int getBoostCount() {
+        return boostCount;
+    }
+
+    public int getPriority() {
+        return priority;
+    }
+
+    public boolean isEnable(final RecommendRequest requestData, final String layerPath) {
+        return enable || ParamUtils.getExpParamValue(requestData, layerPath, getEnableKey()).equals("1");
+    }
+
+    public String getEnableKey() {
+        return enableKey;
+    }
+
+    public Map<String, IndexDescription> getChildIndics() {
+        return childIndics;
+    }
+
+    public Config getParamConfig() {
+        return paramConfig;
+    }
+
+    @Override
+    public String toString() {
+        return "RetrieveConfig{" + "\'" +
+                "priority=" + priority + "\'" +
+                ", recall-weight=" + recallWeight + "\'" +
+                ", bs-weight=" + bsWeight + "\'" +
+                ", as-protect-count=" + asProtectCount + "\'" +
+                ", boost-count=" + boostCount + "\'" +
+                ", enable_key='" + enableKey + "\'" +
+                ", paramConfig=" + paramConfig +
+                '}';
+    }
+
+    public static RetrieveConfig parseRetrieveConfig(Config retrieveConfig) {
+        int recallWeight = retrieveConfig.getInt("recall-weight");
+        int bsWeight = retrieveConfig.getInt("bs-weight");
+        int asProtectCount = retrieveConfig.getInt("as-protect-count");
+        int boostCount = retrieveConfig.getInt("boost-count");
+        //默认是0
+        int priority = loadOptionIntConfig(retrieveConfig, "priority");
+        //默认为true
+        boolean enableSwitch = loadOptionBooleanConfig(retrieveConfig, "enable-switch");
+
+        String enableKey = loadOptionStringConfig(retrieveConfig,"experiment-enable-key");
+        Config paramConfig = loadOptionConfig(retrieveConfig, "param-config");
+        Config indexConfig = loadOptionConfig(retrieveConfig, "children-index"); // FIXME: try load
+        return new RetrieveConfig(priority,
+                recallWeight, bsWeight, asProtectCount, boostCount,
+                enableSwitch, enableKey, indexConfig, paramConfig);
+    }
+
+    private static Map<String, IndexDescription> parseChildIndics(Config indexDescConfig) {
+        Map<String, IndexDescription> indexDescMap = new HashMap<String, IndexDescription>();
+        ConfigObject confObj = indexDescConfig.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            String myName = it.getKey();
+            Config myConf = ((ConfigObject) it.getValue()).toConfig();
+            IndexDescription index = IndexDescription.parseIndexDescription(myName, myConf);
+            indexDescMap.put(myName, index);
+        }
+        return indexDescMap;
+    }
+
+    // FIXME: ConfigLoadUtils
+    private static String loadOptionStringConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getString(path) : null;
+    }
+
+    private static Config loadOptionConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
+    }
+
+    private static int loadOptionIntConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getInt(path) : 0;
+    }
+
+    /**
+     * 默认为true
+     * @param config
+     * @param path
+     * @return
+     */
+    private static boolean loadOptionBooleanConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getBoolean(path) : true;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/RetrievePipelineUtils.java

@@ -0,0 +1,137 @@
+package com.tzld.piaoquan.recommend.framework.retrieve;
+
+
+
+import com.google.common.collect.Maps;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.IndexDescription;
+import com.tzld.piaoquan.recommend.framework.utils.ConfigUtils;
+import com.tzld.piaoquan.recommend.framework.utils.ParamUtils;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang.exception.ExceptionUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+public class RetrievePipelineUtils {
+    private static final Logger LOGGER = LoggerFactory.getLogger(RetrievePipelineUtils.class);
+
+    public static RetrieverPipeline getRetrievePipeline(final Map<Integer, Config> retrieveConfigMap, final RecommendRequest requestData) {
+        List<Config> expConfigList = new ArrayList<Config>();
+
+        // TODO: replace with retrieve-layer-config-key
+
+         Set<Integer> expIdSet = ParamUtils.getExpIdSet(requestData);
+         for (Integer expId : expIdSet) {
+             ConfigUtils.addConfig(expId, retrieveConfigMap, expConfigList);
+        }
+
+        Config retrieveBaseConfig = null;
+        if (retrieveConfigMap.containsKey("base")) {
+            retrieveBaseConfig = retrieveConfigMap.get("base");
+        } else if (retrieveConfigMap.containsKey("0")) {
+            retrieveBaseConfig = retrieveConfigMap.get("0");
+        } else {
+            // TODO: throw exception
+        }
+        return getRetrievePipeline(retrieveBaseConfig, expConfigList);
+    }
+
+    public static int sumTotalRecallCount(Map<String, Candidate> candidateMap) {
+        int total = 0;
+        for (Map.Entry<String, Candidate> candidateEntry : candidateMap.entrySet()) {
+            total += candidateEntry.getValue().getMergeQueueNum();
+        }
+        return total;
+    }
+
+    public static int normalizeRetrieve(Map<String, Candidate> candidates, final int recallCount) {
+        // TODO: normalize retrieve weight
+        int initRecallCount = sumTotalRecallCount(candidates);
+
+        if (initRecallCount <= 0) {
+            return 0;
+        }
+
+        double normalizeRatio = recallCount * 1.0 / initRecallCount;
+        for (Map.Entry<String, Candidate> candidateEntry : candidates.entrySet()) {
+            candidateEntry.getValue().setMergeQueueNum((int)(candidateEntry.getValue().getMergeQueueNum() * normalizeRatio + 1));
+            candidateEntry.getValue().setCandidateNum((int)(candidateEntry.getValue().getCandidateNum() * normalizeRatio + 1));
+        }
+
+        return 0;
+    }
+
+
+    public static RetrieverPipeline getRetrievePipeline(Config mergedConfig) {
+        return new RetrieverPipeline(loadRetrieveConfig(mergedConfig));
+    }
+
+    public static RetrieverPipeline getRetrievePipeline(Config baseConfig, List<Config> configList) {
+        Config mergedConfig = ConfigUtils.mergeConfig(baseConfig, configList);
+        return getRetrievePipeline(mergedConfig);
+    }
+
+
+    public static Map<String, RetrieveChannel> loadRetrieveConfig(Config config) {
+        Map<String, RetrieveChannel> channelsMap = Maps.newHashMap();
+
+        try {
+            Config mergedConfig = config.getConfig("retrieve-config");
+
+            // queues
+            if (!mergedConfig.hasPath("queues")) {
+                // FIXME: throw exception
+                LOGGER.error("queues config not exists!");
+                return null;
+            }
+
+            // retrievers
+            if (!mergedConfig.hasPath("configs")) {
+                // FIXME: throw exception
+                LOGGER.error("retrievers config not exists!");
+                return null;
+            }
+
+            for (ConfigObject.Entry<String, ConfigValue> it : mergedConfig.getConfig("queues").root().entrySet()) {
+                String name = it.getKey();
+                Config entryConf = ((ConfigObject)it.getValue()).toConfig();
+                if (!entryConf.hasPath("retriever")) {
+                    // FIXME: throw exception
+                    LOGGER.error("retriever class not exists for queue {}!", name);
+                    return null;
+                }
+                RetrieveChannel channel = new RetrieveChannel(name);
+                channel.setRetrieverClass(entryConf.getString("retriever"));
+                channelsMap.put(name, channel);
+            }
+
+            for (ConfigObject.Entry<String, ConfigValue> it : mergedConfig.getConfig("configs").root().entrySet()) {
+                Config entryConf = ((ConfigObject)it.getValue()).toConfig();
+                if (channelsMap.containsKey(it.getKey())) {
+                    RetrieveConfig retrieveConfig = RetrieveConfig.parseRetrieveConfig(entryConf);
+                    RetrieveChannel channel = channelsMap.get(it.getKey());
+                    if (channel.getConfig() == null || channel.getConfig().getPriority() < retrieveConfig.getPriority()) {
+                        channel.setConfig(retrieveConfig);
+                    }
+                }
+            }
+
+            //
+        } catch (Exception e) {
+            // error
+        }
+
+        return channelsMap;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/RetrieverPipeline.java

@@ -0,0 +1,42 @@
+package com.tzld.piaoquan.recommend.framework.retrieve;
+
+
+import com.google.common.collect.Maps;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.IndexDescription;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+
+import java.util.Map;
+
+public class RetrieverPipeline {
+
+    private Map<String, RetrieveChannel> retrieveMap;
+
+    public RetrieverPipeline(Map<String, RetrieveChannel> retrieverMap) {
+        this.retrieveMap = retrieverMap;
+    }
+
+    public Map<String, Candidate> doRetrieve(final RecommendRequest requestData, final User user, final String layerPath) {
+        Map<String, Candidate> candidates = Maps.newHashMap();
+        for (Map.Entry<String, RetrieveChannel> retrieverEntry : retrieveMap.entrySet()) {
+            try {
+                if (retrieverEntry.getValue().getConfig().isEnable(requestData, layerPath)) {
+                    Map<String, Candidate> myCandidates = Maps.newHashMap();
+                    retrieverEntry.getValue().doRetrieve(myCandidates, requestData, user);
+
+                    RetrievePipelineUtils.normalizeRetrieve(myCandidates, retrieverEntry.getValue().getConfig().getRecallWeight());
+                    int myRequestRecallCount = RetrievePipelineUtils.sumTotalRecallCount(myCandidates);
+                    candidates.putAll(myCandidates);
+                }
+            } catch(Exception e) { // FIXME:
+                //
+            }
+        }
+
+        return candidates;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/candidate/Candidate.java

@@ -0,0 +1,72 @@
+package com.tzld.piaoquan.recommend.framework.retrieve.candidate;
+
+import com.google.gson.Gson;
+
+public class Candidate {
+    private String candidateKey;
+    private int candidateNum;
+    private QueueName candidateQueueName;
+    private String mergeQueueName;
+    private int mergeQueueNum;
+
+    public Candidate() {
+    }
+
+    public Candidate(Candidate other) {
+        this.candidateKey = other.getCandidateKey();
+        this.candidateNum = other.getCandidateNum();
+        this.candidateQueueName = other.getCandidateQueueName();
+        this.mergeQueueName = other.getQueue();
+        this.mergeQueueNum = other.getMergeQueueNum();
+    }
+
+    public Candidate deepcopy() {
+        return new Candidate(this);
+    }
+
+    public int getMergeQueueNum() {
+        return mergeQueueNum;
+    }
+
+    public void setMergeQueueNum(int mergeQueueNum) {
+        this.mergeQueueNum = mergeQueueNum;
+    }
+
+    public String getCandidateKey() {
+        return candidateKey;
+    }
+
+    public void setCandidateKey(String candidateKey) {
+        this.candidateKey = candidateKey;
+    }
+
+    public int getCandidateNum() {
+        return candidateNum;
+    }
+
+    public void setCandidateNum(int candidateNum) {
+        this.candidateNum = candidateNum;
+    }
+
+    public String getQueue() {
+        return mergeQueueName;
+    }
+
+    public void setQueue(String queue) {
+        this.mergeQueueName = queue;
+    }
+
+    public QueueName getCandidateQueueName() {
+        return candidateQueueName;
+    }
+
+    public void setCandidateQueueName(QueueName candidateQueueName) {
+        this.candidateQueueName = candidateQueueName;
+    }
+
+    @Override
+    public String toString() {
+        return new Gson().toJson(this);
+    }
+}
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/candidate/CandidateInfo.java

@@ -0,0 +1,53 @@
+package com.tzld.piaoquan.recommend.framework.retrieve.candidate;
+
+
+import com.google.gson.Gson;
+
+
+public class CandidateInfo {
+    private String candidate_queue;     // 召回队列
+    private int position;               // 在召回队列中的位置
+    private Candidate candidate;        // 召回配置的信息
+
+    public CandidateInfo() {
+    }
+
+    public CandidateInfo(CandidateInfo other) {
+        this.candidate_queue = other.getCandidate_queue();
+        this.position = other.getPosition();
+        this.candidate = other.candidate != null ? new Candidate(other.candidate) : null;
+    }
+
+    public CandidateInfo deepcopy() {
+        return new CandidateInfo(this);
+    }
+
+    public Candidate getCandidate() {
+        return candidate;
+    }
+
+    public void setCandidate(Candidate candidate) {
+        this.candidate = candidate;
+    }
+
+    public String getCandidate_queue() {
+        return candidate_queue;
+    }
+
+    public void setCandidate_queue(String candidate_queue) {
+        this.candidate_queue = candidate_queue;
+    }
+
+    public int getPosition() {
+        return position;
+    }
+
+    public void setPosition(int position) {
+        this.position = position;
+    }
+
+    @Override
+    public String toString() {
+        return new Gson().toJson(this);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/candidate/Entry.java

@@ -0,0 +1,34 @@
+package com.tzld.piaoquan.recommend.framework.retrieve.candidate;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class Entry<T> {
+    public final T item;
+    public final Map<String, Double> scores;
+    public final Map<String, String> explanations;
+    public final String id;
+
+    public Entry(T item, String id) {
+        this.item = item;
+        this.scores = new HashMap<String, Double>();
+        this.id = id;
+        this.explanations = new HashMap<String, String>();
+    }
+
+    public Entry(Entry<T> other) {
+        this.item = other.item;
+        this.id = other.id;
+        this.scores = other.scores;
+        this.explanations = other.explanations;
+    }
+
+    public void addScore(String name, double score) {
+        scores.put(name, score);
+    }
+
+    public void addExplanation(String name, String explanation) {
+        explanations.put(name, explanation);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/candidate/IndexDescription.java

@@ -0,0 +1,137 @@
+package com.tzld.piaoquan.recommend.framework.retrieve.candidate;
+
+
+
+import com.google.common.collect.Lists;
+import com.typesafe.config.Config;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.Candidate;
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.QueueName;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+
+public class IndexDescription {
+
+    public String name;
+
+    public int recallWeight;
+
+    /**
+     * redis,knn
+     */
+    public String db;
+    public String itemType;
+    public String indexName;
+    public String keyType;
+    public String key;
+    public List<Pair<String, String>> matches;
+    public String ordering;
+
+    /**
+     * 处理类似item_cf:when=day 的情况，提取出
+     * item_cf, when->day
+     */
+    public void processIndexName() {
+        if (indexName.contains(":")) {
+            String[] indics = indexName.split(":");
+            indexName = indics[0];
+            if (indics.length > 1) {
+                matches = Lists.newArrayList();
+                for (int i = 1; i < indics.length; ++i) {
+                    String[] pair = indics[i].split("=");
+                    if (pair.length == 2) {
+                        matches.add(Pair.of(pair[0], pair[1]));
+                    } else {
+                        //log error
+                    }
+                }
+            }
+        }
+    }
+
+
+    /**
+     * 根据当前indexdesc的字段信息，生成candidate，用于后续召回
+     * @param queue
+     * @param indexDescription
+     * @return
+     */
+    public static Candidate convertToCandidate(String queue, IndexDescription indexDescription) {
+        Candidate candidate = new Candidate();
+        indexDescription.processIndexName();
+        QueueName queueName;
+
+        //knn类召回，不需要type
+        if ("knn".equalsIgnoreCase(indexDescription.keyType)) {
+            queueName = new QueueName(indexDescription.itemType, indexDescription.ordering);
+        } else {
+            queueName = new QueueName(indexDescription.itemType, indexDescription.ordering)
+                    .addMatch("type", indexDescription.indexName);
+        }
+
+
+        if (indexDescription.matches != null && !indexDescription.matches.isEmpty()) {
+            for (Map.Entry<String, String> entry : indexDescription.matches) {
+                queueName.addMatch(entry.getKey(), entry.getValue());
+            }
+        }
+
+        //添加类似tag->范冰冰 这类索引
+        if (StringUtils.isNotBlank(indexDescription.keyType)) {
+            queueName.addMatch(indexDescription.keyType, indexDescription.key);
+        }
+
+        candidate.setCandidateKey(queueName.toString());
+        candidate.setCandidateNum(indexDescription.recallWeight);
+        candidate.setMergeQueueNum(indexDescription.recallWeight);
+        candidate.setCandidateQueueName(queueName);
+        candidate.setQueue(queue);
+
+        return candidate;
+    }
+
+    public static final String RECALL_WEIGHT = "recall-weight";
+    public static final String INDEX_DESC = "index-desc";
+    public static final String DB = "db"; // source
+    public static final String ITEM_TYPE = "item-type";
+    public static final String NAME = "name";
+    public static final String KEY_TYPE = "key-type";
+    public static final String KEY = "key";
+    public static final String ORDERING = "ordering";
+    public static final String MATCHES = "matches";
+
+    public static IndexDescription parseIndexDescription(String name, Config conf) {
+        if (!conf.hasPath(RECALL_WEIGHT) || !conf.hasPath(INDEX_DESC)) {
+            // LOGGER.error("name=" + name + ",必须输入recall-weight和indexdesc");
+            return null;
+        }
+
+        Config desc = conf.getConfig(INDEX_DESC);
+        if (!desc.hasPath(DB) || !desc.hasPath(ITEM_TYPE) || !desc.hasPath(NAME) || !desc.hasPath(ORDERING)) {
+            //  LOGGER.error("name=" + name + ",必须输入recall-weight和indexdesc");
+            return null;
+        }
+
+        IndexDescription indexDescription = new IndexDescription();
+        indexDescription.name = name;
+        indexDescription.recallWeight = conf.getInt(RECALL_WEIGHT);
+        indexDescription.db = desc.getString(DB);
+        indexDescription.itemType = desc.getString(ITEM_TYPE);
+        indexDescription.indexName = desc.getString(NAME);
+        indexDescription.ordering = desc.getString(ORDERING);
+
+        if (desc.hasPath(KEY_TYPE)) {
+            indexDescription.keyType = desc.getString(KEY_TYPE);
+        }
+        if (desc.hasPath(KEY)) {
+            indexDescription.key = desc.getString(KEY);
+        }
+
+        return indexDescription;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/candidate/Queue.java

@@ -0,0 +1,53 @@
+package com.tzld.piaoquan.recommend.framework.retrieve.candidate;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * An abstract data type represents a queue of certain item.
+ */
+public class Queue<T> implements Iterable<Entry<T>> {
+    private final List<Entry<T>> entries;
+    private final String name;
+    private final String explain;
+
+    public Queue(String name) {
+        entries = new ArrayList<Entry<T>>();
+        this.name = name;
+        this.explain = null;
+    }
+
+    public Queue(String name, String explain) {
+        this.name = name;
+        this.explain = explain;
+        this.entries = new ArrayList<Entry<T>>();
+    }
+
+    public void reverse() {
+        Collections.reverse(this.entries);
+    }
+
+    public void add(Entry<T> entry) {
+        entries.add(entry);
+    }
+
+    public void addAll(Collection<Entry<T>> paramEntries) {
+        entries.addAll(paramEntries);
+    }
+
+    public List<Entry<T>> get() {
+        return entries;
+    }
+
+    public Iterator<Entry<T>> iterator() {
+        return entries.iterator();
+    }
+
+    public int size() {
+        return entries.size();
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/candidate/QueueName.java

@@ -0,0 +1,131 @@
+package com.tzld.piaoquan.recommend.framework.retrieve.candidate;
+
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.FluentIterable;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Queue names.
+ */
+public class QueueName implements Serializable, Comparable<QueueName> {
+    private static final Function<Pair<String, String>, String> MATCH_PATTERN =
+            new Function<Pair<String, String>, String>() {
+                @Override
+                public String apply(Pair<String, String> input) {
+                    return input.getLeft() + "=" + input.getRight();
+                }
+            };
+    private static final long DEFAULT_LOCAL_CACHE_TTL = 15 * 60 * 1000;
+    private final List<Pair<String, String>> matches = new ArrayList<Pair<String, String>>();
+    private final String itemType;
+    private final String ordering;
+    private final long ttl;
+
+    private String metaChannel;  // meta 渠道
+
+    public QueueName(String itemType, String ordering) {
+        this(itemType, ordering, DEFAULT_LOCAL_CACHE_TTL);
+    }
+
+    public QueueName(String itemType, String ordering, long ttl) {
+        this.itemType = itemType;
+        this.ordering = ordering;
+        this.ttl = ttl;
+    }
+
+    public static QueueName fromString(String string, long ttl) {
+        String[] parts = string.split(":");
+        String itemType = parts[0];
+        String ordering = parts[parts.length - 1].split("=")[1];
+
+        QueueName result = new QueueName(itemType, ordering, ttl);
+        for (int i = 2; i < parts.length - 1; ++i) {
+            String[] sides = parts[i].split("=");
+            result.addMatch(sides[0], sides[1]);
+        }
+        return result;
+    }
+
+    public static QueueName fromString(String string) {
+        return QueueName.fromString(string, DEFAULT_LOCAL_CACHE_TTL);
+    }
+
+    public static Pair<String, String> parseMatchPair(String match) {
+        String[] arr = match.split("=");
+        if (arr.length == 2) {
+            return Pair.of(arr[0], arr[1]);
+        }
+        return Pair.of("", "");
+    }
+
+    public String getMetaChannel() {
+        return metaChannel;
+    }
+
+    public void setMetaChannel(String metaChannel) {
+        this.metaChannel = metaChannel;
+    }
+
+    public String getItemType() {
+        return itemType;
+    }
+
+    public String getOrdering() {
+        return ordering;
+    }
+
+    public long getTtl() {
+        return ttl;
+    }
+
+    public QueueName addMatch(String key, String value) {
+        if (value == null || value.equals("")) {
+            value = "_";
+        } else {
+            value = value.replace("=", "_");
+            value = value.replace(":", "_");
+        }
+        if (key.equals("channel")) {
+            this.setMetaChannel(value);
+        }
+        matches.add(Pair.of(key, value));
+        return this;
+    }
+
+    public Iterable<String> getMatches() {
+        return FluentIterable.from(matches).transform(MATCH_PATTERN);
+    }
+
+    public long getTTL() {
+        return this.ttl;
+    }
+
+    @Override
+    public String toString() {
+        Iterable<String> matchesString = FluentIterable.from(matches)
+                .transform(MATCH_PATTERN);
+
+        return itemType + ":queue:" + Joiner.on(":").join(matchesString) + ":ordering=" + ordering;
+    }
+
+    @Override
+    public int hashCode() {
+        return toString().hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return other instanceof QueueName && ((QueueName) other).toString().equals(toString());
+    }
+
+    @Override
+    public int compareTo(QueueName other) {
+        return other.toString().compareTo(ordering.toString());
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/retrieve/candidate/QueueProvider.java

@@ -0,0 +1,23 @@
+package com.tzld.piaoquan.recommend.framework.retrieve.candidate;
+
+
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+
+public interface QueueProvider<T> {
+
+    Queue<T> load(QueueName name) throws Exception;
+
+    long dbSize();
+
+
+    String get(QueueName name) throws Exception;
+
+    Map<QueueName, Queue<T>> loads(List<QueueName> names) throws Exception;
+
+    Map<QueueName, Queue<T>> loads(List<QueueName> names, long timeout, TimeUnit timeUnit) throws Exception;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/AbstractScorer.java

@@ -0,0 +1,66 @@
+package com.tzld.piaoquan.recommend.framework.score;
+
+
+
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+
+
+public abstract class AbstractScorer {
+    public static Logger LOGGER = LoggerFactory.getLogger(AbstractScorer.class);
+    protected ScorerConfigInfo scorerConfigInfo;
+    protected ModelManager modelManager = ModelManager.getInstance();
+
+    public AbstractScorer(ScorerConfigInfo scorerConfigInfo) {
+        this.scorerConfigInfo = scorerConfigInfo;
+    }
+
+    public void loadModel() {
+    }
+
+    public boolean isEnable() {
+        return !getScorerConfigInfo().getDisableSwitch();
+    }
+
+
+    public void doLoadModel(Class<? extends Model> modelClass) {
+
+        String modelPath = scorerConfigInfo.getModelPath();
+        if (StringUtils.isNotBlank(modelPath)) {
+            try {
+                // 使用 modelPath 作为 modelName 注册
+                modelManager.registerModel(modelPath, modelPath, modelClass);
+                LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
+            } catch (ModelManager.ModelRegisterException e) {
+                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+            } catch (IOException e) {
+                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+            }
+        } else {
+            LOGGER.error("modelpath is null, for model class [{}]", modelClass);
+        }
+    }
+
+    public Model getModel() {
+        if (StringUtils.isBlank(scorerConfigInfo.getModelPath())) {
+            return null;
+        }
+        return modelManager.getModel(scorerConfigInfo.getModelPath());
+    }
+
+    public ScorerConfigInfo getScorerConfigInfo() {
+        return scorerConfigInfo;
+    }
+
+    public abstract List<RankerItem> scoring(final RecommendRequestData requestContext,
+                                             final User user,
+                                             final int requestIndex,
+                                             final List<RankerItem> rankerItems);
+}
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/LRModel.java

@@ -0,0 +1,423 @@
+package com.tzld.piaoquan.recommend.framework.score;
+
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.xiaomi.common.perfcounter.PerfCounter;
+import com.xiaomi.data.spec.platform.feeds.BaseFeature;
+import com.xiaomi.data.spec.platform.feeds.GroupedFeature;
+import com.xiaomi.data.spec.platform.feeds.LRSamples;
+
+import it.unimi.dsi.fastutil.longs.Long2FloatMap;
+import it.unimi.dsi.fastutil.longs.Long2FloatOpenHashMap;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+public class LRModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(LRModel.class);
+    private static final ExecutorService executorService = Executors.newCachedThreadPool();
+    private final int bucketBits = 10;  // power(2, 10) => 1024 个槽位
+    private List<Long2FloatMap> lrModel;
+    private Config config = ConfigFactory.load("hdfs_conf.properties");
+    private Configuration hdfsConf = new Configuration();
+
+    public LRModel() {
+        //配置不同环境的hdfs conf
+        String coreSiteFile = config.hasPath("hdfs.coreSiteFile") ? StringUtils.trim(config.getString("hdfs.coreSiteFile")) : "core-site.xml";
+        String hdfsSiteFile = config.hasPath("hdfs.hdfsSiteFile") ? StringUtils.trim(config.getString("hdfs.hdfsSiteFile")) : "hdfs-site.xml";
+        hdfsConf.addResource(coreSiteFile);
+        hdfsConf.addResource(hdfsSiteFile);
+        this.lrModel = constructModel();
+    }
+
+    public List<Long2FloatMap> getLrModel() {
+        return lrModel;
+    }
+
+    public List<Long2FloatMap> constructModel() {
+        List<Long2FloatMap> initModel = new ArrayList<Long2FloatMap>();
+        int buckets = (int) Math.pow(2, bucketBits);
+        for (int i = 0; i < buckets; i++) {
+            Long2FloatMap internalModel = new Long2FloatOpenHashMap();
+            internalModel.defaultReturnValue(0.0f);
+            initModel.add(internalModel);
+        }
+        return initModel;
+    }
+
+    public int getBucket(long featureHash) {
+        return (int) (((featureHash >> bucketBits) << bucketBits) ^ featureHash);
+    }
+
+    public void putFeature(List<Long2FloatMap> model, long featureHash, float weight) {
+        model.get(getBucket(featureHash)).put(featureHash, weight);
+    }
+
+    public float getWeight(List<Long2FloatMap> model, long featureHash) {
+        return model.get(getBucket(featureHash)).get(featureHash);
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.lrModel == null)
+            return 0;
+
+        int sum = 0;
+        for (Map<Long, Float> model : this.lrModel) {
+            sum += model.size();
+        }
+
+        return sum;
+    }
+
+    public void cleanModel() {
+        this.lrModel = null;
+    }
+
+    public Float score(LRSamples lrSamples) {
+        float sum = 0.0f;
+        for (GroupedFeature gf : lrSamples.getFeatures()) {
+            if (gf != null && gf.getFeatures() != null) {
+                for (BaseFeature fea : gf.getFeatures()) {
+                    if (fea != null) {
+                        float tmp = getWeight(this.lrModel, fea.getIdentifier());
+                        fea.setWeight(tmp);
+                        sum += tmp;
+                    }
+                }
+            }
+        }
+        float pro = (float) (1.0f / (1 + Math.exp(-sum)));
+        lrSamples.setPredict_ctr(pro);
+        return pro;
+    }
+
+    public Float getWeights(LRSamples lrSamples) {
+        float sum = 0.0f;
+        for (GroupedFeature gf : lrSamples.getFeatures()) {
+            if (gf != null && gf.getFeatures() != null) {
+                for (BaseFeature fea : gf.getFeatures()) {
+                    if (fea != null) {
+                        float tmp = getWeight(this.lrModel, fea.getIdentifier());
+                        fea.setWeight(tmp);
+                        sum += tmp;
+                    }
+                }
+            }
+        }
+        lrSamples.setWeight(sum);
+        return sum;
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        List<Long2FloatMap> model = constructModel();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+
+            putFeature(model, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
+            if (cnt++ < 10) {
+                LOGGER.debug("fea: " + items[0] + ", weight: " + items[1]);
+            }
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.lrModel = model;
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+            putFeature(model, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+
+        // counter for alarm
+        PerfCounter.setCounterCount("model.lrmodel.update.success", this.lrModel.size());
+        return true;
+    }
+
+    @Override
+    public boolean loadPartitions(final String modelPath, boolean isRegister) {
+        if (isRegister) {
+            return loadPartitionsParallel(modelPath);
+        } else {
+            return loadPartitionsSingle(modelPath);
+        }
+    }
+
+    /**
+     * 程序运行的过程中, model 太大, 为了尽可能防止full gc, 单线程加载
+     *
+     * @param modelPath
+     * @return
+     */
+
+    public boolean loadPartitionsSingle(final String modelPath) {
+        try {
+            Path path = new Path(modelPath);
+            hdfsConf.setBoolean("fs.hdfs.impl.disable.cache", true);
+
+            FileSystem fs = path.getFileSystem(hdfsConf);
+            FileStatus[] listStatus = fs.listStatus(path);
+
+            //judge null and empty
+            if (listStatus == null || listStatus.length == 0) {
+                LOGGER.error("model path is dir, but hdfs patition path is null");
+                return false;
+            }
+
+            long startTime = System.currentTimeMillis();
+
+            // 初始化model大小,直接分配到old heap
+            List<Long2FloatMap> currLrModel = constructModel();
+
+            //multi thread load hdfs news info files
+            int failedPartitionNum = 0;
+            int partitionsNum = listStatus.length;
+            for (final FileStatus file : listStatus) {
+                String absPath = String.format("%s/%s", modelPath, file.getPath().getName());
+                InputStreamReader fin = null;
+                try {
+                    Path tmpPath = new Path(absPath);
+                    FileSystem tmpFs = tmpPath.getFileSystem(hdfsConf);
+                    FSDataInputStream inputStream = tmpFs.open(tmpPath);
+                    fin = new InputStreamReader(inputStream);
+
+                    BufferedReader input = new BufferedReader(fin);
+                    String line = null;
+                    int cnt = 0;
+
+                    //first stage
+                    while ((line = input.readLine()) != null) {
+
+                        String[] items = line.split("\t");
+                        if (items.length < 2) {
+                            continue;
+                        }
+
+                        // write sync
+                        putFeature(currLrModel, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
+                        cnt++;
+                    }
+                    LOGGER.info("load model [SUCCESS] , file path [{}], load item number [{}]", absPath, cnt);
+                } catch (Exception e) {
+                    failedPartitionNum++;
+                    LOGGER.error("load model file from hdfs occur error [FAILED], path: [{}], [{}]",
+                            absPath, ExceptionUtils.getFullStackTrace(e));
+                } finally {
+                    if (fin != null) {
+                        try {
+                            fin.close();
+                        } catch (IOException e) {
+                            LOGGER.error("close [{}] fail: [{}]", absPath, ExceptionUtils.getFullStackTrace(e));
+                        }
+                    }
+                }
+            }
+
+            if (failedPartitionNum == 0) {
+                this.lrModel = currLrModel;
+                // counter for alarm
+                PerfCounter.setCounterCount("model.lrmodel.update.success", this.lrModel.size());
+
+                LOGGER.info("[end] load model data from hdfs, spend time: [{}ms] model size: [{}], " +
+                        "total partition number [{}], failed partition numbers: [{}], model path [{}]", new Object[]{
+                        (System.currentTimeMillis() - startTime), getModelSize(), partitionsNum, failedPartitionNum, modelPath});
+                return true;
+            } else {
+                LOGGER.error("load model failed parts [{}]", failedPartitionNum);
+                return false;
+            }
+        } catch (Exception e) {
+            LOGGER.error("load model partitions occur error, model path [{}], error: [{}]",
+                    modelPath, ExceptionUtils.getFullStackTrace(e));
+            return false;
+        }
+    }
+
+    /**
+     * concurrency load model from modelpath
+     * put map must be sync
+     * if partitions not 0 && load success: return true
+     * exceptions || 0 partitions || any partitions failed : return false
+     *
+     * @param modelPath
+     * @return
+     */
+    public boolean loadPartitionsParallel(final String modelPath) {
+        try {
+            Path path = new Path(modelPath);
+            FileSystem fs = path.getFileSystem(hdfsConf);
+            FileStatus[] listStatus = fs.listStatus(path);
+
+            //judge null and empty
+            if (listStatus == null || listStatus.length == 0) {
+                LOGGER.error("model path is dir, but hdfs patition path is null");
+                return false;
+            }
+
+            long startTime = System.currentTimeMillis();
+            List<Callable<Long2FloatMap>> callables = new ArrayList<Callable<Long2FloatMap>>();
+            //multi thread load hdfs news info files
+            for (final FileStatus file : listStatus) {
+                callables.add(new Callable<Long2FloatMap>() {
+                    @Override
+                    public Long2FloatMap call() {
+                        // LOGGER.debug("load model file path [{}]", file.getPath().getName());
+                        String abspath = String.format("%s/%s", modelPath, file.getPath().getName());
+                        InputStreamReader fin = null;
+                        Long2FloatMap partModel = new Long2FloatOpenHashMap();
+                        try {
+
+                            Path path = new Path(abspath);
+                            FileSystem fs = path.getFileSystem(hdfsConf);
+                            FSDataInputStream inputStream = fs.open(path);
+                            fin = new InputStreamReader(inputStream);
+
+                            BufferedReader input = new BufferedReader(fin);
+                            String line = null;
+                            int cnt = 0;
+
+                            //first stage
+                            while ((line = input.readLine()) != null) {
+
+                                String[] items = line.split("\t");
+                                if (items.length < 2) {
+                                    continue;
+                                }
+
+                                // write sync
+                                partModel.put(new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
+                                cnt++;
+                            }
+                            LOGGER.info("load model [SUCCESS] , file path [{}], load item number [{}]", abspath, cnt);
+                            return partModel;
+                        } catch (Exception e) {
+                            LOGGER.error("load model file from hdfs occur error [FAILED], path: [{}], [{}]",
+                                    abspath, ExceptionUtils.getFullStackTrace(e));
+                            return null;
+                        } finally {
+                            if (fin != null) {
+                                try {
+                                    fin.close();
+                                } catch (IOException e) {
+                                    LOGGER.error("close [{}] fail: [{}]", abspath, ExceptionUtils.getFullStackTrace(e));
+                                    return null;
+                                }
+                            }
+                            return partModel;
+                        }
+                    }
+                });
+            }
+
+            //invoke callable if failed return
+            List<Future<Long2FloatMap>> futures = null;
+            try {
+                futures = executorService.invokeAll(callables, 10, TimeUnit.MINUTES);
+            } catch (InterruptedException e) {
+                LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+            }
+
+            //wait for task complete
+            int failedPartitionNum = 0;
+            int partitionsNum = listStatus.length;
+            List<Long2FloatMap> currLrModel = constructModel();
+            for (Future<Long2FloatMap> future : futures) {
+                try {
+                    if (future.isDone() && !future.isCancelled()) {
+
+                        Long2FloatMap ret = future.get();
+                        if (ret == null) {
+                            failedPartitionNum++;
+                        }
+                        for (Map.Entry<Long, Float> entry : ret.entrySet()) {
+                            putFeature(currLrModel, entry.getKey(), entry.getValue());
+                        }
+                        ret = null; // gc
+                        continue;
+                    }
+                } catch (InterruptedException e) {
+                    failedPartitionNum++;
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    failedPartitionNum++;
+                    LOGGER.error("ExecutionException [{}]", ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+
+            long endTime = System.currentTimeMillis();
+            // check all load success
+            if (failedPartitionNum == 0) {
+                this.lrModel = currLrModel;
+                // counter for alarm
+                PerfCounter.setCounterCount("model.lrmodel.update.success", this.lrModel.size());
+
+                LOGGER.info("[end] load model data from hdfs, spend time: [{}ms] model size: [{}], " +
+                        "total partition number [{}], failed partition numbers: [{}], model path [{}]", new Object[]{
+                        (endTime - startTime), getModelSize(), partitionsNum, failedPartitionNum, modelPath});
+                return true;
+            }
+
+            return false;
+        } catch (Exception e) {
+            LOGGER.error("load model partitions occur error, model path [{}], error: [{}]",
+                    modelPath, ExceptionUtils.getFullStackTrace(e));
+            return false;
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/Model.java

@@ -0,0 +1,12 @@
+package com.tzld.piaoquan.recommend.framework.score;
+
+import java.io.InputStreamReader;
+
+
+abstract public class Model {
+    public abstract int getModelSize();
+
+    public abstract boolean loadFromStream(InputStreamReader in) throws Exception;
+
+    public abstract boolean loadPartitions(String modelPath, boolean isRegister);
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/ScorerConfig.java

@@ -0,0 +1,144 @@
+package com.tzld.piaoquan.recommend.framework.score;
+
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+
+
+public class ScorerConfig {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
+    private List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
+
+    public ScorerConfig(Config config) {
+        this.load(config);
+    }
+
+    public ScorerConfig(String configFile) {
+        this.load(configFile);
+    }
+
+    public ScorerConfig() {
+    }
+
+    public static Config mergeConfig(Config baselineConfig, List<Config> expConfigs) {
+        if (expConfigs == null || expConfigs.size() == 0) {
+            return baselineConfig;
+        }
+        Config resultConfig = expConfigs.get(0);
+        for (Config config : expConfigs) {
+            resultConfig = resultConfig.withFallback(config);
+        }
+        resultConfig = resultConfig.withFallback(baselineConfig);
+        return resultConfig;
+    }
+
+    public boolean load(String configFile) {
+        Config config = ConfigFactory.parseResources(configFile);
+        return load(config);
+    }
+
+    public boolean load(Config baselineConfig, List<Config> expConfigs) {
+        Config config = mergeConfig(baselineConfig, expConfigs);
+        return load(config);
+    }
+
+    public boolean load(Config config) {
+        Config scorerConfig = config.getConfig("scorer-config");
+
+        try {
+            loadScorers(scorerConfig);
+            int pos = 0;
+            for (ScorerConfigInfo scorerConfigInfo : configInfoList) {
+                LOGGER.debug("scorer at position [{}], priority [{}], scorer name [{}]",
+                        new Object[]{pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName()});
+            }
+            LOGGER.debug("Load scorer config success");
+        } catch (Exception e) {
+            LOGGER.error("Load scorer config failed, [{}]", ExceptionUtils.getFullStackTrace(e));
+            PerfCounter.count(PerfCounterUtils.SCORER_CONFIG_LOAD_FAIL_COUNT, 1);
+            return false;
+        }
+
+        return true;
+    }
+
+    public List<ScorerConfigInfo> getConfigInfoList() {
+        return configInfoList;
+    }
+
+
+    public String loadOptionStringConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getString(path) : null;
+    }
+
+    private Config loadOptionConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
+    }
+
+    private void loadScorers(Config config) throws Exception {
+
+        ConfigObject confObj = config.root();
+        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
+            Config conf = ((ConfigObject) it.getValue()).toConfig();
+            // parse config
+            String configName = it.getKey();
+            String scorerName = conf.getString("scorer-name");
+            int scorerPriority = 0;
+            if (conf.hasPath("scorer-priority"))
+                scorerPriority = conf.getInt("scorer-priority");
+            Boolean disableSwitch = false;
+            if (conf.hasPath("disable-switch")) {
+                disableSwitch = conf.getBoolean("disable-switch");
+            }
+            Config paramConfig = loadOptionConfig(conf, "param-config");
+            // model path
+            String modelPath = loadOptionStringConfig(conf, "model-path");
+            if (modelPath == null) {
+                modelPath = loadOptionStringConfig(conf, "default-model-path");
+                LOGGER.debug("model-path is not exists in config file, use default-model-path instead, modelPath={}", modelPath);
+            }
+            // enable queues
+            Set<String> enableQueues = new HashSet<String>();
+            if (conf.hasPath("enable-queues")) {
+                enableQueues.addAll(conf.getStringList("enable-queues"));
+            }
+            ScorerConfigInfo configInfo = new ScorerConfigInfo(configName,
+                    scorerName,
+                    scorerPriority,
+                    disableSwitch,
+                    enableQueues,
+                    modelPath,
+                    paramConfig
+            );
+            LOGGER.debug("parse scorer config info [{}]", configInfo);
+            // add to ConfigInfoList
+            addConfigByPriority(configInfoList, configInfo);
+        }
+    }
+
+    private void addConfigByPriority(List<ScorerConfigInfo> configInfoList, ScorerConfigInfo addConfigInfo) {
+
+        int pos = 0;
+        for (; pos < configInfoList.size(); pos++) {
+            if (configInfoList.get(pos).getScorerPriority() <= addConfigInfo.getScorerPriority()) {
+                break;
+            }
+        }
+
+        configInfoList.add(pos, addConfigInfo);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/ScorerConfigInfo.java

@@ -0,0 +1,79 @@
+package com.tzld.piaoquan.recommend.framework.score;
+
+
+import com.google.gson.Gson;
+import com.typesafe.config.Config;
+
+import java.util.Set;
+
+
+public class ScorerConfigInfo {
+
+    private String configName;
+    private Integer scorerPriority;
+    private Boolean disableSwitch;
+    private String scorerName;
+    private Set<String> enableQueues;
+    private String modelPath;
+    private Config paramConfig; // param config
+
+    public ScorerConfigInfo(String configName,
+                            String scorerName,
+                            Integer scorerPriority,
+                            Boolean disableSwitch,
+                            Set<String> enableQueues,
+                            String modelPath,
+                            Config paramConfig) {
+
+        this.configName = configName;
+        this.scorerName = scorerName;
+        this.scorerPriority = scorerPriority;
+        this.disableSwitch = disableSwitch;
+        this.enableQueues = enableQueues;
+        this.modelPath = modelPath;
+        this.paramConfig = paramConfig;
+    }
+
+    public Config getParamConfig() {
+        return paramConfig;
+    }
+
+    public Set<String> getEnableQueues() {
+        return enableQueues;
+    }
+
+    public void setEnableQueues(Set<String> enableQueues) {
+        this.enableQueues = enableQueues;
+    }
+
+    public Integer getScorerPriority() {
+        return scorerPriority;
+    }
+
+    public String getScorerName() {
+        return scorerName;
+    }
+
+    public String getConfigName() {
+        return configName;
+    }
+
+    public boolean isQueueEnable(String queueName) {
+        return this.enableQueues == null ||
+                this.enableQueues.isEmpty() ||
+                this.enableQueues.contains(queueName);
+    }
+
+    public String getModelPath() {
+        return modelPath;
+    }
+
+    public Boolean getDisableSwitch() {
+        return disableSwitch;
+    }
+
+    @Override
+    public String toString() {
+        return new Gson().toJson(this);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/ScorerPipeline.java

@@ -0,0 +1,131 @@
+package com.tzld.piaoquan.recommend.framework.score;
+
+
+import com.xiaomi.common.perfcounter.PerfCounter;
+import com.xiaomi.data.recommend.common.PerfCounterUtils;
+import com.xiaomi.data.recommend.common.RecommendRequestData;
+import com.xiaomi.data.recommend.common.model.RankerItem;
+import com.xiaomi.data.recommend.common.model.User;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+public class ScorerPipeline {
+    public static final int corePoolSize = 128;
+    public static final int SCORE_TIME_OUT = 400;
+    public static final Logger LOGGER = LoggerFactory.getLogger(ScorerPipeline.class);
+    public static final ExecutorService executorService = Executors.newFixedThreadPool(corePoolSize);
+
+    public List<AbstractScorer> scorers;
+
+    public ScorerPipeline(List<AbstractScorer> scorers) {
+        this.scorers = scorers;
+    }
+
+    /**
+     * scoring
+     *
+     * @param requestContext
+     * @param user
+     * @param requestIndex
+     * @param rankerItems
+     * @return
+     */
+    public List<RankerItem> scoring(final RecommendRequestData requestContext,
+                                    final User user,
+                                    final int requestIndex,
+                                    final List<RankerItem> rankerItems) {
+        // check
+        if (CollectionUtils.isEmpty(rankerItems)) {
+            return rankerItems;
+        }
+        if (CollectionUtils.isEmpty(scorers)) {
+            PerfCounter.count(PerfCounterUtils.SCORER_SCORERS_EMPTY_COUNT, 1);
+        }
+        long scoreStart = System.currentTimeMillis();
+        List<RankerItem> items = rankerItems;
+        for (final AbstractScorer scorer : scorers) {
+
+            if (!scorer.isEnable()) {
+                continue;
+            }
+
+            final int beforeSize = items.size();
+            final long startTime = System.currentTimeMillis();
+
+            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
+            String[] scorerNames = fullScorerName.split("\\.");
+            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
+
+            final List<RankerItem> scoreRankerItems = items;
+            Callable<List<RankerItem>> callable = new Callable<List<RankerItem>>() {
+                @Override
+                public List<RankerItem> call() throws Exception {
+                    return scorer.scoring(requestContext, user, requestIndex, scoreRankerItems);
+                }
+            };
+
+            // execute score use thread to protected score worst time
+            List<RankerItem> scoredItems = new ArrayList<RankerItem>();
+            try {
+                List<Future<List<RankerItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
+                for (Future<List<RankerItem>> future : futures) {
+                    try {
+                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                            scoredItems.addAll(future.get());
+                        } else {
+                            PerfCounter.count("feeds.scorer.item.fail.number" + scorerName, scoreRankerItems.size());
+                            LOGGER.error("score task is cancelled,  uid [{}] scorename [{}] fail items [{}]",
+                                    new Object[]{user.getId(), scorerName, scoreRankerItems.size()});
+                        }
+                    } catch (Exception e) {
+                        PerfCounter.count("feeds.scorer.item.fail.number" + scorerName, scoreRankerItems.size());
+                        LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
+                                new Object[]{user.getId(), scorerName, ExceptionUtils.getFullStackTrace(e)});
+                    }
+                }
+            } catch (Exception e) {
+                PerfCounter.count("feeds.scorer.item.fail.number" + scorerName, scoreRankerItems.size());
+                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
+                        new Object[]{user.getId(), scorerName, ExceptionUtils.getFullStackTrace(e)});
+            }
+
+            // 变更item
+            if (CollectionUtils.isNotEmpty(scoredItems)) {
+                items = scoredItems;
+            } else {
+                items = new ArrayList<>(items);
+            }
+
+            int position = 0;
+            for (RankerItem item : items) {
+                item.putRankerIndex(scorerName, position++);
+                item.putRankerScore(scorerName, item.getScore());
+            }
+            long spentTime = System.currentTimeMillis() - startTime;
+            PerfCounter.setHistogramValue("feeds.scorer.item.numbers." + scorerName, beforeSize);
+            PerfCounter.setHistogramValue("feeds.scorer.duration." + scorerName, spentTime);
+            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
+                    new Object[]{scorerName, spentTime, beforeSize, items.size()});
+        }
+
+        int position = 0;
+        for (RankerItem item : items) {
+            item.putRankerIndex("finalScore", position++);
+            item.putRankerScore("finalScore", item.getScore());
+        }
+        PerfCounter.setHistogramValue(PerfCounterUtils.SCORER_OVERALL_DURATION, System.currentTimeMillis() - scoreStart);
+        return items;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/ScorerUtils.java

@@ -0,0 +1,107 @@
+package com.tzld.piaoquan.recommend.framework.score;
+
+
+
+import com.typesafe.config.Config;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+
+public class ScorerUtils {
+    private static Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
+
+    /**
+     * init load model
+     *
+     * @param scorers
+     */
+    public static void initLoadModel(List<AbstractScorer> scorers) {
+        for (AbstractScorer scorer : scorers) {
+            if (scorer.isEnable()) {
+                scorer.loadModel();
+            }
+        }
+    }
+
+    public static void initLoadModel(Config config) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(config);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        initLoadModel(scorers);
+    }
+
+    public static ScorerPipeline getScorerPipeline(Config mergeConfig) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(mergeConfig);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        return new ScorerPipeline(scorers);
+    }
+
+    public static ScorerPipeline getScorerPipeline(Config baselineConfig, List<Config> configList) {
+        ScorerConfig scorerConfig = new ScorerConfig();
+        scorerConfig.load(baselineConfig, configList);
+        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
+        return new ScorerPipeline(scorers);
+    }
+
+    /**
+     * construct scorers
+     *
+     * @param scorerConfig
+     * @return
+     */
+    public static List<AbstractScorer> constructScorers(ScorerConfig scorerConfig) {
+
+        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
+        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
+            if (!configInfo.getDisableSwitch()) {
+                try {
+                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
+                            .getConstructor(ScorerConfigInfo.class)
+                            .newInstance(configInfo);
+                    scorers.add(scorer);
+                    LOGGER.debug("construct score [{}]", configInfo.getScorerName());
+                } catch (Exception e) {
+                    PerfCounter.count(PerfCounterUtils.SCORER_CONSTRUCT_FAIL_COUNT, 1);
+                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
+                }
+
+            }
+        }
+        return scorers;
+    }
+
+    /**
+     * construct scorers
+     *
+     * @param scorerConfig
+     * @return
+     */
+    public static List<AbstractScorer> constructQueueScorers(ScorerConfig scorerConfig, String queueName) {
+
+        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
+        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
+            if (!configInfo.getDisableSwitch() && configInfo.isQueueEnable(queueName)) {
+                try {
+                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
+                            .getConstructor(ScorerConfigInfo.class)
+                            .newInstance(configInfo);
+                    scorers.add(scorer);
+                    LOGGER.debug("construct queue scorer [{}] [{}]", queueName, configInfo.getScorerName());
+                } catch (Exception e) {
+                    PerfCounter.count(PerfCounterUtils.SCORER_CONSTRUCT_FAIL_COUNT, 1);
+                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
+                }
+
+            }
+        }
+        return scorers;
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/feature/BytesGroup.java

@@ -0,0 +1,36 @@
+package com.tzld.piaoquan.recommend.framework.score.feature;
+
+
+
+public class BytesGroup {
+    private int id;
+    private String name;
+    private byte[] nameBytes;
+    private byte[] buffer;
+
+    public BytesGroup(int id, String name, byte[] nameBytes) {
+        this.id = id;
+        this.name = name;
+        this.nameBytes = nameBytes;
+    }
+
+    public int getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public byte[] getNameBytes() {
+        return nameBytes;
+    }
+
+    public byte[] getBuffer() {
+        return buffer;
+    }
+
+    public void setBuffer(byte[] buffer) {
+        this.buffer = buffer;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/feature/BytesUtils.java

@@ -0,0 +1,201 @@
+package com.tzld.piaoquan.recommend.framework.score.feature;
+
+
+import com.xiaomi.data.spec.platform.feeds.BaseFeature;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Extract features from user, item & context info. Returns 64-bit murmurhash of feature string as results.
+ */
+public class BytesUtils {
+    private static final byte[] SEPARATOR = "_".getBytes();
+    private static final byte[] FEATURE_SEPARATOR = "#".getBytes();
+
+    private static final int MAX_FEATURE_BYTES_LENGTH = 512;
+    private static final long SEED = 11L;
+
+    private BytesGroup[] groups;
+    private final int debugLevel;
+
+    /**
+     * 一个种特殊的List,在尝试写入null的时候回默默地扔掉.
+     * @param <E> List的元素类型.
+     */
+    public static class NullRejectingArrayList<E> extends ArrayList<E> {
+        public NullRejectingArrayList(int capacity) {
+            super(capacity);
+        }
+
+        public NullRejectingArrayList() {
+            super();
+        }
+
+        @Override
+        public boolean add(E e) {
+            return e != null && super.add(e);
+        }
+    }
+
+    public BytesUtils(BytesGroup[] groups, FeatureUsage usage, int debugLevel) {
+        this.groups = groups;
+        this.debugLevel = debugLevel;
+        for (BytesGroup g : groups) {
+            byte[] buffer = prepareBuffer(g.getName(), g.getNameBytes(), usage);
+            groups[g.getId()].setBuffer(buffer);
+        }
+    }
+
+    public byte[] prepareBuffer(String name, byte[] nameBytes, FeatureUsage usage) {
+        if (usage.disable(name)) {
+            return null;
+        }
+        byte[] buffer = new byte[MAX_FEATURE_BYTES_LENGTH];
+        System.arraycopy(nameBytes, 0, buffer, 0, nameBytes.length);
+        System.arraycopy(FEATURE_SEPARATOR, 0, buffer, nameBytes.length, 1);
+        return buffer;
+    }
+
+    public BaseFeature baseFea(byte[] buffer, int length) {
+        long hash = FeatureHash.MurmurHash64(buffer, 0, length, SEED);
+        String fea = new String(buffer, 0, length);
+        BaseFeature tmp = new BaseFeature();
+        tmp.setIdentifier(hash);
+        if (debugLevel > 0) {
+            tmp.setFea(fea);
+        }
+        return tmp;
+    }
+
+    public BaseFeature makeFea(int id, byte[] value) {
+        byte[] buffer = groups[id].getBuffer();
+        if (buffer == null || value == null) {
+            return null;
+        }
+
+        final int nameLength = groups[id].getNameBytes().length + 1;
+        final int length = nameLength + value.length;
+        System.arraycopy(value, 0, buffer, nameLength, value.length);
+
+        return baseFea(buffer, length);
+    }
+
+    public BaseFeature makeFea(int id, final byte[] p1, final byte[] p2) {
+        byte[] buffer = groups[id].getBuffer();
+        if (buffer == null || p1 == null || p2 == null) {
+            return null;
+        }
+
+        final int nameLength = groups[id].getNameBytes().length + 1;
+        final int length = nameLength + p1.length + 1 + p2.length;
+
+        System.arraycopy(p1, 0, buffer, nameLength, p1.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length, 1);
+        System.arraycopy(p2, 0, buffer, nameLength + p1.length + 1, p2.length);
+
+        return baseFea(buffer, length);
+    }
+
+    public BaseFeature makeFea(int id, final byte[] p1, final byte[] p2, final byte[] p3) {
+        byte[] buffer = groups[id].getBuffer();
+        if (buffer == null || p1 == null || p2 == null || p3 == null) {
+            return null;
+        }
+
+        final int nameLength = groups[id].getNameBytes().length + 1;
+        final int length = nameLength + p1.length + 1 + p2.length + 1 + p3.length;
+        System.arraycopy(p1, 0, buffer, nameLength, p1.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length, 1);
+        System.arraycopy(p2, 0, buffer, nameLength + p1.length + 1, p2.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length + 1 + p2.length, 1);
+        System.arraycopy(p3, 0, buffer, nameLength + p1.length + 1 + p2.length + 1, p3.length);
+
+        return baseFea(buffer, length);
+    }
+
+    public BaseFeature makeFea(int id, final byte[] p1, final byte[] p2, final byte[] p3, final byte[] p4) {
+        byte[] buffer = groups[id].getBuffer();
+        if (buffer == null || p1 == null || p2 == null || p3 == null || p4 == null) {
+            return null;
+        }
+
+        final int nameLength = groups[id].getNameBytes().length + 1;
+        final int length = nameLength + p1.length + 1 + p2.length + 1 + p3.length + 1 + p4.length;
+        System.arraycopy(p1, 0, buffer, nameLength, p1.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length, 1);
+        System.arraycopy(p2, 0, buffer, nameLength + p1.length + 1, p2.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length + 1 + p2.length, 1);
+        System.arraycopy(p3, 0, buffer, nameLength + p1.length + 1 + p2.length + 1, p3.length);
+        System.arraycopy(SEPARATOR, 0, buffer, nameLength + p1.length + 1 + p2.length + 1 + p3.length, 1);
+        System.arraycopy(p4, 0, buffer, nameLength + p1.length + 1 + p2.length + 1 + p3.length + 1, p4.length);
+
+        return baseFea(buffer, length);
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[][] list) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(list.length);
+        for (byte[] t: list) {
+            result.add(makeFea(id, t));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[][] left, byte[] right) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length);
+        for (byte[] l: left) {
+            result.add(makeFea(id, l, right));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[][] left, byte[] right1, byte[] right2) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length);
+        for (byte[] l: left) {
+            result.add(makeFea(id, l, right1, right2));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[][] left, byte[] right1, byte[] right2, byte[] right3) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length);
+        for (byte[] l: left) {
+            result.add(makeFea(id, l, right1, right2, right3));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[] left, byte[][] right) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(right.length);
+        for (byte[] r : right) {
+            result.add(makeFea(id, left, r));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[] left1, byte[] left2, byte[][] right) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(right.length);
+        for (byte[] r : right) {
+            result.add(makeFea(id, left1, left2, r));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[] left1, byte[] left2, byte[] left3, byte[][] right) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(right.length);
+        for (byte[] r : right) {
+            result.add(makeFea(id, left1, left2, left3, r));
+        }
+        return result;
+    }
+
+    public List<BaseFeature> makeFea(int id, byte[][] left, byte[][] right) {
+        List<BaseFeature> result = new NullRejectingArrayList<BaseFeature>(left.length * right.length);
+        for (byte[] l: left) {
+            for (byte[] r: right) {
+                result.add(makeFea(id, l, r));
+            }
+        }
+        return result;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/feature/FeatureHash.java

@@ -0,0 +1,230 @@
+package com.tzld.piaoquan.recommend.framework.score.feature;
+
+
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+
+public class FeatureHash {
+    public static Charset CharSetUTF8 = Charset.forName("UTF-8");
+
+    public static long getUInt32(byte a, byte b, byte c, byte d) {
+        return (d << 24 | (c & 0xFF) << 16 | (b & 0xFF) << 8 | (a & 0xFF));
+    }
+
+    public static long hash64(byte[] data) {
+        return MurmurHash64A(ByteBuffer.wrap(data), 0, data.length, 11L);
+    }
+
+    public static long MurmurHash64A(ByteBuffer buffer, int from, int len, long seed) {
+        final long m = 0xc6a4a7935bd1e995L;
+        final int r = 47;
+
+        long h = (seed) ^ (len * m);
+        int longLength = len / 8;
+
+        for (int i = 0; i < longLength; ++i) {
+            final int bytePos = from + i * 8;
+            long k = buffer.getLong(bytePos);
+
+            k *= m;
+            k ^= k >> r;
+            k *= m;
+            h ^= k;
+            h *= m;
+        }
+
+        final int remainingPos = len & ~7;
+        switch (len % 8) {
+            case 7: h ^= (long)(buffer.get(remainingPos + 6) & 0xFF) << 48;
+            case 6: h ^= (long)(buffer.get(remainingPos + 5) & 0xFF) << 40;
+            case 5: h ^= (long)(buffer.get(remainingPos + 4) & 0xFF) << 32;
+            case 4: h ^= (long)(buffer.get(remainingPos + 3) & 0xFF) << 24;
+            case 3: h ^= (long)(buffer.get(remainingPos + 2) & 0xFF) << 16;
+            case 2: h ^= (long)(buffer.get(remainingPos + 1) & 0xFF) << 8;
+            case 1:
+                h ^= (long)(buffer.get(remainingPos) & 0xFF);
+                h *= m;
+        }
+
+        h ^= h >>> r;
+        h *= m;
+        h ^= h >>> r;
+        return h;
+    }
+
+    public static long MurmurHash32(byte data[], int len, long seed) {
+        long m = 0x5bd1e995L;
+        int r = 24;
+
+        long h = seed ^ len;
+
+        int offset = 0;
+        while (len >= 4) {
+            long k = getUInt32(data[offset], data[offset + 1], data[offset + 2], data[offset + 3]);
+
+            k *= m;
+            k &= 0xFFFFFFFFL;
+            k ^= k >> r;
+            k *= m;
+            k &= 0xFFFFFFFFL;
+
+            h *= m;
+            h &= 0xFFFFFFFFL;
+            h ^= k;
+
+            offset += 4;
+            len -= 4;
+        }
+
+        // Handle the last few bytes of the input array
+        switch (len) {
+            case 3: h ^= data[offset + 2] << 16;
+            case 2: h ^= data[offset + 1] << 8;
+            case 1: h ^= data[offset];
+                h *= m;
+                h &= 0xFFFFFFFFL;
+        } ;
+
+        // Do a few final mixes of the hash to ensure the last few
+        // bytes are well-incorporated.
+
+        h ^= h >> 13;
+        h *= m;
+        h &= 0xFFFFFFFFL;
+        h ^= h >> 15;
+
+        return h;
+    }
+
+    // 64-bit hash for 32-bit platforms
+    public static long MurmurHash64(byte[] buffer, int start, int len, long seed) {
+        final long m = 0x5bd1e995L;
+        final int r = 24;
+        final int original = len;
+
+        long h1 = (seed) ^ len;
+        long h2 = (seed >> 32);
+
+        int offset = start;
+        while (len >= 8) {
+            long k1 = getUInt32(buffer[offset], buffer[offset + 1], buffer[offset + 2], buffer[offset + 3]);
+            // long k1 = buffer.getInt(offset);
+
+            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
+            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
+            offset += 4;
+
+            long k2 = getUInt32(buffer[offset], buffer[offset + 1], buffer[offset + 2], buffer[offset + 3]);
+            // long k2 = buffer.getInt(offset);
+            k2 *= m; k2 &= 0xFFFFFFFFL; k2 ^= k2 >> r; k2 *= m; k2 &= 0xFFFFFFFFL;
+            h2 *= m; h2 &= 0xFFFFFFFFL; h2 ^= k2;
+
+            offset += 4;
+            len -= 8;
+        }
+
+        if (len >= 4) {
+            long k1 = getUInt32(buffer[offset], buffer[offset + 1], buffer[offset + 2], buffer[offset + 3]);
+            // long k1 = buffer.getInt(offset);
+            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
+            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
+            offset += 4;
+            len -= 4;
+        }
+
+        switch (len) {
+            case 3: h2 ^= (buffer[offset + 2] & 0xFF) << 16;
+            case 2: h2 ^= (buffer[offset + 1] & 0xFF) << 8;
+            case 1: h2 ^= (buffer[offset] & 0xFF);
+                h2 *= m;
+                h2 &= 0xFFFFFFFFL;
+        } ;
+
+        h1 ^= h2 >> 18;
+        h1 *= m; h1 &= 0xFFFFFFFFL;
+        h2 ^= h1 >> 22;
+        h2 *= m; h2 &= 0xFFFFFFFFL;
+        h1 ^= h2 >> 17;
+        h1 *= m; h1 &= 0xFFFFFFFFL;
+        h2 ^= h1 >> 19;
+        h2 *= m; h2 &= 0xFFFFFFFFL;
+
+        /*BigInteger ans = BigInteger.valueOf(h1).shiftLeft(32).or(BigInteger.valueOf(h2));
+        return ans.longValue();*/
+        //System.err.println("feature: " + new String(buffer, 0, original) + " length: " + original + " hash: " + (h1 << 32 | h2) + " daze");
+        return h1 << 32 | h2;
+    }
+
+    // 64-bit hash for 32-bit platforms
+    public static BigInteger MurmurHash64(byte data[], int len, long seed) {
+        long m = 0x5bd1e995L;
+        int r = 24;
+
+        long h1 = (seed) ^ len;
+        long h2 = (seed >> 32);
+
+        int offset = 0;
+        while (len >= 8) {
+            long k1 = getUInt32(data[offset], data[offset + 1], data[offset + 2], data[offset + 3]);
+            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
+            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
+
+            long k2 = getUInt32(data[offset + 4], data[offset + 5], data[offset + 6], data[offset + 7]);
+            k2 *= m; k2 &= 0xFFFFFFFFL; k2 ^= k2 >> r; k2 *= m; k2 &= 0xFFFFFFFFL;
+            h2 *= m; h2 &= 0xFFFFFFFFL; h2 ^= k2;
+
+            offset += 8;
+            len -= 8;
+        }
+
+        if (len >= 4) {
+            long k1 = getUInt32(data[offset], data[offset + 1], data[offset + 2], data[offset + 3]);
+            k1 *= m; k1 &= 0xFFFFFFFFL; k1 ^= k1 >> r; k1 *= m; k1 &= 0xFFFFFFFFL;
+            h1 *= m; h1 &= 0xFFFFFFFFL; h1 ^= k1;
+            offset += 4;
+            len -= 4;
+        }
+
+        switch (len) {
+            case 3: h2 ^= (data[offset + 2] & 0xFF) << 16;
+            case 2: h2 ^= (data[offset + 1] & 0xFF) << 8;
+            case 1: h2 ^= (data[offset] & 0xFF);
+                h2 *= m;
+                h2 &= 0xFFFFFFFFL;
+        } ;
+
+        h1 ^= h2 >> 18;
+        h1 *= m; h1 &= 0xFFFFFFFFL;
+        h2 ^= h1 >> 22;
+        h2 *= m; h2 &= 0xFFFFFFFFL;
+        h1 ^= h2 >> 17;
+        h1 *= m; h1 &= 0xFFFFFFFFL;
+        h2 ^= h1 >> 19;
+        h2 *= m; h2 &= 0xFFFFFFFFL;
+
+        BigInteger ans = BigInteger.valueOf(h1).shiftLeft(32).or(BigInteger.valueOf(h2));
+        return ans;
+    }
+
+    public static String hash(String input) {
+        byte[] tt = input.getBytes(CharSetUTF8);
+        return MurmurHash64(tt, tt.length, 11L).toString();
+    }
+
+    public static Long hashToLong(String input) {
+        byte[] tt = input.getBytes(CharSetUTF8);
+        return MurmurHash64(tt, tt.length, 11L).longValue();
+    }
+
+    /** the constant 2^64 */
+    private static final BigInteger TWO_64 = BigInteger.ONE.shiftLeft(64);
+
+    public static String asUnsignedLongString(long l) {
+        BigInteger b = BigInteger.valueOf(l);
+        if (b.signum() < 0) {
+            b = b.add(TWO_64);
+        }
+        return b.toString();
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/score/feature/FeatureUsage.java

@@ -0,0 +1,44 @@
+package com.tzld.piaoquan.recommend.framework.score.feature;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.Set;
+
+public class FeatureUsage implements Serializable {
+    private final Logger LOGGER = LoggerFactory.getLogger(FeatureUsage.class);
+    private Set<String> hash;
+
+    public FeatureUsage() {
+        hash = new HashSet<String>();
+    }
+
+    public FeatureUsage(String confName) {
+        // note that these fields are NOT lazy, because if we're going to get any exceptions, we want to get them on startup.
+        Config config = ConfigFactory.load(confName);
+        config.checkValid(config, "features");
+        ConfigObject conf = config.getConfig("features").root();
+        LOGGER.info("create_feature_usage: " + confName);
+        hash = new HashSet<String>();
+        for (ConfigObject.Entry<String, ConfigValue> it : conf.entrySet()) {
+            String key = it.getKey();
+            Config cf = ((ConfigObject) it.getValue()).toConfig();
+            LOGGER.info("key: " + key + ", value: " + cf.toString());
+            boolean disable = cf.getBoolean("disable");
+            if (disable) {
+                hash.add(key);
+            }
+        }
+    }
+
+    public boolean disable(String fea) {
+        return hash.contains(fea);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/userattention/AbstractUserAttentionExtractor.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.recommend.framework.userattention;
+
+
+
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+
+
+/**
+ * @author sunmingze
+ * @date 2023/11/27
+ */
+
+public abstract class AbstractUserAttentionExtractor {
+    protected UserAttentionExtractorConfig extractorConfig;
+
+    public AbstractUserAttentionExtractor(UserAttentionExtractorConfig extractorConfig) {
+        this.extractorConfig = extractorConfig;
+    }
+
+    public String getExtractorName() {
+        String[] arr = this.extractorConfig.getName().split(".");
+        return arr.length > 0 ? arr[arr.length - 1] : "UnKnownExtractor";
+    }
+
+    public UserAttentionExtractorConfig getExtractorConfig() {
+        return extractorConfig;
+    }
+
+    public boolean isEnable() {
+        return !this.extractorConfig.getDisableSwitch();
+    }
+
+    public abstract void extractAttention(final RecommendRequest requestData, final User user);
+
+    public void loadModel() {
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/userattention/UserAttentionExtractorConfig.java

@@ -0,0 +1,49 @@
+package com.tzld.piaoquan.recommend.framework.userattention;
+import com.typesafe.config.Config;
+
+public class UserAttentionExtractorConfig {
+    private final String name;  // attention extractor name
+    private final Integer priority;
+    private final Boolean disableSwitch;
+    private final String modelPath;
+    private final Config paramConfig; // param config
+
+    public UserAttentionExtractorConfig(String name,
+                                    Config paramConfig,
+                                    Integer priority,
+                                    Boolean disableSwitch,
+                                    String modelPath) {
+        this.name = name;
+        this.paramConfig = paramConfig;
+        this.priority = priority;
+        this.modelPath = modelPath;
+        this.disableSwitch = disableSwitch;
+    }
+
+    public Boolean getDisableSwitch() {
+        return disableSwitch;
+    }
+
+    public Integer getPriority() {
+        return priority;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Config getParamConfig() {
+        return paramConfig;
+    }
+
+    @Override
+    public String toString() {
+        return "AttentionExtractorConfig{" +
+                "name='" + name + '\'' +
+                ", priority=" + priority +
+                ", disableSwitch=" + disableSwitch +
+                ", modelPath='" + modelPath + '\'' +
+                ", paramConfig=" + paramConfig +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/userattention/UserAttentionExtractorPipeline.java

@@ -0,0 +1,41 @@
+package com.tzld.piaoquan.recommend.framework.userattention;
+
+
+
+import com.tzld.piaoquan.recommend.framework.common.user.User;
+import com.tzld.piaoquan.recommend.framework.common.user.UserAttention;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class UserAttentionExtractorPipeline {
+    private List<AbstractUserAttentionExtractor> extractorList = new ArrayList<AbstractUserAttentionExtractor>();
+
+    public UserAttentionExtractorPipeline(List<AbstractUserAttentionExtractor> extractorList) {
+        this.extractorList = extractorList;
+    }
+
+    public void extractAttention(final RecommendRequest requestData, final User user) {
+        if (user.getUserAttention() == null) {
+            UserAttention userAttention = new UserAttention();
+            user.setUserAttention(userAttention);
+        }
+
+        long startTime = System.currentTimeMillis();
+
+        for (AbstractUserAttentionExtractor extractor : extractorList) {
+            try {
+                if (extractor.isEnable()) {
+                    long currStartTime = System.currentTimeMillis();
+                    extractor.extractAttention(requestData, user);
+                    //
+                }
+            } catch (Exception e) {
+                //
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/userattention/UserAttentionExtractorUtils.java

@@ -0,0 +1,148 @@
+package com.tzld.piaoquan.recommend.framework.userattention;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Copyright (c) 2018 XiaoMi Inc. All Rights Reserved.
+ * Authors: Liu Jianquan <liujianquan@xiaomi.com>
+ */
+
+public class UserAttentionExtractorUtils {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(UserAttentionExtractorUtils.class);
+
+    public static void initLoadModel(Config config) {
+        initLoadModel(constructExtractorList(constructExtractorConfigList(config)));
+    }
+
+    public static UserAttentionExtractorPipeline getAttentionPipeline(Config baselineConfig, List<Config> configList) {
+        return new UserAttentionExtractorPipeline(constructExtractorList(constructExtractorConfigList(baselineConfig, configList)));
+    }
+
+    public static UserAttentionExtractorPipeline getAttentionPipeline(Config mergeConfig) {
+        return new UserAttentionExtractorPipeline(constructExtractorList(constructExtractorConfigList(mergeConfig)));
+    }
+
+    public static List<UserAttentionExtractorConfig> constructExtractorConfigList(Config baseConfig, List<Config> expConfigList) {
+        Config mergedConfig = mergeConfig(baseConfig, expConfigList);
+        return constructExtractorConfigList(mergedConfig);
+    }
+
+    public static List<UserAttentionExtractorConfig> constructExtractorConfigList(Config config) {
+        long startTime = System.currentTimeMillis();
+        List<UserAttentionExtractorConfig> configList = new ArrayList<UserAttentionExtractorConfig>();
+        try {
+            Config mergedConfig = config.getConfig("attention-extractor-config");
+            for (ConfigObject.Entry<String, ConfigValue> it : mergedConfig.root().entrySet()) {
+                LOGGER.debug("deal with base config, attention key: [{}]", it.getKey());
+                Config entryConf = ((ConfigObject) it.getValue()).toConfig();
+                UserAttentionExtractorConfig extractorConfig = parseExtractorConfig(entryConf);
+                addConfigByPriority(configList, extractorConfig);
+            }
+            // debug log
+            if (LOGGER.isDebugEnabled()) {
+                for (UserAttentionExtractorConfig extractorConfig : configList) {
+                    LOGGER.debug("attention extractor config [{}] [{}]", extractorConfig.getName(), extractorConfig.toString());
+                }
+            }
+        } catch (Exception e) {
+            LOGGER.error("construct extractor config {} failed {}", config.toString(), ExceptionUtils.getFullStackTrace(e));
+        }
+        return configList;
+    }
+
+    public static Config getAttentionMergeConfig(Config baseConfig, List<Config> expConfigList) {
+        return mergeConfig(baseConfig, expConfigList);
+    }
+
+    private static List<AbstractUserAttentionExtractor> constructExtractorList(List<UserAttentionExtractorConfig> configList) {
+        long startTime = System.currentTimeMillis();
+        List<AbstractUserAttentionExtractor> extractorList = new ArrayList<AbstractUserAttentionExtractor>();
+        for (UserAttentionExtractorConfig config : configList) {
+            try {
+                AbstractUserAttentionExtractor extractor = (AbstractUserAttentionExtractor) Class.forName(config.getName())
+                        .getConstructor(UserAttentionExtractorConfig.class)
+                        .newInstance(config);
+                extractorList.add(extractor);
+            } catch (Exception e) {
+                LOGGER.error("instance extractor {} failed {}", config.getName(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        return extractorList;
+    }
+
+    private static String loadOptionStringConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getString(path) : null;
+    }
+
+    private static Config loadOptionConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
+    }
+
+    private static int loadOptionIntConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getInt(path) : 0;
+    }
+
+    private static boolean loadOptionBooleanConfig(Config config, String path) {
+        return config.hasPath(path) ? config.getBoolean(path) : false;
+    }
+
+    private static UserAttentionExtractorConfig parseExtractorConfig(Config extractorConfig) {
+        String attentionExtractorName = extractorConfig.getString("name");
+        Integer priority = loadOptionIntConfig(extractorConfig, "attention-priority");
+        Boolean disableSwitch = loadOptionBooleanConfig(extractorConfig, "disable-switch");
+        Config paramConfig = loadOptionConfig(extractorConfig, "param-config");
+        String modelPath = loadOptionStringConfig(extractorConfig, "model-path");
+        return new UserAttentionExtractorConfig(attentionExtractorName,
+                paramConfig,
+                priority,
+                disableSwitch,
+                modelPath);
+    }
+
+    private static Config mergeConfig(Config baselineConfig, List<Config> expConfigs) {
+        if (expConfigs == null || expConfigs.size() == 0) {
+            return baselineConfig;
+        }
+        Config resultConfig = expConfigs.get(0);
+        for (Config config : expConfigs) {
+            resultConfig = resultConfig.withFallback(config);
+        }
+        resultConfig = resultConfig.withFallback(baselineConfig);
+        return resultConfig;
+    }
+
+    /**
+     * init load model
+     *
+     * @param attentionExtractors
+     */
+    private static void initLoadModel(List<AbstractUserAttentionExtractor> attentionExtractors) {
+        for (AbstractUserAttentionExtractor attentionExtractor : attentionExtractors) {
+            if (attentionExtractor.isEnable())
+                attentionExtractor.loadModel();
+        }
+    }
+
+    private static void addConfigByPriority(List<UserAttentionExtractorConfig> configInfoList, UserAttentionExtractorConfig extractorConfig) {
+
+        int pos = 0;
+        for (; pos < configInfoList.size(); pos++) {
+            if (configInfoList.get(pos).getPriority() <= extractorConfig.getPriority()) {
+                break;
+            }
+        }
+        configInfoList.add(pos, extractorConfig);
+    }
+}
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/utils/CandidateQueueManager.java

@@ -0,0 +1,47 @@
+package com.tzld.piaoquan.recommend.framework.utils;
+
+
+import com.tzld.piaoquan.recommend.framework.retrieve.candidate.QueueName;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Queue;
+
+
+public class CandidateQueueManager {
+    private static final Logger LOGGER = LoggerFactory.getLogger(CandidateQueueManager.class);
+
+    public static final String ORDER_PUBLISH_TIME = "publishtime";
+
+    public static void addVersion(QueueName original, RecommendRequest requestData) {
+        String version = "10000";
+    }
+
+    public static String getItemType(RecommendRequest requestData) {
+        return "";
+    }
+
+    // category
+    public static QueueName getCategoryNewQueue(RecommendRequest requestData, String category) {
+        QueueName queueName = new QueueName(getItemType(requestData), "publishtime")
+                .addMatch("type", "news_category")
+                .addMatch("category", category);
+        return queueName;
+    }
+
+
+    // subcategory
+    public static QueueName getSubCategoryCtrQueue(RecommendRequest requestData, String subCategory) {
+        QueueName queueName = new QueueName(getItemType(requestData), "ctr")
+                .addMatch("type", "news_subcategory")
+                .addMatch("subcategory", subCategory);
+        return queueName;
+    }
+
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/utils/ConfigUtils.java

@@ -0,0 +1,68 @@
+package com.tzld.piaoquan.recommend.framework.utils;
+
+
+
+import com.typesafe.config.Config;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author sunmingze
+ * @date 2019/8/30
+ */
+public class ConfigUtils {
+
+    public static Config mergeConfig(Config baselineConfig, List<Config> expConfigs) {
+        if (expConfigs == null || expConfigs.size() == 0) {
+            return baselineConfig;
+        }
+        Config resultConfig = expConfigs.get(0);
+        for (Config config : expConfigs) {
+            resultConfig = resultConfig.withFallback(config);
+        }
+        resultConfig = resultConfig.withFallback(baselineConfig);
+        return resultConfig;
+    }
+
+    public static void addConfig(int expId, Map<Integer, Config> map, List<Config> expConfigList) {
+        if (expId != 0 && map.containsKey(expId)) {
+            expConfigList.add(map.get(expId));
+        }
+    }
+
+    public static double loadOrDefault(Config config, String path, double defaultValue) {
+        if (config.hasPath(path)) {
+            return config.getInt(path);
+        }
+        return defaultValue;
+    }
+
+    public static int loadOrDefault(Config config, String path, int defaultValue) {
+        if (config.hasPath(path)) {
+            return config.getInt(path);
+        }
+        return defaultValue;
+    }
+
+    public static boolean loadOrDefault(Config config, String path, boolean defaultValue) {
+        if (config.hasPath(path)) {
+            return config.getBoolean(path);
+        }
+        return defaultValue;
+    }
+
+    public static String loadOrDefault(Config config, String path, String defaultValue) {
+        if (config.hasPath(path)) {
+            return config.getString(path);
+        }
+        return defaultValue;
+    }
+
+    public static Config loadOrDefault(Config config, String path, Config defaultValue) {
+        if (config.hasPath(path)) {
+            return config.getConfig(path);
+        }
+        return defaultValue;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/utils/Configuration.java

@@ -0,0 +1,32 @@
+package com.tzld.piaoquan.recommend.framework.utils;
+
+import com.typesafe.config.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.*;
+
+
+
+public class Configuration {
+    private static Config config = null;
+    private static Boolean isRegisteConf = false;
+    private static String registeServiceName = null;
+    private static Config reference = null;
+
+    private static Map<String, Config> configMap = null;
+
+    public static Config getConfig() {
+        return Configuration.config;
+    }
+
+    public static Config getConfigByFileName(String fileName, Config defalt) {
+        if (configMap.containsKey(fileName)) {
+            return configMap.get(fileName);
+        }
+        return defalt;
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/utils/Constants.java

@@ -0,0 +1,261 @@
+package com.tzld.piaoquan.recommend.framework.utils;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Constants {
+
+    public static final String PAGE_CITY_ID = "pageCityId";
+    public static final String LOC_CITY_ID = "locCityId";
+
+    public static final String LONGITUDE = "longitude";
+    public static final String LATITUDE = "latitude";
+    public static final String DEAL_ID = "dealId";
+
+    public static final String FIRST_CATE_ID = "firstCateId";
+    public static final String SCEOND_CATE_ID = "secondCateId";
+    public static final String THIRD_CATE_ID = "thirdCateId";
+    public static final String ORDER_ID = "orderId";
+    public static final String MONEY = "money";
+    public static final String SESSION = "session";
+
+    public static final String TRACE_ID = "traceid";
+
+    public static final String STRATEGY = "strategy";
+    public static final String GLOBAL_ID = "globalId";
+    public static final String USERID = "userid";
+    public static final String UUID = "uuid";
+    public static final String TRIGGER_TYPE = "triggerType";
+    public static final String TRIGGER_SOURCE = "triggerSource";
+    public static final String ABTEST_INFO = "abtestInfo";
+    public static final String ABTEST_INFO_JSON = "abtestInfoJson";
+    public static final String CONTEXT_INFO = "contextInfo";
+
+    public static final String USER_SESSION_MAP = "userSessionMap";
+    public static final String EXTRA = "extra";
+    public static final String CHANNEL = "channel";
+
+    public static final String TYPE = "type";
+    public static final String SOURCE = "source";
+    public static final String ENV = "env";
+
+
+
+    /*
+     * cat 等打点常量
+     */
+
+    /**
+     * 从bates获取用户行为历史数据的返回错误打点key
+     */
+    public static final String BAYES_UHR_ERROR = "BAYES_UHR_ERROR";
+
+
+    /**
+     * candidate indexQueue keys
+     * 解析配置文件格式
+     */
+    public static final String RECALL_WEIGHT = "recall-weight";
+    public static final String INDEX_DESC = "index-desc";
+    public static final String ITEM_TYPE = "item-type";
+    /**
+     * templateNAME
+     */
+    public static final String INDEX_NAME = "index-name";
+
+    /**
+     * source
+     */
+    public static final String QUEUE_NAME = "queue-name";
+    public static final String DB = "db";
+    public static final String LBS_TYPE = "lbs-type";
+
+
+    public static final String TEMPLATE_ID = "template-id";
+    public static final String ORDERING = "ordering";
+
+
+    public static final String KEY_TYPE = "key-type";
+    public static final String KEY_TYPE_NAME = "key-type-name";
+    public static final String KEY = "key";
+    public static final String URL = "url";
+    public static final String LBS_ID = "lbs-id";
+
+
+    public static final String queue = "lbs-type";
+    public static final String MATERIAL_ID = "material-id";
+    public static final String MATCHES = "matches";
+    public static String KEY_TYPE_VALUE = "key-type-value";
+
+    /**
+     * item type
+     */
+    public static final String ITEM_TYPE_POI = "poi";
+    public static final String ITEM_TYPE_SGPOI = "sg_poi";
+    public static final String ITEM_TYPE_MCPOI = "mc_poi";
+    public static final String ITEM_TYPE_ATT = "attention";
+    public static final String ITEM_TYPE_PRICE = "price";
+    public static final String ITEM_TYPE_DEAL = "deal";
+    public static final String ITEM_TYPE_SKU = "sku";
+    public static final String ITEM_TYPE_WAIMAI_POI = "poi_waimai";
+
+    public static final String WAIMAI_STR = "waimai";
+    public static final String POI_STR = "poi";
+    public static final String DEAL_STR = "deal";
+
+    /**
+     * indexQueue setCandidateKey generation
+     * triggerpoi_{type}_{lbs_type}_{lbs_id}_{topic_id}
+     */
+    public static final String MATERIAL_PREFIX_KEY = "triggerpoi";
+
+    /**
+     * construct key
+     */
+    public static final String PREFIX_KEY_TYPE = "prefix";
+    public static final String CANDIDATE_KEY_TYPE = "candidate";
+
+
+    /**
+     * 召回形式
+     */
+    public static final String DB_LOCAL = "local";
+    public static final String DB_REDIS = "redis";
+    public static final String DB_KNN = "knn";
+
+    /**
+     * 每个 item type 对应不同的 defaul feature.
+     */
+    public static final String DEFAULT_USER_FEATURE_KEY = "";
+    public static final String DEFAULT_ITEM_FEATURE_KEY = "";
+
+    public static final String CHANNEL_NAME = "smartpush";
+    public static final String CHANNEL_NAME_TRIGGER = CHANNEL_NAME + "-trigger";
+    public static final String CHANNEL_NAME_PIPE = CHANNEL_NAME + "-pipe";
+    public static final String CHANNEL_NAME_RANKER = CHANNEL_NAME + "-ranker";
+    public static final String CHANNEL_NAME_Before_RANKER = CHANNEL_NAME + "-BeforeRanker";
+    public static final String CHANNEL_NAME_After_RANKER = CHANNEL_NAME + "-AfterRanker";
+    public static final String CHANNEL_NAME_Before_RANKER_RECALL = CHANNEL_NAME + "-BeforeRanker_Recall";
+    public static final String CHANNEL_NAME_After_RANKER_RECALL = CHANNEL_NAME + "-AfterRanker_Recall";
+
+    public static final String CHANNEL_NAME_FILTER = CHANNEL_NAME + "-filter";
+    public static final String CHANNEL_NAME_RERANKER = CHANNEL_NAME + "-reranker";
+    public static final String CHANNEL_NAME_RETRIEVER = CHANNEL_NAME + "-retriever";
+    public static final String CHANNEL_NAME_RECALLER = CHANNEL_NAME + "-recaller";
+    public static final String CHANNEL_NAME_HANDLER = CHANNEL_NAME + "-handler";
+    public static final String CHANNEL_NAME_HANDLER_BOOLEAN = CHANNEL_NAME_HANDLER + "-boolean";
+    public static final String CHANNEL_NAME_CONSTRUCTOR = CHANNEL_NAME + "-constructor";
+    public static final String CHANNEL_NAME_RETURN = CHANNEL_NAME + "-return";
+
+    public static final String STAY_CLUSTER_NAME = "nlpml-udm-godeye-lbs";
+    public static final String STAY_CATEGORY_NAME = "LbsStayUuidV2";
+
+    /**
+     * pipeline constants
+     */
+    public static final String User = "User";
+    public static final String RECOMMEND_REQUEST_DATA = "RecommendRequestData";
+    public static final String RANK_RESULT = "RANK_RESULT";
+    public static final String RESULT = "RESULT";
+    public static final String SERVICE_CODE = "SERVICE_CODE";
+    public static final String GO_FAST = "GO_FAST";
+
+    public static final String RANK_ITEMS = "rankerItems";
+
+    public static final Integer RESULT_SUCCESS = 0;
+    public static final Integer PARAM_ERROR = 1;
+    public static final Integer CANDIDATE_NULL = 2;
+
+    /**
+     * U2T
+     */
+    public static final String CLIENT_TYPE = "clientType";
+    public static final String CLIENT_VERSION = "clientVersion";
+
+    /**
+     * 来源
+     */
+    public static final String SOURCE_BATCH = "batch";
+    public static final String SOURCE_RT = "rt";
+
+
+
+    /**
+     * 单策略索引召回截断50
+     */
+    public static final Integer RECALL_LIMIT_NUMBER = 50;
+
+
+    /*
+     * ItemTemplate.extraMap 的一些key 的取值
+     */
+
+    /*
+     * Event 打点的一些 name 常量
+     */
+    public static final String METRIC_SUCCESS = "success";
+
+    /**
+     * 未知异常
+     */
+    public static final String EXCEPTION_ERR = "exception";
+    /**
+     * 返回码校验错误
+     */
+    public static final String METRIC_SERVICE_CODE_FAILURE = "serevice-code-failure";
+    /**
+     * 参数校验错误
+     */
+    public static final String METRIC_PARAM_CHECK_FAILURE = "param-failure";
+
+
+    /**
+     * 空返回列表
+     */
+    public static final String METRIC_BLANK_RETRUN = "blank-return";
+
+
+    /**
+     * 空返回元素
+     */
+    public static final String METRIC_ELEMENT_BLANK_RETRUN = "blank-element-return";
+
+
+    /**
+     * 空行为
+     */
+    public static final String METRIC_BLANK_ACTION = "blank-action";
+
+    public static final String METRIC_TRUE = "true";
+    public static final String METRIC_FALSE = "false";
+
+    /**
+     * 经纬度无效
+     */
+    public static final String LAT_LNG_INVALID = "invalid-lat-lng";
+
+
+    /**
+     * 候选为空
+     */
+    public static final String CANDIDATE_MAP_EMPTY = "candidateMapEmpty";
+
+    /**
+     * metric status key
+     */
+    public static final String METRIC_STATUS = "status";
+
+    /**
+     * 业务来源标识
+     */
+    public static final String SG_BU_NAME = "shangou";
+
+    /**
+     * 是否上传日志到log表
+     * */
+
+    public static final boolean UPLOAD_DATA2LOG_FLAG = true;
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/utils/HttpUtils.java

@@ -0,0 +1,122 @@
+package com.tzld.piaoquan.recommend.framework.utils;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.protocol.HTTP;
+import org.apache.http.util.EntityUtils;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Map;
+
+public class
+HttpUtils {
+    private static int CONNECT_TIME_OUT = 1000;
+    private static int CONNECT_REQUEST_TIME_OUT = 800;
+    private static int SOCKET_TIME_OUT = 800;
+    private static int MAX_PER_ROUTE = 1000;
+    private static int MAX_TOTAL = 1000;
+
+    private static PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
+    //set timeout
+    private static RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(
+            CONNECT_TIME_OUT).setSocketTimeout(SOCKET_TIME_OUT).setConnectionRequestTimeout(
+            CONNECT_REQUEST_TIME_OUT).build();
+
+    // Increase max total connection to MAX_TOTAL
+    static {
+        cm.setMaxTotal(MAX_TOTAL);
+
+        // Increase default max connection per route to MAX_PER_ROUTE
+        cm.setDefaultMaxPerRoute(MAX_PER_ROUTE);
+
+    }
+
+    private static CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(
+            cm).build();
+
+    public static String getContent(String host, String path, Map<String, String> parameters)
+            throws IOException {
+        try {
+            URIBuilder uriBuilder = new URIBuilder().setScheme("http").setHost(host).setPath(path);
+            for (Map.Entry<String, String> pEntry : parameters.entrySet()) {
+                uriBuilder.setParameter(pEntry.getKey(), pEntry.getValue());
+            }
+            URI uri = uriBuilder.build();
+            HttpGet httpGet = new HttpGet(uri);
+            httpGet.setConfig(requestConfig);
+
+            try {
+                HttpResponse httpResponse = httpClient.execute(httpGet);
+                String result = EntityUtils.toString(httpResponse.getEntity(), HTTP.UTF_8);
+                return result;
+            } finally {
+                httpGet.releaseConnection();
+            }
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new IOException(e);
+        }
+    }
+
+    public static Pair<Integer, String> getResponse(String host, String path, Map<String, String> parameters)
+            throws IOException {
+        try {
+            URIBuilder uriBuilder = new URIBuilder().setScheme("http").setHost(host).setPath(path);
+            for (Map.Entry<String, String> pEntry : parameters.entrySet()) {
+                uriBuilder.setParameter(pEntry.getKey(), pEntry.getValue());
+            }
+            URI uri = uriBuilder.build();
+            HttpGet httpGet = new HttpGet(uri);
+            httpGet.setConfig(requestConfig);
+
+            try {
+                HttpResponse httpResponse = httpClient.execute(httpGet);
+                int status = 0;
+                if (httpResponse != null && httpResponse.getStatusLine() != null) {
+                    status = httpResponse.getStatusLine().getStatusCode();
+                }
+                String result = null;
+                if (status == 200) {
+                    result = EntityUtils.toString(httpResponse.getEntity(), HTTP.UTF_8);
+                }
+                return Pair.of(status, result);
+            } finally {
+                httpGet.releaseConnection();
+            }
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new IOException(e);
+        }
+    }
+
+    public static String postContent(String uri, UrlEncodedFormEntity entity) throws IOException {
+        try {
+            HttpPost httpPost = new HttpPost(uri);
+            httpPost.setConfig(requestConfig);
+            httpPost.setEntity(entity);
+            try {
+                HttpResponse httpResponse = httpClient.execute(httpPost);
+                String result = EntityUtils.toString(httpResponse.getEntity());
+                return result;
+            } finally {
+                httpPost.releaseConnection();
+            }
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new IOException(e);
+        }
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/utils/ParamUtils.java

@@ -0,0 +1,113 @@
+package com.tzld.piaoquan.recommend.framework.utils;
+
+import com.google.common.collect.Maps;
+
+
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+
+
+public class ParamUtils {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ParamUtils.class);
+
+    /**
+     *
+     * @param requestData
+     * @return
+     */
+    public static Set<String> getExpIdStrSet(RecommendRequest requestData) {
+        Set<String> result = new HashSet<>();
+        String expIdStr = getFullExp(requestData);
+        if (!StringUtils.isEmpty(expIdStr)) {
+            String[] expArr = expIdStr.split(":");
+            for (String expId : expArr) {
+                result.add(expId);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * 获取实验id集合,包括新实验平台和老实验平台, 返回 integer 集合
+     *
+     * @param requestData
+     * @return
+     */
+    public static Set<Integer> getExpIdSet(RecommendRequest requestData) {
+        Set<Integer> result = new HashSet<Integer>();
+
+        Set<String> expArr = getExpIdStrSet(requestData);
+        if (expArr == null || expArr.isEmpty()) {
+            return result;
+        }
+
+        for (String expId : expArr) {
+            try {
+                result.add(Integer.valueOf(expId));
+            } catch (Exception e) {
+                LOGGER.error("Exception parsing eid error {}", e);
+            }
+        }
+
+        return result;
+    }
+
+    /**
+     * 修正带":|"的 expid，如 1:2:3:|123,124 -> 1:2:3
+     *
+     * @param requestData
+     * @return
+     */
+    public static String getFullExp(RecommendRequest requestData) {
+        return "";
+    }
+
+
+    /**
+     * 老实验平台, 获取某一层的实验id, layer 从1开始算
+     *
+     * @param requestData
+     * @param layer       index from 1
+     * @return
+     */
+    public static int getExpId(RecommendRequest requestData, int layer) {
+
+        // TODO
+        // 获取实验ID
+
+        return 0;
+    }
+
+    /**
+     * 或者老实验平台某一层的实验id
+     *
+     * @param wholeExpId
+     * @param layer      index from 1
+     * @return
+     */
+    public static int getExpId(String wholeExpId, int layer) {
+        int expId = 0;
+        if (layer < 1) {
+            return expId;
+        }
+        if (StringUtils.isNotBlank(wholeExpId)) {
+            String[] items = wholeExpId.split(":");
+            if (items.length >= layer) {
+                try {
+                    expId = Integer.parseInt(items[layer - 1]);
+                    // string is a number
+                } catch (NumberFormatException e) {
+                    LOGGER.error("exp id error: [{}]", wholeExpId);
+                }
+            }
+        }
+        return expId;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/utils/threadpool/FixedThreadPoolHelper.java

@@ -0,0 +1,12 @@
+package com.tzld.piaoquan.recommend.framework.utils.threadpool;
+
+
+public class FixedThreadPoolHelper extends ThreadPoolHelper {
+    public FixedThreadPoolHelper(int nThread) {
+        super(ThreadPoolUtils.newFixedThreadPool(nThread));
+    }
+
+    public FixedThreadPoolHelper(int nThread, String name) {
+        super(ThreadPoolUtils.newFixedThreadPool(nThread, name), "FixedPool_" + name);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/utils/threadpool/ThreadPoolHelper.java

@@ -0,0 +1,95 @@
+package com.tzld.piaoquan.recommend.framework.utils.threadpool;
+
+
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ *   线程池Helper
+ *   可以命名线程池; 保存ThreadPoolExecutor实例, 方便监控线程池信息
+ */
+public class ThreadPoolHelper {
+    private String name;
+    private ThreadPoolExecutor threadPoolExecutor;
+    private Timer counterTimer;
+
+    ThreadPoolHelper(ThreadPoolExecutor threadPoolExecutor) {
+        this.threadPoolExecutor = threadPoolExecutor;
+        this.name = "";
+    }
+
+    ThreadPoolHelper(ThreadPoolExecutor threadPoolExecutor, String name) {
+        this.threadPoolExecutor = threadPoolExecutor;
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public ThreadPoolExecutor getThreadPoolExecutor() {
+        return threadPoolExecutor;
+    }
+
+    public long getTaskCount() {
+        return this.threadPoolExecutor.getTaskCount();
+    }
+
+    public long getActiveCount() {
+        return this.threadPoolExecutor.getActiveCount();
+    }
+
+    public long getCorePoolSize() {
+        return this.threadPoolExecutor.getCorePoolSize();
+    }
+
+    public long getPoolSize() {
+        return this.threadPoolExecutor.getPoolSize();
+    }
+
+    public long getMaximumPoolSize() {
+        return this.threadPoolExecutor.getMaximumPoolSize();
+    }
+
+    public long getLargestPoolSize() {
+        return this.threadPoolExecutor.getLargestPoolSize();
+    }
+
+    public long getTaskQueueSize() {
+        return this.threadPoolExecutor.getQueue().size();
+    }
+
+    public void counterThreadPoolInfo(String prefix) {
+        // print counters
+    }
+
+    /**
+     * 开启线程池PerfCounter
+     * @param prefix    perf counter name prefix
+     * @return          this
+     */
+    public ThreadPoolHelper enableCounter(final String prefix) {
+        if (counterTimer == null) {
+            synchronized (this) {
+                if (counterTimer == null) {
+                    counterTimer = new Timer();
+                }
+            }
+        }
+        counterTimer.schedule(new TimerTask() {
+            @Override
+            public void run() {
+                counterThreadPoolInfo(prefix);
+            }
+        }, 1000, 1000);
+        return this;
+    }
+
+    public void cancelCounter() {
+        if (counterTimer != null) {
+            counterTimer.cancel();
+            counterTimer.purge();
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/framework/utils/threadpool/ThreadPoolUtils.java

@@ -0,0 +1,55 @@
+package com.tzld.piaoquan.recommend.framework.utils.threadpool;
+
+
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.*;
+
+/**
+ * 命名线程池工场类
+ * 最后创建的线程名称:
+ *   name_{counter}
+ */
+class NamedThreadFactory implements ThreadFactory {
+    private int counter = 0;
+    private String name;
+
+    NamedThreadFactory(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public Thread newThread(Runnable r) {
+        counter += 1;
+        return new Thread(r, name + "_" + counter);
+    }
+}
+
+
+public class ThreadPoolUtils {
+    public static ThreadPoolExecutor newFixedThreadPool(int nThreads, String poolName) {
+        return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
+                new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("Fixed_" + poolName));
+    }
+
+    public static ThreadPoolExecutor newFixedThreadPool(int nThreads) {
+        return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
+                new LinkedBlockingQueue<Runnable>());
+    }
+
+    public static ThreadPoolExecutor newCachedThreadPool(String poolName) {
+        return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
+                new SynchronousQueue<Runnable>(), new NamedThreadFactory("Cached_" + poolName));
+    }
+
+    public static ThreadPoolExecutor newCachedThreadPool() {
+        return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
+                new SynchronousQueue<Runnable>());
+    }
+}
+
+
+
+
+
+

recommend-server-service/src/main/resources/feeds_merge.conf

@@ -0,0 +1,44 @@
+queue-config = {
+  top-queue = {
+    class = ""
+    children = {
+      // recent exploit
+      feed-recent-clicked-last-index = {
+        class = ""
+      }
+      //exploit
+      feed-category-exploit-index = {
+        class = ""
+      }
+      feed-tag-exploit-index = {
+        class = ""
+      }
+    }
+  }
+}
+
+
+rule-config = {
+  // 顶层队列
+  top-queue = {
+    //topmerge中会修改比例
+    merge-rule = {
+      feed-recent-clicked-last-index = {
+        recall-percentage = 0.04
+        max-merge-num = 2
+        priority = 4
+      }
+      feed-category-exploit-index = {
+        recall-percentage = 0.04
+        max-merge-num = 3
+        priority = 1
+      }
+      feed-tag-exploit-index  = {
+        recall-percentage = 0.04
+        min-merge-num = 1
+        max-merge-num = 2
+        priority = 1
+      }
+    }
+  }
+}

recommend-server-service/src/main/resources/feeds_recall.conf

@@ -0,0 +1,13 @@
+recall-config = {
+
+  filter-config = {
+    dedup-filter-config = {
+      filter-name = ""
+      disable-expids = []
+    }
+
+    history-filter-config = {
+      filter-name = ""
+      disable-expids = []
+    }
+}

recommend-server-service/src/main/resources/feeds_score.conf

@@ -0,0 +1,15 @@
+scorer-config = {
+
+  lr-score-config = {
+    scorer-name = ""
+    scorer-priority = 100
+  }
+
+  // 完成度模型
+  xgb-score-config = {
+    scorer-name = ""
+    scorer-priority = 99
+    model-path = ""
+  }
+
+}