Merge branch 'feature_refactor' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -2,13 +2,10 @@ package com.tzld.piaoquan.recommend.server.common.base;
 
 import com.google.common.collect.Maps;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.CandidateInfo;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import lombok.Data;
 
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 
@@ -39,12 +36,6 @@ public class RankItem implements Comparable<RankItem> {
 
     // 记录召回信息
     private String queue;
-    private CandidateInfo candidateInfo;
-    private List<CandidateInfo> candidateInfoList = new ArrayList<>(); // 兼容多个召回key命中
-
-    // merge信息
-    private List<String> mergeQueuePath = new ArrayList<>();
-    private List<String> mergeDecisionLabels = new ArrayList<>();
 
     // 多样性过滤因子
     // 排序因子
@@ -86,23 +77,6 @@ public class RankItem implements Comparable<RankItem> {
         this.scoreStr = other.scoreStr;
         this.video = other.video;
 
-        // merge queue
-        this.queue = other.queue;
-        if (other.getMergeQueuePath() != null)
-            this.mergeQueuePath.addAll(other.getMergeQueuePath());
-        if (other.getMergeDecisionLabels() != null)
-            this.mergeDecisionLabels.addAll(other.getMergeDecisionLabels());
-
-
-        // candidateinfo
-        this.candidateInfo = other.getCandidateInfo() != null ? other.getCandidateInfo().deepcopy() : null;
-        if (other.getCandidateInfoList() != null) {
-            for (CandidateInfo tmpCandidateInfo : other.getCandidateInfoList()) {
-                this.candidateInfoList.add(tmpCandidateInfo.deepcopy());
-            }
-        }
-
-
     }
 
 
@@ -120,36 +94,4 @@ public class RankItem implements Comparable<RankItem> {
         }
     }
 
-    public RankItem deepcopy() {
-        return new RankItem(this);
-    }
-
-
-    public void addToCandidateInfoList(CandidateInfo candidateInfo) {
-        this.candidateInfoList.add(candidateInfo);
-    }
-
-
-    public void addMergeQueuePath(String mergeQueueName) {
-        this.mergeQueuePath.add(mergeQueueName);
-    }
-
-    public void addMergeDecisionLabel(String label) {
-        this.mergeDecisionLabels.add(label);
-    }
-
-    public void putRankerScore(String rankerName, Double rankerScore) {
-        this.rankerScore.put(rankerName, rankerScore);
-    }
-
-    public String getLastMergeQueueName() {
-        int pathSize = this.mergeQueuePath.size();
-        if (pathSize > 0) {
-            return this.mergeQueuePath.get(pathSize - 1);
-        } else {
-            return getCandidateInfo().getCandidateQueueName();
-        }
-    }
-
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Candidate.java

@@ -1,40 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.candidiate;
-
-
-import com.google.gson.Gson;
-import lombok.Data;
-
-
-/**
- * 记录召回下一路队列索引信息
- */
-@Data
-public class Candidate {
-    private String candidateKey;
-    private int candidateNum;
-    private QueueName candidateQueueName;
-    // 上层待合并的queue name
-    private String mergeQueueName;
-    private int mergeQueueNum;
-
-    public Candidate() {
-    }
-
-    public Candidate(Candidate other) {
-        this.candidateKey = other.getCandidateKey();
-        this.candidateNum = other.getCandidateNum();
-        this.candidateQueueName = other.getCandidateQueueName();
-        this.mergeQueueName = other.getMergeQueueName();
-        this.mergeQueueNum = other.getMergeQueueNum();
-    }
-
-    public Candidate deepcopy() {
-        return new Candidate(this);
-    }
-
-
-    @Override
-    public String toString() {
-        return new Gson().toJson(this);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/CandidateInfo.java

@@ -1,31 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.candidiate;
-
-
-import com.google.gson.Gson;
-import lombok.Data;
-
-@Data
-public class CandidateInfo {
-    private String candidateQueueName;     // 队列名称
-    private int position;               // 在召回中的位置
-    private Candidate candidate;        // 队列配置的信息
-
-    public CandidateInfo() {
-    }
-
-    public CandidateInfo(CandidateInfo other) {
-        this.candidateQueueName = other.getCandidateQueueName();
-        this.position = other.getPosition();
-        this.candidate = other.candidate != null ? new Candidate(other.candidate) : null;
-    }
-
-    public CandidateInfo deepcopy() {
-        return new CandidateInfo(this);
-    }
-
-
-    @Override
-    public String toString() {
-        return new Gson().toJson(this);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Entry.java

@@ -1,39 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.candidiate;
-
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * 一个队列召回的内容实体
- * @param <T>
- */
-public class Entry<T> {
-    public final String id;
-    public final T item;
-    public final Map<String, Double> scores;
-    public final Map<String, String> explanations;
-
-
-    public Entry(T item, String id) {
-        this.item = item;
-        this.id = id;
-        this.scores = new HashMap<String, Double>();
-        this.explanations = new HashMap<String, String>();
-    }
-
-    public Entry(Entry<T> other) {
-        this.item = other.item;
-        this.id = other.id;
-        this.scores = other.scores;
-        this.explanations = other.explanations;
-    }
-
-    public void addScore(String name, double score) {
-        scores.put(name, score);
-    }
-
-    public void addExplanation(String name, String explanation) {
-        explanations.put(name, explanation);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Queue.java

@@ -1,56 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.candidiate;
-
-
-import java.util.*;
-
-/**
- * An abstract data type represents a queue of certain item.
- *   一个队列内召回Items完整list
- */
-public class Queue<T> implements Iterable<Entry<T>> {
-    private final List<Entry<T>> entries;
-    private final String name;
-    private final String explain;
-
-    public Queue(String name) {
-        entries = new ArrayList<Entry<T>>();
-        this.name = name;
-        this.explain = null;
-    }
-
-    public Queue(String name, String explain) {
-        this.name = name;
-        this.explain = explain;
-        this.entries = new ArrayList<Entry<T>>();
-    }
-
-    public void reverse() {
-        Collections.reverse(this.entries);
-    }
-
-    public void add(Entry<T> entry) {
-        entries.add(entry);
-    }
-
-    public void addAll(Collection<Entry<T>> paramEntries) {
-        entries.addAll(paramEntries);
-    }
-
-    public List<Entry<T>> get() {
-        return entries;
-    }
-
-    public Iterator<Entry<T>> iterator() {
-        return entries.iterator();
-    }
-
-    public int size() {
-        return entries.size();
-    }
-
-    public void limit(int num) {
-        if (!entries.isEmpty() && num >= 0 && num < entries.size()) {
-            entries.subList(num, entries.size()).clear();
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/QueueName.java

@@ -1,114 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.candidiate;
-
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.FluentIterable;
-import lombok.Data;
-import org.apache.commons.lang3.tuple.Pair;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Queue names.召回队列命名处理
- */
-@Data
-public class QueueName implements Serializable, Comparable<QueueName> {
-    private static final Function<Pair<String, String>, String> MATCH_PATTERN =
-            new Function<Pair<String, String>, String>() {
-                @Override
-                public String apply(Pair<String, String> input) {
-                    return input.getLeft() + "=" + input.getRight();
-                }
-            };
-    public static final long DEFAULT_LOCAL_CACHE_TTL = 15 * 60 * 1000;
-    private final List<Pair<String, String>> matches = new ArrayList<Pair<String, String>>();
-    private final String itemType;
-    private final String ordering;
-    private final long ttl;
-
-    private String metaChannel;  // meta 渠道
-
-    public long getTTL(){
-        return this.ttl;
-    }
-
-    public QueueName(String itemType, String ordering) {
-        this(itemType, ordering, DEFAULT_LOCAL_CACHE_TTL);
-    }
-
-    public QueueName(String itemType, String ordering, long ttl) {
-        this.itemType = itemType;
-        this.ordering = ordering;
-        this.ttl = ttl;
-    }
-
-    public static QueueName fromString(String string, long ttl) {
-        String[] parts = string.split(":");
-        String itemType = parts[0];
-        String ordering = parts[parts.length - 1].split("=")[1];
-
-        QueueName result = new QueueName(itemType, ordering, ttl);
-        for (int i = 2; i < parts.length - 1; ++i) {
-            String[] sides = parts[i].split("=");
-            result.addMatch(sides[0], sides[1]);
-        }
-        return result;
-    }
-
-    public static QueueName fromString(String string) {
-        return QueueName.fromString(string, DEFAULT_LOCAL_CACHE_TTL);
-    }
-
-    public static Pair<String, String> parseMatchPair(String match) {
-        String[] arr = match.split("=");
-        if (arr.length == 2) {
-            return Pair.of(arr[0], arr[1]);
-        }
-        return Pair.of("", "");
-    }
-
-    public QueueName addMatch(String key, String value) {
-        if (value == null || value.equals("")) {
-            value = "_";
-        } else {
-            value = value.replace("=", "_");
-            value = value.replace(":", "_");
-        }
-        if (key.equals("channel")) {
-            this.setMetaChannel(value);
-        }
-        matches.add(Pair.of(key, value));
-        return this;
-    }
-
-    public Iterable<String> getMatches() {
-        return FluentIterable.from(matches).transform(MATCH_PATTERN);
-    }
-
-
-    @Override
-    public String toString() {
-        Iterable<String> matchesString = FluentIterable.from(matches)
-                .transform(MATCH_PATTERN);
-
-        return itemType + ":queue:" + Joiner.on(":").join(matchesString) + ":ordering=" + ordering;
-    }
-
-    @Override
-    public int hashCode() {
-        return toString().hashCode();
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        return other instanceof QueueName && ((QueueName) other).toString().equals(toString());
-    }
-
-    @Override
-    public int compareTo(QueueName other) {
-        return other.toString().compareTo(ordering.toString());
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/User.java

@@ -1,52 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.common;
-
-import lombok.Data;
-
-@Data
-public class User {
-
-    private String id;
-    private String uid;
-    private String mid;
-    private UserAttention userAttention;
-
-    // user profile 基础画像信息
-    private String machine_model;
-    private String machine_brand;
-    private String sdk;
-    private String gender;
-    private String ifCreater;
-    private String userType;
-    private String region;
-    private String city;
-
-    // user-content info 用户内容tag统计数据
-    private UserAction last1monthVideoTags;
-    private UserAction last7dayVideoTags;
-    private UserAction last1dayVideoTags;
-    private UserAction lastSessionVideoTags;
-
-    private UserAction last1monthTitleTags;
-    private UserAction last7dayTitleTags;
-    private UserAction last1dayTitleTags;
-    private UserAction lastSessionTitleTags;
-
-    // 用户关注up主信息
-    private UserAction last1monthPublishers;
-
-    // user-action info 用户
-    private UserAction last1monthUserAction;
-    private UserAction last7dayUserAction;
-    private UserAction last1dayUserAction;
-    private UserAction last1hourUserAction;
-    private UserAction lastSessionUserAction;
-
-    // user-group info
-    private String userGroup;
-
-    // user-vector
-    private String userVector;
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/UserAction.java

@@ -1,21 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.common;
-
-import lombok.Data;
-
-import java.util.List;
-
-@Data
-public class UserAction {
-    //记录用户行为中 itemids，tags等信息
-
-    private List<String> followed;
-    private List<String> follow_and_realplay;
-    private List<String> played;
-    private List<String> shared;
-
-    private List<String> feedsPlayed;
-    private List<String> realPlayed;
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/UserAttention.java

@@ -1,26 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.common;
-
-public class UserAttention {
-    /**
-     分维度的用户特征数据, 包含：
-     1、分不同时间维度的ItemList
-     30天、7天、1天、1H、实时，可以提供给I2I类召回策略用
-
-     2、分不同时间维度的categoryList
-     30天、7天、1天、1H、实时，可以提供给CB类召回策略用
-
-     3、用户group信息
-     天级别更新数据，可以提供给usergroup -> Item召回策略用。
-
-     4、分时间维度的发布者List
-     30天、7天、1天、1H、实时，可以提供 发布者 -> Item类召回用
-
-     */
-
-
-
-
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/IndexCandidateQueue.java

@@ -1,88 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.apache.commons.lang3.tuple.Pair;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-
-/**
- * 定义召回下子队列的Queue 融合信息
- */
-public abstract class IndexCandidateQueue extends StrategyQueue {
-
-    protected Map<String, Candidate> myCandidates = new HashMap<String, Candidate>();
-
-    public IndexCandidateQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    public abstract int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId);
-
-
-    @Override
-    public int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        // TODO: sublist by every queue
-        return MergeUtils.simpleMergeWithProtection(items, rankerItemsListMap, recNum, expId);
-    }
-
-    @Override
-    public int candidate(final Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
-        myCandidates.clear();
-        int n = addCandidateKey(candidateMap, recallNum, user, requestData, requestIndex, expId);
-        candidateMap.putAll(myCandidates);
-
-        return n;
-    }
-
-    protected int addCandidateKey(Map<String, Candidate> candidates, String key, int num, String queue) {
-        if (candidates.containsKey(key)) {
-            return 0;
-        } else {
-            Candidate candidate = new Candidate();
-            candidate.setCandidateKey(key);
-            candidate.setCandidateNum(num);
-            candidate.setMergeQueueName(queue);
-            candidates.put(key, candidate);
-            return num;
-        }
-    }
-
-    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue) {
-        return this.addCandidateKey(candidates, queueName, num, queue, true, true, true);
-    }
-
-    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue, boolean needScore, boolean needPornRank) {
-        return this.addCandidateKey(candidates, queueName, num, queue, needScore, needPornRank, true);
-    }
-
-    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue, boolean needScore, boolean needPornRank, boolean needExposeHistoryFilter) {
-        if (candidates.containsKey(queueName.toString())) {
-            return 0;
-        } else {
-            Candidate candidate = new Candidate();
-            candidate.setCandidateKey(queueName.toString());
-            candidate.setCandidateNum(num);
-            candidate.setMergeQueueName(queue);
-            candidates.put(queueName.toString(), candidate);
-            return num;
-        }
-    }
-
-    @Override
-    public String toString() {
-        return "IndexCandidateQueue{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/MergeRule.java

@@ -1,45 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import lombok.Data;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-
-/**
- * Merge 规则配置
- */
-@Data
-public class MergeRule {
-    public String queueName;
-
-    public Set<Integer> enableExpIdSet = null;
-    public Set<Integer> disableExpIdSet = null;
-
-    public double recallPercentage = 0.1; // 召回百分比, 默认占 10%
-    public int minMergeNum = 0; // 合并保护规则: 最小合并条数
-    public int maxMergeNum = 100; // 合并保护规则: 最大合并条数
-
-    private Map<String, String> properties;
-
-    public MergeRule() {
-        this.properties = new HashMap<String, String>();
-    }
-
-    public boolean isDisabled(final int expId) {
-        if (enableExpIdSet != null && !enableExpIdSet.contains(expId)) {
-            return true;
-        }
-        if (disableExpIdSet != null && disableExpIdSet.contains(expId)) {
-            return true;
-        }
-        return false;
-    }
-
-    public void putProperty(String key, String value) {
-        this.properties.put(key, value);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/MergeUtils.java

@@ -1,313 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.google.common.base.Function;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.CandidateInfo;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.apache.commons.lang3.tuple.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.*;
-
-
-public class MergeUtils {
-    public static final Logger LOGGER = LoggerFactory.getLogger(MergeUtils.class);
-
-    // mergeConfFile_topQueueName 对应的 StrategyQueue 缓存
-    private static final Map<String, Config> queueConfigMap = new HashMap<>();
-
-    public static StrategyQueue createTopQueue(String mergeConfFile, String topQueueName) {
-        String key = mergeConfFile + "_" + topQueueName;
-        Config mergeQueueConf;
-        if (queueConfigMap.containsKey(key)) {
-            mergeQueueConf = queueConfigMap.get(key);
-        } else {
-            mergeQueueConf = ConfigFactory.parseResources(mergeConfFile);
-            queueConfigMap.put(key, mergeQueueConf);
-        }
-        StrategyQueueConfig strategyQueueConfig = new StrategyQueueConfig();
-        if (strategyQueueConfig.load(mergeQueueConf)) {
-            LOGGER.debug("Merger config init succ");
-        } else {
-            LOGGER.error("Merge config init failed: init MergeBiz using queue config {}", mergeConfFile);
-        }
-
-        StrategyQueue topQueue = null;
-        try {
-            topQueue = MergeUtils.constructStrategyQueue(topQueueName, strategyQueueConfig);
-        } catch (InstantiationException e) {
-            LOGGER.error("construct top StrategyQueue: [{}]", e);
-        } catch (IllegalAccessException e) {
-            LOGGER.error("construct top StrategyQueue: [{}]", e);
-        } catch (ClassNotFoundException e) {
-            LOGGER.error("construct top StrategyQueue: [{}]", e);
-        } catch (NoSuchMethodException e) {
-            LOGGER.error("construct top StrategyQueue: [{}]", e);
-        } catch (InvocationTargetException e) {
-            LOGGER.error("construct top StrategyQueue: [{}]", e);
-        }
-        return topQueue;
-    }
-
-    public static StrategyQueue constructStrategyQueue(String queueName, StrategyQueueConfig strategyQueueConfig) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
-        if (!strategyQueueConfig.getStrategyQueueInfoMap().containsKey(queueName)) {
-            return null;
-        }
-
-        StrategyQueueInfo strategyQueueInfo = strategyQueueConfig.getStrategyQueueInfoMap().get(queueName);
-        StrategyQueue strategyQueue = (StrategyQueue) Class.forName(strategyQueueInfo.getQueueClass())
-                .getConstructor(StrategyQueueInfo.class, strategyQueueConfig.getClass())
-                .newInstance(strategyQueueInfo, strategyQueueConfig);
-
-        return strategyQueue;
-    }
-
-
-    /**
-     * 分发items到策略树中
-     * 分发的过程中不要破坏相对顺序
-     *
-     * @param strategyQueue
-     * @param items
-     */
-    public static void distributeItemsToMultiQueues(StrategyQueue strategyQueue, List<RankItem> items) {
-        List<StrategyQueue> strategyQueueList = strategyQueue.getAllQueues();
-
-        Multimap<String, RankItem> mergeQueuesItems = ArrayListMultimap.create();
-        for (RankItem item : items) {
-            for (CandidateInfo candidateInfo : item.getCandidateInfoList()) {
-                String mergeQueue = candidateInfo.getCandidate().getMergeQueueName();
-                RankItem currentItem = item.deepcopy();
-
-                // set candidate info
-                currentItem.setQueue(mergeQueue);
-                currentItem.setCandidateInfo(candidateInfo);
-
-                mergeQueuesItems.put(mergeQueue, currentItem);
-            }
-        }
-        for (StrategyQueue queue : strategyQueueList) {
-            String mergeQueueName = queue.getStrategyQueueInfo().getQueueName();
-            if (mergeQueuesItems.containsKey(mergeQueueName)) {
-                List<RankItem> currMergeQueueItems = new ArrayList<RankItem>(mergeQueuesItems.get(mergeQueueName));
-                queue.setItems(currMergeQueueItems);
-            }
-        }
-    }
-
-
-
-
-    /**
-     * 基于 score 字段融合: score 较大的 Item 优先
-     *
-     * @param resultRankerItems
-     * @param rankerItemsListMap
-     * @param freeRecNum
-     * @param expId
-     */
-    public static void simpleMergeByScore(List<RankItem> resultRankerItems,
-                                          final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap,
-                                          int freeRecNum,
-                                          int expId) {
-        // 先定义按score排序优选的独立额
-        PriorityQueue<Pair<String, Integer>> mergePriorityQueue = new PriorityQueue<Pair<String, Integer>>(freeRecNum, Comparator.comparing(o -> rankerItemsListMap.get(o.getLeft()).getRight().get(o.getRight())));
-
-        // 大于最小mergenum 同时小于maxMergenum，在队列中add
-        for (Pair<MergeRule, List<RankItem>> entry : rankerItemsListMap.values()) {
-            if (entry.getLeft().isDisabled(expId) || entry.getRight() == null || entry.getRight().isEmpty()) {
-                continue;
-            }
-
-            MergeRule myRule = entry.getLeft();
-            if (myRule.minMergeNum < myRule.maxMergeNum && entry.getRight().size() > myRule.minMergeNum) {
-                Collections.sort(entry.getRight().subList(myRule.minMergeNum, entry.getRight().size()));
-                mergePriorityQueue.add(Pair.of(myRule.queueName, myRule.minMergeNum));
-            }
-        }
-        //
-        while (freeRecNum > 0 && !mergePriorityQueue.isEmpty()) {
-            Pair<String, Integer> item = mergePriorityQueue.poll();
-            String myName = item.getLeft();
-            int myIndex = item.getRight();
-
-            resultRankerItems.add(rankerItemsListMap.get(myName).getRight().get(myIndex));
-            resultRankerItems.get(resultRankerItems.size() - 1).addMergeDecisionLabel("score:" + myName);
-            freeRecNum -= 1;
-            myIndex += 1;
-
-            if (freeRecNum > 0 && rankerItemsListMap.get(myName).getRight().size() > myIndex && myIndex < rankerItemsListMap.get(myName).getLeft().maxMergeNum) {
-                mergePriorityQueue.add(Pair.of(myName, myIndex));
-            }
-        }
-    }
-
-    /**
-     * 队列融合:
-     * 1. 各队列基于排序不变;
-     * 2. 各队列保证最小合并数量 (minMergeNum) 条目;
-     * 3. 各队列限制最大合并数量 (maxMergeNum) 条目;
-     * 4. 从各队列取 Score 最高条目.
-     *
-     * @param rankerItemsListMap
-     * @param recNum
-     * @param expId
-     * @return
-     */
-    public static int simpleMergeWithProtection(List<RankItem> resultRankerItems,
-                                                final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap,
-                                                int recNum,
-                                                int expId) {
-        if (resultRankerItems == null) {
-            resultRankerItems = new LinkedList<RankItem>();
-        }
-        // 保证各队列最小合并条目后剩余可自由调配条目数量
-        int freeRecNum = recNum;
-        for (Pair<MergeRule, List<RankItem>> entry : rankerItemsListMap.values()) {
-
-            if (entry.getLeft().isDisabled(expId) || entry.getRight() == null || entry.getRight().isEmpty()) {
-                continue;
-            }
-
-            MergeRule myRule = entry.getLeft();
-            int mySize = entry.getRight().size();
-            int myMinMergeNum = Math.min(myRule.minMergeNum, mySize);
-            freeRecNum -= myMinMergeNum;
-
-            int prevSize = resultRankerItems.size();
-            // 先保证最小长度添加
-            resultRankerItems.addAll(entry.getRight().subList(0, myMinMergeNum));
-
-            int afterSize = resultRankerItems.size();
-            for (int ix = prevSize; ix < afterSize; ++ix) {
-                resultRankerItems.get(ix).addMergeDecisionLabel("min_protect:" + myRule.queueName);
-            }
-        }
-
-        if (freeRecNum > 0) {
-            //保证最小长度后，其他按score融合
-            simpleMergeByScore(resultRankerItems, rankerItemsListMap, freeRecNum, expId);
-        }
-
-        return resultRankerItems.size();
-    }
-
-    /**
-     * 按优先级队列融合:
-     * 1. 各队列基于 priority 依次融合: 各队列保证最小合并数量 (minMergeNum) 条目;
-     * 2. 若没有凑足 recNum, 各队列限制最大合并数量 (maxMergeNum) 条目条件下按 score 优先选择;
-     *
-     * @param rankerItemsListMap
-     * @param recNum
-     * @param user
-     * @param requestData
-     * @param requestIndex
-     * @param expId
-     * @return
-     */
-    public static int simpleMergeByPriority(List<RankItem> resultRankerItems, final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        if (resultRankerItems == null) {
-            resultRankerItems = new LinkedList<RankItem>();
-        }
-
-        final List<Map.Entry<String, Pair<MergeRule, List<RankItem>>>> rankerItemsList = new LinkedList<Map.Entry<String, Pair<MergeRule, List<RankItem>>>>(rankerItemsListMap.entrySet());
-
-        // 保证各队列最小合并条目后剩余可自由调配条目数量
-        Collections.sort(rankerItemsList, new Comparator<Map.Entry<String, Pair<MergeRule, List<RankItem>>>>() {
-            private final int DEFAULT_PRIORITY = 0;
-
-            @Override
-            public int compare(Map.Entry<String, Pair<MergeRule, List<RankItem>>> o1, Map.Entry<String, Pair<MergeRule, List<RankItem>>> o2) {
-                int p1 = getPriority(o1.getValue().getLeft()), p2 = getPriority(o2.getValue().getLeft());
-                return p1 - p2;
-            }
-
-            private int getPriority(MergeRule rule) {
-                if (rule.getProperties().containsKey("priority")) {
-                    return Integer.valueOf(rule.getProperties().get("priority"));
-                }
-                return DEFAULT_PRIORITY;
-            }
-        });
-
-        int freeRecNum = recNum;
-        for (int i = 0; i < rankerItemsList.size(); ++i) {
-            if (rankerItemsList.get(i).getValue().getLeft().isDisabled(expId) || rankerItemsList.get(i).getValue().getRight() == null || rankerItemsList.get(i).getValue().getRight().isEmpty()) {
-                continue;
-            }
-
-            MergeRule myRule = rankerItemsList.get(i).getValue().getLeft();
-            int mySize = rankerItemsList.get(i).getValue().getRight().size();
-            int myMinMergeNum = Math.min(myRule.minMergeNum, mySize);
-            freeRecNum -= myMinMergeNum;
-
-            int prevSize = resultRankerItems.size();
-            resultRankerItems.addAll(rankerItemsList.get(i).getValue().getRight().subList(0, myMinMergeNum));
-            if (mySize > myMinMergeNum) {
-                Collections.sort(rankerItemsList.get(i).getValue().getRight().subList(myMinMergeNum, mySize));
-            }
-            int afterSize = resultRankerItems.size();
-            for (int ix = prevSize; ix < afterSize; ++ix) {
-                resultRankerItems.get(ix).addMergeDecisionLabel("min_protect:" + myRule.queueName);
-            }
-        }
-
-        if (freeRecNum > 0) {
-            simpleMergeByScore(resultRankerItems, rankerItemsListMap, freeRecNum, expId);
-        }
-
-        return resultRankerItems.size();
-    }
-
-    /**
-     * 基于 tag 和 category 重排序, 保证具有相同的分类或标签的 Item 在 windowSize 窗口内只有 similarLimit 条
-     *
-     * @param items
-     * @param expectedRecommendItemsCount
-     * @param windowSize
-     * @param similarLimit
-     */
-    public static void diversityRerank(List<RankItem> items, Function<Pair<RankItem, RankItem>, Boolean> similarFunc, int expectedRecommendItemsCount, final int windowSize, final int similarLimit) {
-        List<RankItem> rerankedItems = new LinkedList<RankItem>();
-        expectedRecommendItemsCount = Math.min(expectedRecommendItemsCount, items.size());
-        for (int ix = 0; ix < expectedRecommendItemsCount; ++ix) {
-            int windowStartPosition = Math.max(ix - windowSize + 1, 0);
-            int nextId = 0;
-            while (nextId < items.size()) {
-                int similarCount = 0;
-                for (int jx = windowStartPosition; jx < ix && similarCount < similarLimit; ++jx) {
-                    if (similarFunc.apply(Pair.of(items.get(nextId), rerankedItems.get(jx)))) {
-                        similarCount += 1;
-                    }
-                }
-                if (similarCount < similarLimit) {
-                    break;
-                }
-                nextId += 1;
-            }
-            if (nextId >= items.size()) {
-                nextId = 0;
-            }
-
-            RankItem nextItem = items.get(nextId);
-            nextItem.addMergeDecisionLabel("sim_rerank:" + windowStartPosition + "_" + windowSize);
-            items.remove(nextId);
-            rerankedItems.add(nextItem);
-        }
-
-        if (items.size() > 0) {
-            rerankedItems.addAll(items);
-        }
-
-        items.clear();
-        items.addAll(rerankedItems);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/SimilarityUtils.java

@@ -1,116 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.google.common.collect.Sets;
-import com.google.common.base.Function;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import org.apache.commons.lang3.tuple.Pair;
-
-import javax.annotation.Nullable;
-
-
-public class SimilarityUtils {
-
-    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameUserTagOrCategoryFunc() {
-        return new Function<Pair<RankItem, RankItem>, Boolean>() {
-            @Nullable
-            @Override
-            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
-                return isSameCategory(itemPair.getLeft(), itemPair.getRight()) || isSameUserTag(itemPair.getLeft(), itemPair.getRight());
-            }
-        };
-    }
-
-    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameTitleTagFunc() {
-        return new Function<Pair<RankItem, RankItem>, Boolean>() {
-            @Nullable
-            @Override
-            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
-                return isSameTitleTag(itemPair.getLeft(), itemPair.getRight());
-            }
-        };
-    }
-
-    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameUserTagFunc() {
-        return new Function<Pair<RankItem, RankItem>, Boolean>() {
-            @Nullable
-            @Override
-            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
-                return isSameUserTag(itemPair.getLeft(), itemPair.getRight());
-            }
-        };
-    }
-
-    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameCategoryFunc() {
-        return new Function<Pair<RankItem, RankItem>, Boolean>() {
-            @Nullable
-            @Override
-            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
-                return isSameCategory(itemPair.getLeft(), itemPair.getRight());
-            }
-        };
-    }
-
-    public static Function<Pair<RankItem, RankItem>, Boolean> getIsUserTagFunc() {
-        return new Function<Pair<RankItem, RankItem>, Boolean>() {
-            @Nullable
-            @Override
-            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
-                return isSameUserTag(itemPair.getLeft(), itemPair.getRight());
-            }
-        };
-    }
-
-    public static boolean isSameCategory(RankItem rankerItem1, RankItem rankerItem2) {
-        if (rankerItem1 == null || rankerItem2 == null ||
-                rankerItem1.getRankItemCategories() == null ||
-                rankerItem2.getRankItemCategories() == null) {
-            return false;
-        }
-
-        return !Sets.intersection(rankerItem1.getRankItemCategories().keySet(),
-                        rankerItem2.getRankItemCategories().keySet())
-                .isEmpty();
-    }
-
-    public static boolean isSameTitleTag(RankItem rankerItem1, RankItem rankerItem2) {
-        if (rankerItem1 == null || rankerItem2 == null ||
-                rankerItem1.getRankItemTitleTags() == null ||
-                rankerItem2.getRankItemTitleTags() == null) {
-            return false;
-        }
-        return !Sets.intersection(rankerItem1.getRankItemTitleTags().keySet(),
-                        rankerItem2.getRankItemTitleTags().keySet())
-                .isEmpty();
-    }
-
-    public static boolean isSameUserTag(RankItem rankerItem1, RankItem rankerItem2) {
-        if (rankerItem1 == null || rankerItem2 == null ||
-                rankerItem1.getRankItemUserTags() == null ||
-                rankerItem2.getRankItemUserTags() == null) {
-            return false;
-        }
-        return !Sets.intersection(rankerItem1.getRankItemUserTags().keySet(),
-                        rankerItem2.getRankItemUserTags().keySet())
-                .isEmpty();
-    }
-
-    public static boolean isSameQueue(RankItem rankerItem1, RankItem rankerItem2) {
-        if (rankerItem1 == null || rankerItem2 == null ||
-                rankerItem1.getQueue() == null) {
-            return false;
-        }
-        return rankerItem1.getQueue().equals(rankerItem2.getQueue());
-    }
-
-    public static boolean isSameSubCategory(RankItem item1, RankItem item2) {
-        if (item1 == null || item2 == null || item1.getRankItemSubCategories() == null
-                || item2.getRankItemSubCategories() == null) {
-            return false;
-        }
-
-        return !Sets.intersection(item1.getRankItemSubCategories().keySet(),
-                        item2.getRankItemSubCategories().keySet())
-                .isEmpty();
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/SimpleMergeQueue.java

@@ -1,36 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.apache.commons.lang3.tuple.Pair;
-
-import java.util.*;
-
-
-public class SimpleMergeQueue extends StrategyQueue {
-    public SimpleMergeQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
-        return MergeUtils.simpleMergeWithProtection(items, rankerItemsListMap, recNum, expId);
-    }
-
-    @Override
-    public int candidate(Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
-        return simpleCandidateByPercentage(candidateMap, recallNum, user, requestData, requestIndex, expId);
-    }
-
-    @Override
-    public String toString() {
-        return "SimpleMergeQueue{" +
-                "rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueue.java

@@ -1,194 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.google.common.collect.Maps;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import lombok.Data;
-import org.apache.commons.lang3.tuple.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.*;
-
-@Data
-public abstract class StrategyQueue {
-    // 得等召回结果，定义为RankItem
-    protected final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsList = new HashMap<>();
-    private final Logger LOGGER = LoggerFactory.getLogger(StrategyQueue.class);
-    protected Map<String, StrategyQueue> children = Maps.newHashMap();
-
-    protected List<RankItem> items = new ArrayList<>();
-    private StrategyQueueInfo strategyQueueInfo;
-    private List<StrategyQueue> allQueues = null;
-
-    public StrategyQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        this.strategyQueueInfo = strategyQueueInfo;
-
-        // 自顶向下构建child queue 初始化
-        for (String childName : strategyQueueInfo.getChildren()) {
-            if (!strategyQueueConfig.getStrategyQueueInfoMap().containsKey(childName)) {
-                LOGGER.error("No config for Child queue [{}]", childName);
-                continue;
-            }
-            StrategyQueueInfo childQueueInfo = strategyQueueConfig.getStrategyQueueInfoMap().get(childName);
-
-            StrategyQueue childQueue = null;
-            String childQueueClass = childQueueInfo.getQueueClass();
-            try {
-                childQueue = (StrategyQueue) Class.forName(childQueueClass)
-                        .getConstructor(childQueueInfo.getClass(), strategyQueueConfig.getClass())
-                        .newInstance(childQueueInfo, strategyQueueConfig);
-            } catch (InstantiationException e) {
-                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
-            } catch (IllegalAccessException e) {
-                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
-            } catch (ClassNotFoundException e) {
-                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
-            } catch (NoSuchMethodException e) {
-                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
-            } catch (InvocationTargetException e) {
-                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
-            }
-            if (childQueue != null) {
-                putChild(childName, childQueue);
-            } else {
-                LOGGER.error("construct child queue [{}] failed", childName);
-            }
-        }
-
-    }
-
-    public List<StrategyQueue> getAllQueues() {
-        if (allQueues == null) {
-            allQueues = new LinkedList<>();
-
-            if (!strategyQueueInfo.isLeaf()) {
-                for (Map.Entry<String, StrategyQueue> child : children.entrySet()) {
-                    allQueues.addAll(child.getValue().getAllQueues());
-                }
-            }
-
-            allQueues.add(this);
-        }
-        return allQueues;
-    }
-
-
-    private void putChild(String queueName, StrategyQueue child) {
-        children.put(queueName, child);
-    }
-
-
-    public void clearItems() {
-        items.clear();
-    }
-
-    public void addItem(RankItem item) {
-        this.items.add(item);
-    }
-
-    public abstract int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId);
-
-    // 定义索引Key 生成方式
-    public abstract int candidate(Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId);
-
-    public final int merge(final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
-        this.beforeMerge(recNum, user, requestData, requestIndex, expId);
-        this.doMerge(rankerItemsList, recNum, user, requestData, requestIndex, expId);
-        this.doDeDup();
-        return this.afterMerge();
-    }
-
-    // Merge 前从每个Queue中获取Item
-    // 放入RankerItem中
-    public void beforeMerge(final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
-        if (strategyQueueInfo.isLeaf()) {
-            LOGGER.debug("leaf queue [{}] with [{}] items to merge", getStrategyQueueInfo().getQueueName(), items.size());
-        } else {
-            rankerItemsList.clear();
-            clearItems();
-            int queueCount = 0, itemCount = 0;
-            for (MergeRule rule : strategyQueueInfo.getRulesList()) {
-                if (rule.isDisabled(expId)) {
-                    continue;
-                }
-                int myRecNum = Math.min(recNum, rule.maxMergeNum);
-                //递归的执行子节点的merge逻辑
-                int actualRecNum = children.get(rule.queueName).merge(myRecNum, user, requestData, requestIndex, expId);
-                queueCount += 1;
-                itemCount += actualRecNum;
-                rankerItemsList.put(rule.queueName, Pair.of(rule, children.get(rule.queueName).getItems()));
-            }
-            LOGGER.debug(getStrategyQueueInfo().getQueueName() + " merge info: [{}] queues with [{}] items to merge", queueCount, itemCount);
-        }
-    }
-
-    /**
-     * id dedup 去重操作
-     *
-     * @return
-     */
-    public final int doDeDup() {
-        List<RankItem> pureItems = new ArrayList<RankItem>();
-        Set<String> deDupItems = new HashSet<String>();
-        for (RankItem item : items) {
-            if (deDupItems.contains(item.getId())) {
-                continue;
-            }
-            deDupItems.add(item.getId());
-            pureItems.add(item);
-        }
-        items = pureItems;
-        return items.size();
-    }
-
-    // 给item增加融合队列名称
-    public int afterMerge() {
-        int n = items.size();
-        for (RankItem item : items) {
-            item.addMergeQueuePath(getStrategyQueueInfo().getQueueName());
-        }
-        LOGGER.debug("number of items after [{}] merge: [{}]", getStrategyQueueInfo().getQueueName(), n);
-        return n;
-    }
-
-    // 读取子队列的merge规则，按队列生成candidate
-    protected int simpleCandidateByPercentage(Map<String, Candidate> candidateMap, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int n = 0;
-
-        for (MergeRule rule : strategyQueueInfo.getRulesList()) {
-            if (rule.isDisabled(expId)) {
-                continue;
-            }
-
-            int myRecallNum = (int) (recallNum * rule.recallPercentage);
-            if (!children.containsKey(rule.queueName)) {
-                LOGGER.error("Rule for not exist child queue [{}]", rule.queueName);
-            }
-
-            Map<String, Candidate> myCandidates = new HashMap<>();
-            StrategyQueue strategyQueue = children.get(rule.queueName);
-            if (strategyQueue != null) {
-                n += strategyQueue.candidate(myCandidates, myRecallNum, user, requestData, requestIndex, expId);
-            }
-            candidateMap.putAll(myCandidates);
-        }
-
-        return n;
-    }
-
-    @Override
-    public String toString() {
-        return "StrategyQueue{" +
-                "rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                ", strategyQueueInfo=" + strategyQueueInfo +
-                ", allQueues=" + allQueues +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueueConfig.java

@@ -1,229 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigObject;
-import com.typesafe.config.ConfigValue;
-import lombok.Data;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.*;
-
-@Data
-public class StrategyQueueConfig {
-    private final Logger LOGGER = LoggerFactory.getLogger(StrategyQueueConfig.class);
-
-    private static final String CHILDREN_STRING = "children";
-    private static final String STRATEGY_QUEUE_CLASS_CONF_STRING = "class";
-
-    public static final String DEFAULT_STRATEGY_CLASS_NAME = "com.tzld.piaoquan.recommend.server.framework.merger.SimpleMergeQueue";
-
-    private static final String MERGE_RULE_STRING = "merge-rule";
-    private static final String QUEUE_NAME_STRING = "queue-name";
-    private static final String RECALL_PERCENTAGE_STRING = "recall-percentage";
-    private static final String ENABLE_EXPERIMENT_LIST_STRING = "enable-exp";
-    private static final String DISABLE_EXPERIMENT_LIST_STRING = "disable-exp";
-    private static final String MIN_MERGE_NUM_STRING = "min-merge-num";
-    private static final String MAX_MERGE_NUM_STRING = "max-merge-num";
-
-    private static final Set<String> RESERVED_PROPERTIES = new HashSet<String>(Arrays.asList(QUEUE_NAME_STRING, RECALL_PERCENTAGE_STRING, ENABLE_EXPERIMENT_LIST_STRING, DISABLE_EXPERIMENT_LIST_STRING, MIN_MERGE_NUM_STRING, MAX_MERGE_NUM_STRING));
-
-    Map<String, StrategyQueueInfo> strategyQueueInfoMap;
-
-
-    public boolean load(Config config) {
-        config.checkValid(config, "queue-config");
-        Config queueConf = config.getConfig("queue-config");
-        if (loadMergeQueues(queueConf)) {
-            LOGGER.debug("Load queues config success");
-        } else {
-            LOGGER.error("Load queues config failed");
-            return false;
-        }
-
-        config.checkValid(config, "rule-config");
-        Config rulesConf = config.getConfig("rule-config");
-        if (loadMergeRules(rulesConf)) {
-            LOGGER.debug("Load rules config success");
-        } else {
-            LOGGER.error("Load rules config failed");
-            return false;
-        }
-        // TODO: validate queue structure & rules
-        for (String queueName : strategyQueueInfoMap.keySet()) {
-            StrategyQueue queue = null;
-            try {
-                queue = MergeUtils.constructStrategyQueue(queueName, this);
-            } catch (ClassNotFoundException e) {
-                e.printStackTrace();
-            } catch (NoSuchMethodException e) {
-                e.printStackTrace();
-            } catch (IllegalAccessException e) {
-                e.printStackTrace();
-            } catch (InvocationTargetException e) {
-                e.printStackTrace();
-            } catch (InstantiationException e) {
-                e.printStackTrace();
-            }
-
-            if (queue == null) {
-                LOGGER.error("Queue config info & rules error");
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    public boolean load(String configFile) {
-        Config queueConfig = ConfigFactory.parseResources(configFile);
-        return load(queueConfig);
-    }
-
-    private boolean loadMergeQueues(Config queueConf) {
-        strategyQueueInfoMap = new HashMap<String, StrategyQueueInfo>();
-
-        ConfigObject confObj = queueConf.root();
-        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
-            String myName = it.getKey();
-            Config myConf = ((ConfigObject) it.getValue()).toConfig();
-            if (loadMergeQueueInfo(myName, myConf)) {
-                LOGGER.debug("Load queue [{}] info success", myName);
-            } else {
-                LOGGER.error("Load queue [{}] info failed", myName);
-                return false;
-            }
-        }
-        return true;
-    }
-
-    private boolean loadMergeQueueInfo(String name, Config conf) {
-        LOGGER.debug("Start load queue [{}]", name);
-        if (!strategyQueueInfoMap.containsKey(name)) {
-            strategyQueueInfoMap.put(name, new StrategyQueueInfo(name));
-        }
-        StrategyQueueInfo mergeQueueInfo = strategyQueueInfoMap.get(name);
-
-        if (conf.hasPath(STRATEGY_QUEUE_CLASS_CONF_STRING)) {
-            mergeQueueInfo.setQueueClass(conf.getString(STRATEGY_QUEUE_CLASS_CONF_STRING));
-        } else {
-            mergeQueueInfo.setQueueClass(DEFAULT_STRATEGY_CLASS_NAME);
-        }
-
-        // 递归的把child节点放在queue中
-        if (conf.hasPath(CHILDREN_STRING)) {
-            ConfigObject childrenConf = conf.getObject(CHILDREN_STRING);
-            for (ConfigObject.Entry<String, ConfigValue> it : childrenConf.entrySet()) {
-                String key = it.getKey();
-                Config childConf = ((ConfigObject) it.getValue()).toConfig();
-                String childName = key;
-
-                boolean isLoadSuccess = loadMergeQueueInfo(childName, childConf);
-                if (isLoadSuccess) {
-                    LOGGER.debug("Load queue [{}] SUCCESS", childName);
-                } else {
-                    LOGGER.error("Load queue [{}] FAILED: [{}]", childName, it.getValue());
-                    return false;
-                }
-
-                mergeQueueInfo.addChild(childName);
-            }
-        } else {
-            mergeQueueInfo.setLeaf(true);
-        }
-
-        LOGGER.debug("Load queue [{}] success", name);
-        return true;
-    }
-
-    private boolean loadMergeRules(Config conf) {
-        ConfigObject confObj = conf.root();
-
-        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
-            String myName = it.getKey();
-            Config subTreeConf = ((ConfigObject) it.getValue()).toConfig();
-            boolean isParseSuccess = parseMergeRule(myName, subTreeConf);
-            if (isParseSuccess) {
-                LOGGER.debug("Parse rules for queue [{}] SUCCESS", myName);
-            } else {
-                LOGGER.error("Parse rules for queue [{}] FAILED: [{}]", myName, it.getValue());
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    private boolean parseMergeRule(String name, Config ruleConf) {
-        if (!strategyQueueInfoMap.containsKey(name)) {
-            LOGGER.error("skip rules for not exist queue [{}]", name);
-            return true;
-        } else {
-            LOGGER.debug("parse rules for queue [{}]", name);
-        }
-
-        if (!ruleConf.hasPath(MERGE_RULE_STRING)) {
-            LOGGER.debug("no merge rules for queue [{}]", name);
-            return true;
-        }
-
-        ConfigObject ruleConfObj = ruleConf.getObject(MERGE_RULE_STRING);
-
-        for (ConfigObject.Entry<String, ConfigValue> it : ruleConfObj.entrySet()) {
-            String ruleName = it.getKey();
-            MergeRule rule = new MergeRule();
-
-            Config myConf = ((ConfigObject) it.getValue()).toConfig();
-
-            rule.recallPercentage = getOrDefault(myConf, RECALL_PERCENTAGE_STRING, 0.1);
-            rule.minMergeNum = getOrDefault(myConf, MIN_MERGE_NUM_STRING, 0);
-            rule.maxMergeNum = getOrDefault(myConf, MAX_MERGE_NUM_STRING, 500);
-
-            // 子队列名规则: 用 queue-name 显式指定, 默认为对应规则名
-            rule.queueName = getOrDefault(myConf, QUEUE_NAME_STRING, ruleName);
-            if (myConf.hasPath(ENABLE_EXPERIMENT_LIST_STRING)) {
-                List<Integer> expIds = myConf.getIntList(ENABLE_EXPERIMENT_LIST_STRING);
-                rule.enableExpIdSet = new HashSet<Integer>(expIds);
-            }
-            if (myConf.hasPath(DISABLE_EXPERIMENT_LIST_STRING)) {
-                List<Integer> expIds = myConf.getIntList(DISABLE_EXPERIMENT_LIST_STRING);
-                rule.disableExpIdSet = new HashSet<Integer>(expIds);
-            }
-
-            for (ConfigObject.Entry<String, ConfigValue> properties : myConf.entrySet()) {
-                if (!RESERVED_PROPERTIES.contains(properties.getKey())) {
-                    rule.putProperty(properties.getKey(), myConf.getString(properties.getKey()));
-                }
-            }
-
-            if (strategyQueueInfoMap.get(name).getChildren().contains(rule.queueName)) {
-                strategyQueueInfoMap.get(name).addRule(rule);
-            }
-        }
-        return true;
-    }
-
-    private int getOrDefault(Config conf, String path, int defaultValue) {
-        if (conf.hasPath(path)) {
-            return conf.getInt(path);
-        }
-        return defaultValue;
-    }
-
-    private String getOrDefault(Config conf, String path, String defaultValue) {
-        if (conf.hasPath(path)) {
-            return conf.getString(path);
-        }
-        return defaultValue;
-    }
-
-    private double getOrDefault(Config conf, String path, double defaultValue) {
-        if (conf.hasPath(path)) {
-            return conf.getDouble(path);
-        }
-        return defaultValue;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueueInfo.java

@@ -1,39 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import lombok.Data;
-
-import java.util.*;
-
-
-/**
- * 每一个召回 or 队列的融合配置信息
- */
-
-@Data
-public class StrategyQueueInfo {
-    private String queueName;
-
-    private boolean isLeaf = false;
-
-    private String queueClass; // Full path of StrategyQueue Class
-
-    private List<MergeRule> rulesList;
-
-    private Set<String> children;
-
-    public StrategyQueueInfo(String name) {
-        this.queueName = name;
-        rulesList = new LinkedList<MergeRule>();
-        children = new HashSet<String>();
-    }
-
-    public void addRule(MergeRule rule) {
-        this.rulesList.add(rule);
-    }
-
-    public void addChild(String childName) {
-        this.children.add(childName);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/GBDTModel.java

@@ -1,196 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.util.*;
-
-
-
-public class GBDTModel extends Model {
-    private static final Logger LOGGER = LoggerFactory.getLogger(GBDTModel.class);
-    private static final float LINEAR_TRANSFORM_LOWER_BOUND = 0f;
-    private static final float LINEAR_TRANSFORM_SLOPE = 1f;
-    private int featureCount = 0;
-    private Map<String, Integer> featureIdMap = null;
-    private List<HashMap<Integer, Node>> boosterModel = null;
-
-    private float transform(final float score) {
-        return (score - LINEAR_TRANSFORM_LOWER_BOUND) * LINEAR_TRANSFORM_SLOPE;
-    }
-
-    private int getFeatureId(final String feature) {
-        return featureIdMap.containsKey(feature) ? featureIdMap.get(feature) : -1;
-    }
-
-    @Override
-    public int getModelSize() {
-        return boosterModel.size();
-    }
-
-    @Override
-    public boolean loadFromStream(InputStreamReader in) throws Exception {
-        List<HashMap<Integer, Node>> model = new ArrayList<HashMap<Integer, Node>>();
-        HashMap<Integer, Node> tree = null;
-
-        featureIdMap = new HashMap<String, Integer>();
-        featureCount = 0;
-
-        BufferedReader input = new BufferedReader(in);
-        String line;
-        while ((line = input.readLine()) != null) {
-            String[] tokens = line.trim().split(":");
-            if (tokens.length == 1) {
-                if (tokens[0].startsWith("booster")) {
-                    if (tree != null && tree.size() > 0) {
-                        model.add(tree);
-                    }
-
-                    tree = new HashMap<Integer, Node>();
-                }
-            } else if (tokens.length == 2) {
-                Node node = new Node();
-
-                Integer id = Integer.parseInt(tokens[0]);
-
-                String[] items = tokens[1].split(" ");
-                if (items.length == 1) {
-                    node.isLeaf = true;
-
-                    String[] dt = items[0].split("=");
-                    node.value = Float.parseFloat(dt[1]);
-                } else if (items.length == 2) {
-                    node.isLeaf = false;
-                    String[] fieldDescriptions = null;
-                    node.compareType = NodeCompareType.LT;
-                    if (items[0].substring(1, items[0].length() - 1).contains("<=")) {
-                        fieldDescriptions = items[0].substring(1, items[0].length() - 1).split("<=");
-                        node.compareType = NodeCompareType.LE;
-                    } else {
-                        fieldDescriptions = items[0].substring(1, items[0].length() - 1).split("<");
-                    }
-                    // feature to id
-                    int featureId = getFeatureId(fieldDescriptions[0]);
-                    if (featureId < 0) {
-                        featureIdMap.put(fieldDescriptions[0], featureCount);
-                        featureId = featureCount;
-                        featureCount += 1;
-                    }
-                    node.splitFeatureId = featureId;
-
-                    if (fieldDescriptions.length == 1) {
-                        node.fieldType = NodeFieldType.BINARY;
-                    } else {
-                        node.fieldType = NodeFieldType.QUANTITATIVE;
-                        node.splitCondition = Float.parseFloat(fieldDescriptions[1]);
-                    }
-
-                    String[] childrenDescriptions = items[1].split(",");
-                    for (String childDescription : childrenDescriptions) {
-                        String[] descs = childDescription.split("=");
-                        if (descs[0].equals("yes")) {
-                            node.yes = Integer.parseInt(descs[1]);
-                        } else if (descs[0].equals("no")) {
-                            node.no = Integer.parseInt(descs[1]);
-                        } else if (descs[0].equals("missing")) {
-                            node.missing = Integer.parseInt(descs[1]);
-                        }
-                    }
-                }
-                tree.put(id, node);
-            }
-        }
-        if (tree != null && !tree.isEmpty()) {
-            model.add(tree);
-        }
-
-        LOGGER.info("Boosted tree model load over and tree number is " + model.size());
-        input.close();
-        in.close();
-        if (model != null && model.size() > 0) {
-            boosterModel = model;
-            return true;
-        } else {
-            return false;
-        }
-    }
-
-    private float score(final Map<Integer, Node> tree, final boolean[] featureIsExists, final Float[] featureValues) {
-        int id = 0;
-        while (true) {
-            Node currentNode = tree.get(id);
-            if (currentNode.isLeaf) {
-                return currentNode.value;
-            }
-
-            if (currentNode.fieldType == NodeFieldType.BINARY) {
-                if (!featureIsExists[currentNode.splitFeatureId]) {
-                    id = currentNode.no;
-                } else {
-                    id = currentNode.yes;
-                }
-            } else if (currentNode.fieldType == NodeFieldType.QUANTITATIVE) {
-                if (!featureIsExists[currentNode.splitFeatureId]) {
-                    id = currentNode.missing;
-                } else {
-                    Float value = featureValues[currentNode.splitFeatureId];
-
-                    if ((currentNode.compareType == NodeCompareType.LT && value < currentNode.splitCondition)
-                            || (currentNode.compareType == NodeCompareType.LE && value <= currentNode.splitCondition)) {
-                        id = currentNode.yes;
-                    } else {
-                        id = currentNode.no;
-                    }
-                }
-            } else {
-                LOGGER.error("Reach undefined condition: {}", id);
-            }
-        }
-    }
-
-    public Float score(final Map<String, Double> features, Map<String, Double> featuresScore) {
-        Float result = 0f;
-
-        boolean[] featureIsExists = new boolean[featureCount];
-        Float[] featureValues = new Float[featureCount];
-        for (Map.Entry<String, Double> entry : features.entrySet()) {
-            int featureId = getFeatureId(entry.getKey());
-            if (featureId >= 0) {
-                featureIsExists[featureId] = true;
-                featureValues[featureId] = entry.getValue().floatValue();
-            }
-        }
-
-        for (Map<Integer, Node> tree : boosterModel) {
-            result += score(tree, featureIsExists, featureValues);
-        }
-
-        Float transformedResult = transform(result);
-
-        LOGGER.debug("[calc_dwelltime]features: " + Arrays.toString(features.entrySet().toArray()) + ", prediction:" + result + " , transformed:" + transformedResult);
-
-        return transformedResult;
-    }
-
-    private enum NodeFieldType {
-        BINARY, QUANTITATIVE
-    }
-
-    private enum NodeCompareType {
-        LT, LE
-    }
-
-    private class Node {
-
-        boolean isLeaf;
-        float value;
-        NodeFieldType fieldType;
-        NodeCompareType compareType;
-        int splitFeatureId;
-        Float splitCondition;
-        Integer yes, no, missing;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/LRModel.java

@@ -1,169 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-
-import com.tzld.piaoquan.recommend.feature.model.sample.BaseFeature;
-import com.tzld.piaoquan.recommend.feature.model.sample.GroupedFeature;
-import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
-import it.unimi.dsi.fastutil.longs.Long2FloatMap;
-import it.unimi.dsi.fastutil.longs.Long2FloatOpenHashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-
-public class LRModel extends Model {
-    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
-    private static final Logger LOGGER = LoggerFactory.getLogger(LRModel.class);
-    private final int bucketBits = 10;  // power(2, 10) => 1024 个槽位
-    private List<Long2FloatMap> lrModel;
-
-    public LRModel() {
-        //配置不同环境的hdfs conf
-        this.lrModel = constructModel();
-    }
-
-    public List<Long2FloatMap> getLrModel() {
-        return lrModel;
-    }
-
-    public List<Long2FloatMap> constructModel() {
-        List<Long2FloatMap> initModel = new ArrayList<Long2FloatMap>();
-        int buckets = (int) Math.pow(2, bucketBits);
-        for (int i = 0; i < buckets; i++) {
-            Long2FloatMap internalModel = new Long2FloatOpenHashMap();
-            internalModel.defaultReturnValue(0.0f);
-            initModel.add(internalModel);
-        }
-        return initModel;
-    }
-
-    public int getBucket(long featureHash) {
-        return (int) (((featureHash >> bucketBits) << bucketBits) ^ featureHash);
-    }
-
-    public void putFeature(List<Long2FloatMap> model, long featureHash, float weight) {
-        model.get(getBucket(featureHash)).put(featureHash, weight);
-    }
-
-    public float getWeight(List<Long2FloatMap> model, long featureHash) {
-        return model.get(getBucket(featureHash)).get(featureHash);
-    }
-
-    @Override
-    public int getModelSize() {
-        if (this.lrModel == null)
-            return 0;
-        int sum = 0;
-        for (Map<Long, Float> model : this.lrModel) {
-            sum += model.size();
-        }
-        return sum;
-    }
-
-    public void cleanModel() {
-        this.lrModel = null;
-    }
-
-    public Float score(LRSamples lrSamples) {
-        float sum = 0.0f;
-        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
-            GroupedFeature groupedFeature = lrSamples.getFeatures(i);
-            if (groupedFeature != null && groupedFeature.getFeaturesCount() != 0) {
-                for (int j = 0; j < groupedFeature.getFeaturesCount(); j++) {
-                    BaseFeature baseFeature = groupedFeature.getFeatures(j);
-                    if (baseFeature != null) {
-                        float weight = getWeight(this.lrModel, baseFeature.getIdentifier());
-                        baseFeature.toBuilder().setWeight(weight);
-                        sum += weight;
-                    }
-                }
-            }
-        }
-
-        float pro = (float) (1.0f / (1 + Math.exp(-sum)));
-        lrSamples.toBuilder().setPredictCtr(pro);
-        return pro;
-    }
-
-    public Float getWeights(LRSamples lrSamples) {
-        float sum = 0.0f;
-
-        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
-            GroupedFeature gf = lrSamples.getFeatures(i);
-            if (gf != null && gf.getFeatures(i) != null) {
-                for (int j = 0; j < gf.getFeaturesCount(); j++) {
-                    BaseFeature fea = gf.getFeatures(j);
-                    if (fea != null) {
-                        float tmp = getWeight(this.lrModel, fea.getIdentifier());
-                        fea.toBuilder().setWeight(tmp);
-                        sum += tmp;
-                    }
-                }
-            }
-        }
-        lrSamples.toBuilder().setWeight(sum);
-        return sum;
-    }
-
-    /**
-     * 目前模型比较大,分两个阶段load模型
-     * (1). load 8M 模型, 并更新;
-     * (2). load 剩余的模型
-     * 中间提供一段时间有损的打分服务
-     *
-     * @param in
-     * @return
-     * @throws IOException
-     */
-    @Override
-    public boolean loadFromStream(InputStreamReader in) throws IOException {
-
-        List<Long2FloatMap> model = constructModel();
-        BufferedReader input = new BufferedReader(in);
-        String line = null;
-        int cnt = 0;
-
-        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
-        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", lrModel.size(), curTime);
-        //first stage
-        while ((line = input.readLine()) != null) {
-            String[] items = line.split("\t");
-            if (items.length < 2) {
-                continue;
-            }
-
-            putFeature(model, new BigInteger(items[0].trim()).longValue(), Float.valueOf(items[1].trim()).floatValue());
-            if (cnt++ < 10) {
-                LOGGER.debug("fea: " + items[0] + ", weight: " + items[1]);
-            }
-            if (cnt > MODEL_FIRST_LOAD_COUNT) {
-                break;
-            }
-        }
-        //model update
-        this.lrModel = model;
-        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
-        //final stage
-        while ((line = input.readLine()) != null) {
-            String[] items = line.split("\t");
-            if (items.length < 2) {
-                continue;
-            }
-            putFeature(model, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
-        }
-        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
-
-        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
-        input.close();
-        in.close();
-        return true;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/Model.java

@@ -1,11 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-
-import java.io.InputStreamReader;
-
-abstract public class Model {
-    public abstract int getModelSize();
-
-    public abstract boolean loadFromStream(InputStreamReader in) throws Exception;
-}
-

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ModelManager.java

@@ -1,238 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-
-import com.aliyun.oss.OSS;
-import com.aliyun.oss.OSSClientBuilder;
-import com.aliyun.oss.common.auth.CredentialsProvider;
-import com.aliyun.oss.common.auth.DefaultCredentialProvider;
-import com.aliyun.oss.model.OSSObject;
-import com.ctrip.framework.apollo.Config;
-import com.ctrip.framework.apollo.ConfigService;
-import lombok.extern.slf4j.Slf4j;
-
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-
-@Slf4j
-public class ModelManager {
-    private static final int SCHEDULE_PERIOD = 10;
-    private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
-    private static ModelManager instance;
-    Map<String, ModelLoadTask> loadTasks = new HashMap<>();
-    Map<String, String> modelPathMap = new HashMap<>();
-    private OSS client;
-    private String bucketName;
-
-    private final String modelOssEndpoint = "model.oss.internal.endpoint";
-    private final String modelOssAccessKeyId = "model.oss.accessKeyId";
-    private final String modelOssAccessKeySecret = "model.oss.accessKetSecret";
-    private final String modelOssBucketName = "model.oss.bucketName";
-
-    private ModelManager() {
-        // config load
-        Config config = ConfigService.getAppConfig();
-        String endpoint = config.getProperty(modelOssEndpoint, "");
-        String accessKeyId = config.getProperty(modelOssAccessKeyId, "");
-        String accessKetSecret = config.getProperty(modelOssAccessKeySecret, "");
-        // oss client
-        CredentialsProvider credentialsProvider = new DefaultCredentialProvider(accessKeyId, accessKetSecret);
-        this.client = new OSSClientBuilder().build(endpoint, credentialsProvider);
-        this.bucketName = config.getProperty(modelOssBucketName, "");
-
-        config.addChangeListener(changeEvent -> {
-            if (changeEvent.isChanged(modelOssEndpoint)
-                    || changeEvent.isChanged(modelOssAccessKeyId)
-                    || changeEvent.isChanged(modelOssAccessKeySecret)) {
-                String endpointNew = config.getProperty(modelOssEndpoint, "");
-                String accessKeyIdNew = config.getProperty(modelOssAccessKeyId, "");
-                String accessKetSecretNew = config.getProperty(modelOssAccessKeySecret, "");
-                CredentialsProvider credentialsProviderNew = new DefaultCredentialProvider(accessKeyIdNew,
-                        accessKetSecretNew);
-                this.client = new OSSClientBuilder().build(endpointNew, credentialsProviderNew);
-            }
-            if (changeEvent.isChanged(modelOssBucketName)) {
-                this.bucketName = config.getProperty(modelOssBucketName, "");
-            }
-        });
-
-
-        start(SCHEDULE_PERIOD);
-    }
-
-    public static ModelManager getInstance() {
-        if (instance == null) {
-            synchronized (ModelManager.class) {
-                if (instance == null) {
-                    instance = new ModelManager();
-                }
-            }
-        }
-        return instance;
-    }
-
-    /**
-     * 添加一个加载任务到管理器
-     *
-     * @param modelName  Model的名字, 注册到ModelManager的不同model需要不同的名字
-     * @param path       Model在OSS上的全路径
-     * @param modelClass Model的子类型
-     */
-    public void registerModel(String modelName, String path, Class<? extends Model> modelClass) throws ModelRegisterException, IOException {
-        if (modelPathMap.containsKey(modelName)) {
-            // fail fast
-            // throw new RuntimeException(modelName + " already exists");
-            // hard code  广告需要视频模型打分数据，配置要分开
-            return;
-
-        }
-
-        modelPathMap.put(modelName, path);
-        if (loadTasks.containsKey(path)) {
-            ModelLoadTask loadTask = loadTasks.get(path);
-            loadTask.refCount++;
-        } else {
-            ModelLoadTask task = new ModelLoadTask(path, modelClass);
-            task.refCount++;
-            loadTasks.put(path, task);
-            loadModel(task, false, true);
-        }
-    }
-
-    /**
-     * 删除一个加载任务
-     *
-     * @param modelName Model的名字, 需要和registerModel的名字一致
-     */
-    private void unRegisterModel(String modelName) {
-        if (modelPathMap.containsKey(modelName)) {
-            String path = modelPathMap.get(modelName);
-            if (loadTasks.containsKey(path)) {
-                ModelLoadTask task = loadTasks.get(path);
-                task.refCount--;
-                if (task.refCount == 0) {
-                    loadTasks.remove(path);
-                }
-            }
-            modelPathMap.remove(modelName);
-        }
-    }
-
-    /**
-     * @param modelName
-     * @return
-     */
-    public Model getModel(String modelName) {
-        if (modelPathMap.containsKey(modelName) && loadTasks.containsKey(modelPathMap.get(modelName))) {
-            return loadTasks.get(modelPathMap.get(modelName)).model;
-        } else {
-            return null;
-        }
-    }
-
-    /**
-     * 开始调度
-     *
-     * @param period
-     */
-    protected void start(long period) {
-        final Runnable task = new Runnable() {
-            public void run() {
-                // 模型更新开关
-                // boolean modelUpdateSwitch = Configuration.getBoolean("recommend-service-framework.model-update-switch", true);
-                boolean modelUpdateSwitch = true;
-                log.info("model update switch [{}]", modelUpdateSwitch);
-                if (modelUpdateSwitch) {
-                    updateModels(false);
-                }
-            }
-        };
-        scheduler.scheduleAtFixedRate(task, 10, period, TimeUnit.MINUTES); // 10分钟
-    }
-
-    /**
-     * 更新模型
-     */
-    public void updateModels(final boolean isForceLoads) {
-        log.info("begin to update: [{}]", loadTasks.keySet().size());
-        for (String modelPath : loadTasks.keySet()) {
-            log.debug("load task model path [{}]", modelPath);
-            ModelLoadTask task = loadTasks.get(modelPath);
-            loadModel(task, isForceLoads, false);
-        }
-    }
-
-    /**
-     * 检查并加载模型
-     * <p>
-     * 从oss加载：
-     * https://help.aliyun.com/zh/oss/developer-reference/streaming-download-7?spm=a2c4g.11186623.0.0.4b527c7dm8LejC
-     *
-     * @param loadTask
-     */
-    private void loadModel(final ModelLoadTask loadTask, final boolean isForceLoads, final boolean isRegister) {
-        if (loadTask.isLoading) {
-            return;
-        }
-        loadTask.isLoading = true;
-        OSSObject ossObj = null;
-        try {
-            ossObj = client.getObject(bucketName, loadTask.path);
-            long timeStamp = ossObj.getObjectMetadata().getLastModified().getTime();
-            if (loadTask.lastModifyTime < timeStamp || isForceLoads) {
-                log.info("model file changed: [{}], ready to update, last modify: [{}], current model time: [{}]",
-                        loadTask.path, loadTask.lastModifyTime, timeStamp);
-
-                Model model = loadTask.modelClass.newInstance();
-                if (model.loadFromStream(new InputStreamReader(ossObj.getObjectContent()))) {
-                    loadTask.model = model;
-                    loadTask.lastModifyTime = timeStamp;
-                }
-            }
-            ossObj.close();
-        } catch (Exception e) {
-            log.error("update model fail", e);
-        } finally {
-            loadTask.isLoading = false;
-            if (ossObj != null) {
-                try {
-                    ossObj.close();
-                } catch (IOException e) {
-                    log.error("close ossObj fail", e);
-                }
-            }
-        }
-    }
-
-    public class ModelRegisterException extends Exception {
-
-        public ModelRegisterException(String s) {
-            super(s);
-        }
-    }
-
-    /**
-     * 调度的任务单元
-     */
-    private class ModelLoadTask {
-
-        private int refCount;
-        private final String path;
-        private long lastModifyTime;
-        private boolean isLoading;
-        private final Class<? extends Model> modelClass;
-        private Model model;
-
-        ModelLoadTask(String path, Class<? extends Model> modelClass) {
-            this.refCount = 0;
-            this.path = path;
-            this.lastModifyTime = 0;
-            this.modelClass = modelClass;
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/OssConfig.java

@@ -1,15 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-import lombok.Data;
-
-/**
- * @author dyp
- */
-@Data
-public class OssConfig {
-
-    private String accessKeyId;
-    private String accessKeySecret;
-    private String endpoint;
-    private String bucketName;
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ThompsonSamplingModel.java

@@ -1,89 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.base.VideoActionFeature;
-import org.apache.commons.math3.distribution.BetaDistribution;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.math.BigInteger;
-import java.util.HashMap;
-import java.util.Map;
-
-
-public class ThompsonSamplingModel extends Model {
-    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25;  // 32M
-    private static final Logger LOGGER = LoggerFactory.getLogger(ThompsonSamplingModel.class);
-
-    // key = videoid, value = < push, exp, play, realplay, share, retures >
-    private Map<Long, VideoActionFeature> thompsonSamplingModel;
-
-    private static final int alpha = 20;
-    private static final int beta_returns = 100;
-
-    public ThompsonSamplingModel() {
-        //配置不同环境的hdfs conf
-        this.thompsonSamplingModel = new HashMap<>();
-    }
-
-    public Map<Long, VideoActionFeature> getThompsonSamplingModel() {
-        return this.thompsonSamplingModel;
-    }
-
-
-
-
-    @Override
-    public boolean loadFromStream(InputStreamReader in) throws IOException {
-        Map<Long, VideoActionFeature> initModel = new HashMap<>();
-        BufferedReader input = new BufferedReader(in);
-        String line = null;
-        int cnt = 0;
-        while ((line = input.readLine()) != null) {
-            String[] items = line.split("\t");
-            if (items.length < 3) {
-                continue;
-            }
-            Long videoId = new BigInteger(items[0].trim()).longValue();
-            VideoActionFeature videoFeature = new VideoActionFeature();
-            videoFeature.setView(Double.valueOf(items[1].trim()));
-            videoFeature.setPlay(Double.valueOf(items[2].trim()));
-            videoFeature.setShare(Double.valueOf(items[3].trim()));
-            videoFeature.setReturns(Double.valueOf(items[5].trim()));
-            initModel.put(videoId, videoFeature);
-        }
-
-        this.thompsonSamplingModel = initModel;
-        input.close();
-        in.close();
-        return true;
-    }
-
-    @Override
-    public int getModelSize() {
-        if (this.thompsonSamplingModel == null)
-            return 0;
-        int sum = this.thompsonSamplingModel.size();
-        return sum;
-    }
-
-    public double score(RankItem rankItem) {
-        double score = 0.0f;
-        VideoActionFeature videoActionFeature = this.thompsonSamplingModel.getOrDefault(rankItem.getVideoId(), new VideoActionFeature());
-
-        int alpha = (int) videoActionFeature.getReturns() + ThompsonSamplingModel.alpha;
-        int beta = beta_returns + (int) videoActionFeature.getView();
-        score = this.betaSampler(alpha, beta);
-        return score;
-    }
-
-    public double betaSampler(double alpha, double beta) {
-        BetaDistribution betaSample = new BetaDistribution(alpha, beta);
-        return betaSample.sample();
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/AbstractFilter.java

@@ -1,31 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-public abstract class AbstractFilter<T> {
-    protected final static Logger LOGGER = LoggerFactory.getLogger(AbstractFilter.class);
-    protected final FilterConfigInfo filterConfigInfo;
-    protected final RecommendRequest requestContext;
-    protected final User user;
-
-    public AbstractFilter(FilterConfigInfo filterConfigInfo,
-                          RecommendRequest requestContext,
-                          User user) {
-
-        this.filterConfigInfo = filterConfigInfo;
-        this.requestContext = requestContext;
-        this.user = user;
-    }
-
-    public FilterConfigInfo getFilterConfigInfo() {
-        return filterConfigInfo;
-    }
-
-    public abstract void doFilter(Candidate candidate, List<T> t);
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/BaseRecaller.java

@@ -1,280 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller;
-
-
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.*;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.provider.QueueProvider;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
-import java.util.stream.Collectors;
-
-
-public class BaseRecaller<Video> {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(BaseRecaller.class);
-    private static final long DEFAULT_QUEUE_LOAD_TIMEOUT = 1500; // ms
-    private static final long DEFAULT_PARALLEL_FILTER_TIMEOUT = 1500; // ms
-
-    private static final String FILTER_CONF = "filter_config.conf"; // ms
-    private static final ExecutorService filterExecutorService = new ThreadPoolExecutor(128, 128,
-            0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1000), r -> new Thread(r, "filterExecutorService"),
-            new ThreadPoolExecutor.CallerRunsPolicy());
-
-    private static final ExecutorService fetchQueueExecutorService = new ThreadPoolExecutor(128, 128,
-            0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1000), r -> new Thread(r, "fetchQueueExecutorService"),
-            new ThreadPoolExecutor.CallerRunsPolicy());
-    private final QueueProvider<Video> queueProvider;
-
-    private static final FilterConfig filterConfig;
-
-    private final long QUEUE_LOAD_TIMEOUT;
-
-    static {
-        filterConfig = new FilterConfig(FILTER_CONF);
-    }
-
-    public BaseRecaller(QueueProvider<Video> queueProvider) {
-        this(queueProvider, DEFAULT_QUEUE_LOAD_TIMEOUT);
-    }
-
-    public BaseRecaller(QueueProvider<Video> queueProvider,
-                        long queueLoadTimeout) {
-        this.queueProvider = queueProvider;
-        this.QUEUE_LOAD_TIMEOUT = queueLoadTimeout;
-    }
-
-
-    public BaseRecaller(QueueProvider<Video> queueProvider,
-                        FilterConfig filterConfig,
-                        long queueLoadTimeout) {
-        this.queueProvider = queueProvider;
-        this.QUEUE_LOAD_TIMEOUT = queueLoadTimeout;
-    }
-
-
-    public String extractItemId(Entry<Video> entry) {
-        return entry.id;
-    }
-
-
-    public boolean isValidItem(Video item) {
-        return item != null;
-    }
-
-    /**
-     * 把queue中的entry, 补充candidiate信息，放入RankItem中
-     *
-     * @param entries
-     * @param candidate
-     * @return
-     */
-    private List<RankItem> toHits(final Iterable<Entry<Video>> entries, final Candidate candidate) {
-
-        List<RankItem> result = new ArrayList<RankItem>();
-        for (Entry entry : entries) {
-            RankItem item = new RankItem();
-            item.setId(extractItemId(entry));
-            item.putRankerScore("L3Score", (Double) entry.scores.get("ordering"));
-            item.setQueue(candidate.getMergeQueueName()); // merge queue
-
-            CandidateInfo candidateInfo = new CandidateInfo();
-            candidateInfo.setCandidateQueueName(candidate.getCandidateKey());
-            candidateInfo.setCandidate(candidate);
-
-            item.setCandidateInfo(candidateInfo);
-            item.addToCandidateInfoList(candidateInfo);
-            result.add(item);
-        }
-        return result;
-    }
-
-
-    // 读取redis中的数据放入queue中
-    public Map<Candidate, Queue<Video>> loadQueues(List<Candidate> candidates) {
-        // update queueName
-        Iterable<Candidate> updateCandidates = FluentIterable.from(candidates).transform(candidate -> {
-            try {
-                long ttl = QueueName.DEFAULT_LOCAL_CACHE_TTL;
-                candidate.setCandidateQueueName(QueueName.fromString(candidate.getCandidateKey(), ttl));
-            } catch (Exception e) {
-                candidate.setCandidateQueueName(null);
-                LOGGER.error("error parse QueueName [{}]", candidate.getCandidateKey());
-            }
-            return candidate;
-        });
-
-        // parse queues
-        Iterable<QueueName> queueNames = FluentIterable.from(updateCandidates).transform(candidate -> candidate.getCandidateQueueName());
-
-        // parallel load queues, redis 或者缓存获取index
-        Map<QueueName, Queue<Video>> queues = Maps.newConcurrentMap();
-        try {
-            // 格式
-            queues = queueProvider.loads(Lists.newArrayList(queueNames), QUEUE_LOAD_TIMEOUT, TimeUnit.MILLISECONDS);
-        } catch (Exception e) {
-            LOGGER.error("load queue occur error [{}]", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        // parse candidate map
-        Map<Candidate, Queue<Video>> candidateQueueMap = Maps.newConcurrentMap();
-        for (Candidate candidate : updateCandidates) {
-            QueueName name = candidate.getCandidateQueueName();
-            Queue<Video> entries = queues.get(name);
-            if (queues.containsKey(name) && entries != null) {
-                candidateQueueMap.put(candidate, entries);
-            }
-        }
-        return candidateQueueMap;
-    }
-
-
-    /**
-     * recall
-     *
-     * @param requestData
-     * @param user
-     * @param recallCandidates
-     * @return
-     */
-    public List<RankItem> recalling(final RecommendRequest requestData, final User user, List<Candidate> recallCandidates) {
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        stopwatch.reset().start();
-        // load from redis
-        List<Callable<Map<Candidate, Queue<Video>>>> fetchQueueCalls = Lists.newArrayList();
-        fetchQueueCalls.add(() -> {
-            boolean isFromRedis = true;
-            return obtainQueue(recallCandidates, requestData, user, isFromRedis);
-        });
-
-        // 多线程执行load
-        List<Future<Map<Candidate, Queue<Video>>>> fetchQueueFutures = null;
-        try {
-            fetchQueueFutures = fetchQueueExecutorService.invokeAll(fetchQueueCalls, DEFAULT_QUEUE_LOAD_TIMEOUT,
-                    TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("[fetch queue error] inter fail: {}", ExceptionUtils.getStackTrace(e));
-        } catch (Exception e) {
-            LOGGER.error("[fetch queue error] ex", ExceptionUtils.getStackTrace(e));
-        }
-
-        Map<Candidate, Queue<Video>> candidateQueueMap = Maps.newHashMap();
-        if (CollectionUtils.isNotEmpty(fetchQueueFutures)) {
-            for (Future<Map<Candidate, Queue<Video>>> future : fetchQueueFutures) {
-                if (future.isDone() && !future.isCancelled()) {
-                    Map<Candidate, Queue<Video>> result = null;
-                    try {
-                        result = future.get();
-                    } catch (InterruptedException e) {
-                        LOGGER.error("[fetch queue error] InterruptedException {}", ExceptionUtils.getStackTrace(e));
-                    } catch (ExecutionException e) {
-                        LOGGER.error("[fetch queue error] ex {}", ExceptionUtils.getStackTrace(e));
-                    }
-                    if (result != null) {
-                        candidateQueueMap.putAll(result);
-                    }
-                }
-            }
-        }
-
-        stopwatch.reset().start();
-        List<RankItem> result = convertToRankItem(candidateQueueMap, requestData, user);
-
-        return result;
-    }
-
-
-    // 转成RankItem
-    // 同时给Filter预留处理
-    private List<RankItem> convertToRankItem(Map<Candidate, Queue<Video>> candidateQueueMap,
-                                             RecommendRequest requestData,
-                                             User user) {
-
-
-        final FilterPipeline<Video> recallFilter = new FilterPipeline<>(filterConfig, requestData, user);
-
-        final List<Callable<List<RankItem>>> callables = new ArrayList<>();
-        for (final Map.Entry<Candidate, Queue<Video>> entry : candidateQueueMap.entrySet()) {
-            callables.add(() -> {
-                List<RankItem> candidateHits = new ArrayList<>();
-                final Candidate candidate = entry.getKey();
-                List<Entry<Video>> entriesValue = CommonCollectionUtils.iterableToList(entry.getValue());
-                try {
-                    final Map<Video, Entry<Video>> entryMap = new HashMap<>();
-                    List<Video> entriesList = entriesValue.stream().map(e -> {
-                        entryMap.put(e.item, e);
-                        return e.item;
-                    }).collect(Collectors.toList());
-                    List<Video> entriesAfterFilter = recallFilter.doFilter(candidate, entriesList);
-                    // 1. filter  TODO 待后续增加自定义filter
-                    if (CollectionUtils.isNotEmpty(entriesAfterFilter)) {
-                        List<Entry<Video>> entries = entriesAfterFilter.stream()
-                                .limit(candidate.getCandidateNum())
-                                .map(entryMap::get)
-                                .collect(Collectors.toList());
-                        // 2. toHits
-                        candidateHits.addAll(toHits(entries, candidate));
-                    }
-
-                } catch (Exception e) {
-                    LOGGER.error("recall filter queue occur error, queue [{}], error: [{}]", candidate.toString(), ExceptionUtils.getFullStackTrace(e));
-                }
-                return candidateHits;
-            });
-        }
-
-        Map<String, RankItem> hits = new HashMap<>();
-        try {
-            List<Future<List<RankItem>>> futures = filterExecutorService.invokeAll(callables, DEFAULT_PARALLEL_FILTER_TIMEOUT, TimeUnit.MILLISECONDS);
-            for (Future<List<RankItem>> future : futures) {
-                try {
-                    if (future.isDone() && !future.isCancelled() && future.get() != null) {
-                        List<RankItem> part = future.get();
-                        // add to result
-                        for (RankItem item : part) {
-                            // merge candidate Info
-                            if (hits.containsKey(item.getId())) {
-                                hits.get(item.getId()).addToCandidateInfoList(item.getCandidateInfo());
-                            } else {
-                                hits.put(item.getId(), item);
-                            }
-                        }
-                        LOGGER.debug("collect items from filter task, userid: [{}], item size [{}]", "", part.size());
-                    } else {
-                        LOGGER.error("parallel recall filter Canceled {} ", "");
-                    }
-                } catch (Exception e) {
-                    LOGGER.error("parallel recall filter occur error, uid: [{}], Exception [{}]",
-                            "", ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        } catch (Exception e) {
-            LOGGER.error("parallel recall filter occur error, uid: [{}], Exception [{}]",
-                    "", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        List<RankItem> result = new ArrayList<>(hits.values());
-
-        return result;
-    }
-
-
-    private Map<Candidate, Queue<Video>> obtainQueue(List<Candidate> refactorCandidates, RecommendRequest requestData, User user, boolean isFromRedis) {
-        return loadQueues(refactorCandidates);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterConfig.java

@@ -1,92 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller;
-
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigObject;
-import com.typesafe.config.ConfigValue;
-
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
-
- */
-
-public class FilterConfig {
-    private static Logger LOGGER = LoggerFactory.getLogger(FilterConfig.class);
-    private List<FilterConfigInfo> filterConfigInfoList = new ArrayList<FilterConfigInfo>();
-
-    public FilterConfig(Config config) {
-        this.load(config);
-    }
-
-
-    public FilterConfig(String configFile) {
-        this.load(configFile);
-    }
-
-    public FilterConfig() {
-    }
-
-    public boolean load(String configFile) {
-        Config filterConfig = ConfigFactory.parseResources(configFile);
-        return load(filterConfig);
-    }
-
-    public boolean load(Config config) {
-        Config recallConfig = config.getConfig("recall-config");
-
-        Config filterConf = recallConfig.getConfig("filter-config");
-        try {
-            loadFilters(filterConf);
-        } catch (Exception e) {
-            return false;
-        }
-
-        return true;
-    }
-
-    public List<FilterConfigInfo> getFilterConfigInfoList() {
-        return filterConfigInfoList;
-    }
-
-    private void loadFilters(Config config) throws Exception {
-        ConfigObject confObj = config.root();
-        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
-            Config conf = ((ConfigObject) it.getValue()).toConfig();
-
-            // parse config
-            String configName = it.getKey();
-            String filterName = conf.getString("filter-name");
-            int filterPriority = 0;
-            if (conf.hasPath("filter-priority"))
-                filterPriority = conf.getInt("filter-priority");
-
-            List<Integer> disableExpIds = new ArrayList<Integer>();
-            if (conf.hasPath("disable-expids")) {
-                disableExpIds = conf.getIntList("disable-expids");
-            }
-            FilterConfigInfo filterConfigInfo = new FilterConfigInfo(configName,
-                    filterName, filterPriority, disableExpIds);
-            LOGGER.debug("parse filter config info [{}]", filterConfigInfo);
-
-            addConfigByPriority(filterConfigInfoList, filterConfigInfo);
-        }
-    }
-
-    private void addConfigByPriority(List<FilterConfigInfo> configInfoList, FilterConfigInfo addConfigInfo) {
-
-        int pos = 0;
-        for (; pos < configInfoList.size(); pos++) {
-            if (configInfoList.get(pos).getFilterPriority() <= addConfigInfo.getFilterPriority()) {
-                break;
-            }
-        }
-        configInfoList.add(pos, addConfigInfo);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterConfigInfo.java

@@ -1,59 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller;
-
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.FluentIterable;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
-
-public class FilterConfigInfo {
-    private Set<Integer> disableExpIds;
-    private String filterName;
-    private Integer filterPriority;
-    private String configName;
-
-    public FilterConfigInfo(String configName,
-                            String filterName,
-                            Integer filterPriority,
-                            List<Integer> expIds) {
-        this.configName = configName;
-        this.filterName = filterName;
-        this.filterPriority = filterPriority;
-        this.disableExpIds = new HashSet<Integer>();
-        this.disableExpIds.addAll(expIds);
-    }
-
-    public Integer getFilterPriority() {
-        return filterPriority;
-    }
-
-    public String getFilterName() {
-        return filterName;
-    }
-
-    public Set<Integer> getDisableExpIds() {
-        return disableExpIds;
-    }
-
-    public String getConfigName() {
-        return configName;
-    }
-
-    @Override
-    public String toString() {
-        return configName + ":" + filterPriority + ":" + filterName + ":" +
-                Joiner.on(",").join(FluentIterable.from(disableExpIds).transform(new Function<Integer, String>() {
-                    @Nullable
-                    @Override
-                    public String apply(Integer integer) {
-                        return integer.toString();
-                    }
-                }));
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterPipeline.java

@@ -1,62 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-
-public class FilterPipeline<T> {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(FilterPipeline.class);
-    public int filterNum;
-    private final FilterConfig config;
-    private final List<AbstractFilter<T>> filters;
-    private final RecommendRequest requestContext;
-    private final User user;
-
-    public FilterPipeline(FilterConfig config,
-                          RecommendRequest requestContext,
-                          User user) {
-
-        this.config = config;
-        this.requestContext = requestContext;
-        this.user = user;
-        this.filters = new ArrayList<>();
-        this.constructFilters(config);
-    }
-
-    public List<AbstractFilter<T>> getFilters() {
-        return filters;
-    }
-
-    public void constructFilters(FilterConfig config) {
-        // add filter
-        if (config != null) {
-            for (FilterConfigInfo filterConfigInfo : config.getFilterConfigInfoList()) {
-                try {
-                    AbstractFilter<T> filter = (AbstractFilter) Class.forName(filterConfigInfo.getFilterName())
-                            .getConstructor(FilterConfigInfo.class, RecommendRequest.class, User.class)
-                            .newInstance(filterConfigInfo, this.requestContext, this.user);
-
-                    this.filters.add(filter);
-                } catch (Exception e) {
-                    LOGGER.error("Filter config info construct error, [{}] [{}]",
-                            filterConfigInfo, ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-    }
-
-    public List<T> doFilter(Candidate candidate, List<T> list) {
-        for (AbstractFilter filter : filters) {
-            filter.doFilter(candidate, list);
-        }
-        return list;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/CacheEntry.java

@@ -1,24 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-
-import com.google.common.base.Optional;
-import lombok.Data;
-
-@Data
-public class CacheEntry<T> {
-    private String itemId; // itemId
-    private Optional<T> item;
-    private Long expireTime; // 静态特征失效时间
-    private Long dynamicFeatureExpireTime; // 动态特征失效时间
-
-    public CacheEntry(String itemId,
-                      Long expireTime,
-                      Long dynamicFeatureExpireTime,
-                      Optional<T> item) {
-        this.itemId = itemId;
-        this.expireTime = expireTime;
-        this.dynamicFeatureExpireTime = dynamicFeatureExpireTime;
-        this.item = item;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/Index.java

@@ -1,44 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-import com.alibaba.fastjson.JSONObject;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.Data;
-import org.apache.commons.lang3.StringUtils;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-@Data
-public class Index {
-    private String indexName = null;
-    private List<IndexEntry> indexEntryList = new ArrayList<>();
-
-
-    // TODO
-    // redis中一路召回队列key 的结果 转化到Index中
-    public Index(String indexName, String indexValue) {
-        if (StringUtils.isBlank(indexName) || StringUtils.isBlank(indexValue)) {
-            return;
-        }
-        this.indexName = indexName;
-        JSONObject jsonObject = JSONObject.parseObject(indexValue);
-        String valueList = jsonObject.getString("value_list");
-        valueList = "[" + valueList + "]";
-        List<List<String>> videoScores = JSONUtils.fromJson(valueList, new TypeToken<List<List<String>>>() {
-        }, Collections.emptyList());
-
-        videoScores.stream().forEach(itemAndScore -> {
-                    IndexEntry indexEntry = new IndexEntry();
-                    if (itemAndScore.size() >= 2) {
-                        indexEntry.setItemId(itemAndScore.get(0));
-                        indexEntry.setScore(Double.valueOf(itemAndScore.get(1)));
-                        this.indexEntryList.add(indexEntry);
-                    }
-                }
-        );
-
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/IndexEntry.java

@@ -1,10 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-import lombok.Data;
-
-@Data
-public class IndexEntry {
-    private String itemId;
-    private Double score;
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/QueueProvider.java

@@ -1,20 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-
-
-
-public interface QueueProvider<T> {
-
-    Queue<T> load(QueueName name) throws Exception;
-
-    Index get(QueueName name) throws Exception;
-
-    Map<QueueName, Queue<T>> loads(List<QueueName> names) throws Exception;
-
-    Map<QueueName, Queue<T>> loads(List<QueueName> names, long timeout, TimeUnit timeUnit) throws Exception;
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/RedisBackedQueue.java

@@ -1,196 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Iterables;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.utils.FixedThreadPoolHelper;
-import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
-
-
-public class RedisBackedQueue implements QueueProvider<Video> {
-    private static final Logger logger = LoggerFactory.getLogger(RedisBackedQueue.class);
-    private static final int DEFAULT_TTL = 7 * 24 * 60 * 60;
-    private static final long CACHE_MAXIMUMSIZE = 50 * 10000; // default 50w
-    private static final long CACHE_TIMEOUT_MS = 5 * 60 * 1000L;
-
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(64);
-
-    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
-    private final ExecutorService queueRefreshExecutorService = new FixedThreadPoolHelper(4, "queueRefresh").getThreadPoolExecutor();
-
-    // 不能为static
-    private final BlockingQueue<QueueName> asyncRefreshQueue = new LinkedBlockingQueue<QueueName>(50000);
-    private final RedisSmartClient client;
-    private final LoadingCache<QueueName, Pair<Long, Queue<Video>>> cache;
-
-    /**
-     * 实例化 RedisBackedQueue, 并提供基于Guava Cache的本地缓存管理工作
-     *
-     * @param client         redis-cluster 连接池
-     * @param cacheTimeOutMs 缓存有效期, 开启缓存的情况下,默认缓存5min,以写时间为基准
-     */
-    public RedisBackedQueue(RedisSmartClient client, final long cacheTimeOutMs) {
-        this.client = client;
-
-        cache = CacheBuilder.newBuilder()
-                .expireAfterWrite((long) (cacheTimeOutMs * 1.3), TimeUnit.MILLISECONDS)
-                .maximumSize(CACHE_MAXIMUMSIZE)
-                .build(new RedisBackedQueueWithoutMetaCacheLoader(client));
-
-        // protect key
-        for (int i = 0; i < 4; i++) {
-            queueRefreshExecutorService.execute(new CacheRefreshRunnable());
-        }
-    }
-
-    public RedisBackedQueue(RedisSmartClient client) {
-        this(client, CACHE_TIMEOUT_MS);
-    }
-
-    public RedisSmartClient getClient() {
-        return client;
-    }
-
-
-    // redis读取队列内内容
-    public Index get(QueueName name) throws Exception {
-        String valueJson = client.get(name.toString());
-        Index index = new Index(name.toString(), valueJson);
-        return index;
-    }
-
-
-
-    /**
-     * 加载指定索引, 并指定索引本地缓存的TTL
-     * @param name QueueName
-     * @return
-     * @throws ExecutionException
-     */
-    public Queue<Video> load(QueueName name) throws ExecutionException {
-
-        Pair<Long, Queue<Video>> cachedQueue = cache.get(name);
-        if (cachedQueue == null) {
-            // 清理本地缓存中当前key的数据
-            cache.invalidate(name);
-            logger.debug("invalidate queue [{}]", name);
-            return new Queue<Video>(name.toString());
-        }
-        // update key refresh time
-        if (cachedQueue.getKey() == Long.MIN_VALUE) {
-            cache.put(name, Pair.of(System.currentTimeMillis() + name.getTTL(), cachedQueue.getValue()));
-        }
-
-        if (name.getTTL() == 0) {
-            cache.invalidate(name);
-            logger.debug("invalidate queue [{}]", name);
-        }
-
-        // check and refresh
-        if (name.getTTL() > 0 && System.currentTimeMillis() > cachedQueue.getKey()) {
-            boolean isok = asyncRefreshQueue.offer(name);
-            if (logger.isDebugEnabled()) {
-                logger.debug("offer async queue name [{}], result [{}]", name, isok);
-            }
-            if (asyncRefreshQueue.size() > CACHE_MAXIMUMSIZE) {
-                logger.warn("index backgroud blocking queue length [{}], it is danger", asyncRefreshQueue.size());
-            }
-            if (!isok) {
-                logger.error("index backgroud offer blocking queue error, queuename [{}], with queue lenght [{}]", name, asyncRefreshQueue.size());
-            }
-        }
-
-        return cachedQueue.getValue();
-    }
-
-    public Map<QueueName, Queue<Video>> loads(List<QueueName> names) throws ExecutionException {
-        return this.loads(names, 200, TimeUnit.MILLISECONDS);
-    }
-
-    public Map<QueueName, Queue<Video>> loads(List<QueueName> names, long timeout, TimeUnit timeUnit) throws ExecutionException {
-
-        final Iterable<List<QueueName>> namesIterable = Iterables.partition(names, 1);
-        final List<Callable<Map<QueueName, Queue<Video>>>> callables =
-                new ArrayList<Callable<Map<QueueName, Queue<Video>>>>();
-        for (final List<QueueName> queueNames : namesIterable) {
-            callables.add(() -> {
-                Map<QueueName, Queue<Video>> result = new HashMap<>();
-                for (final QueueName name : queueNames) {
-                    try {
-                        Queue queue = load(name);
-                        result.put(name, queue);
-                    } catch (Exception e) {
-                        logger.error("load queue error [{}] [{}]", name, ExceptionUtils.getFullStackTrace(e));
-                    }
-                }
-                return result;
-            });
-        }
-
-        Map<QueueName, Queue<Video>> loadQueue = new ConcurrentHashMap<QueueName, Queue<Video>>();
-        try {
-            List<Future<Map<QueueName, Queue<Video>>>> futures = executorService.invokeAll(callables);
-            for (Future<Map<QueueName, Queue<Video>>> future : futures) {
-                if (future.isDone() && !future.isCancelled()) {
-                    try {
-                        Map<QueueName, Queue<Video>> ret = future.get();
-                        loadQueue.putAll(ret);
-                    } catch (ExecutionException ee) {
-                        logger.error("Failed to execute load a queue partition for {}", ExceptionUtils.getFullStackTrace(ee));
-                    }
-                }
-            }
-        } catch (InterruptedException ie) {
-            logger.error("Interrupted when waiting for parallel queue load finish for {}", ExceptionUtils.getFullStackTrace(ie));
-        }
-        return loadQueue;
-    }
-
-
-
-    // 使用Guava Cache 的异步刷新接口, 异步刷新到期的key
-    public class CacheRefreshRunnable implements Runnable {
-
-        @Override
-        public void run() {
-            while (true) {
-                QueueName queueName = null;
-                try {
-                    queueName = asyncRefreshQueue.take(); // take async item, blocked when no item
-                    final long startTime = System.currentTimeMillis();
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("async refresh thread get queuename [{}]", queueName); // get itemid
-                    }
-
-                    Pair<Long, Queue<Video>> entry = cache.get(queueName);
-                    // check refresh
-                    if (startTime > entry.getKey()) {
-                        cache.refresh(queueName);
-                    }
-
-                    long endTime = System.currentTimeMillis();
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("parallel refresh queue [{}] spenttime [{}]", queueName, endTime - startTime);
-                    }
-                } catch (Exception e) {
-                    logger.error("schedule refresh queue error [{}], exception [{}]", queueName, ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/RedisBackedQueueWithoutMetaCacheLoader.java

@@ -1,56 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-import com.google.common.base.Function;
-import com.google.common.cache.CacheLoader;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Entry;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-
-public class RedisBackedQueueWithoutMetaCacheLoader extends CacheLoader<QueueName, Pair<Long, Queue<Video>>> {
-
-    private static final Logger logger = LoggerFactory.getLogger(RedisBackedQueueWithoutMetaCacheLoader.class);
-    private final RedisSmartClient client;
-
-    public RedisBackedQueueWithoutMetaCacheLoader(RedisSmartClient client) {
-        this.client = client;
-    }
-
-    @Override
-    public Pair<Long, Queue<Video>> load(QueueName name) throws Exception {
-        String valueJson = client.get(name.toString());
-        Index index = new Index(name.toString(), valueJson);
-
-        if (null == index || CollectionUtils.isEmpty(index.getIndexEntryList())) {
-//            logger.error("empty_index_fetch");
-        }
-
-        //IndexEntry 转化到Entry中 video 格式
-        Queue<Video> queue = new Queue<Video>(name.toString());
-        Iterable<Entry<Video>> iterable = Iterables.transform(index.getIndexEntryList(), new Function<IndexEntry, Entry<Video>>() {
-            @Nullable
-            @Override
-            public Entry<Video> apply(IndexEntry indexEntry) {
-                Video recallData = new Video();
-                recallData.setVideoId(Long.valueOf(indexEntry.getItemId()));
-                recallData.setPushFrom(name.toString());
-                Entry<Video> entry = new Entry<Video>(recallData, indexEntry.getItemId());
-                entry.addScore("ordering", indexEntry.getScore());
-                return entry;
-            }
-        });
-        queue.addAll(Lists.newArrayList(iterable));
-
-        return Pair.of(Long.MIN_VALUE, queue);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/AbstractScorer.java

@@ -1,72 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.service.score.model.Model;
-import com.tzld.piaoquan.recommend.server.service.score.model.ModelManager;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-
-public abstract class AbstractScorer {
-    public static Logger LOGGER = LoggerFactory.getLogger(AbstractScorer.class);
-    protected ScorerConfigInfo scorerConfigInfo;
-    protected ModelManager modelManager = ModelManager.getInstance();
-
-    public AbstractScorer(ScorerConfigInfo scorerConfigInfo) {
-        this.scorerConfigInfo = scorerConfigInfo;
-    }
-
-    public void loadModel() {
-    }
-
-    public boolean isEnable() {
-        return !getScorerConfigInfo().getDisableSwitch();
-    }
-
-
-    public void doLoadModel(Class<? extends Model> modelClass) {
-
-        String modelPath = scorerConfigInfo.getModelPath();
-        if (StringUtils.isNotBlank(modelPath)) {
-            try {
-                // 使用 modelPath 作为 modelName 注册
-                modelManager.registerModel(modelPath, modelPath, modelClass, Collections.emptyMap());
-                LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
-            } catch (ModelManager.ModelRegisterException e) {
-                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
-            } catch (IOException e) {
-                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
-            }
-        } else {
-            LOGGER.error("modelpath is null, for model class [{}]", modelClass);
-        }
-    }
-
-    public Model getModel() {
-        if (StringUtils.isBlank(scorerConfigInfo.getModelPath())) {
-            return null;
-        }
-        return modelManager.getModel(scorerConfigInfo.getModelPath());
-    }
-
-    public ScorerConfigInfo getScorerConfigInfo() {
-        return scorerConfigInfo;
-    }
-
-    public abstract List<RankItem> scoring(final ScoreParam param,
-                                           final UserFeature userFeature,
-                                           final List<RankItem> rankItems);
-
-    public abstract List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
-                                           final Map<String, String> userFeatureMap,
-                                           final List<RankItem> rankItems);
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseGBDTModelScorer.java

@@ -1,19 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.tzld.piaoquan.recommend.server.service.score.model.GBDTModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class BaseGBDTModelScorer extends AbstractScorer {
-    private static Logger LOGGER = LoggerFactory.getLogger(BaseGBDTModelScorer.class);
-
-    public BaseGBDTModelScorer(ScorerConfigInfo scorerConfigInfo) {
-        super(scorerConfigInfo);
-    }
-
-    @Override
-    public synchronized void loadModel() {
-        doLoadModel(GBDTModel.class);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseLRModelScorer.java

@@ -1,20 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public abstract class BaseLRModelScorer extends AbstractScorer {
-
-    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRModelScorer.class);
-
-    public BaseLRModelScorer(ScorerConfigInfo scorerConfigInfo) {
-        super(scorerConfigInfo);
-    }
-
-    @Override
-    public void loadModel() {
-        doLoadModel(LRModel.class);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseThompsonSamplingScorer.java

@@ -1,32 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.service.score.model.ThompsonSamplingModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-
-public abstract class BaseThompsonSamplingScorer extends AbstractScorer {
-
-    private static Logger LOGGER = LoggerFactory.getLogger(BaseThompsonSamplingScorer.class);
-
-    public BaseThompsonSamplingScorer(ScorerConfigInfo scorerConfigInfo) {
-        super(scorerConfigInfo);
-    }
-
-    @Override
-    public void loadModel() {
-        doLoadModel(ThompsonSamplingModel.class);
-    }
-    @Override
-    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
-                                           final Map<String, String> userFeatureMap,
-                                           final List<RankItem> rankItems){
-
-        return rankItems;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScoreParam.java

@@ -1,13 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
-import lombok.Data;
-
-/**
- * @author dyp
- */
-@Data
-public class ScoreParam {
-    private String mid;
-    private RequestContext requestContext;
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerConfig.java

@@ -1,140 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigObject;
-import com.typesafe.config.ConfigValue;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-
-public class ScorerConfig {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
-    private final List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
-
-    public ScorerConfig(Config config) {
-        this.load(config);
-    }
-
-    public ScorerConfig(String configFile) {
-        this.load(configFile);
-    }
-
-    public ScorerConfig() {
-    }
-
-    public static Config mergeConfig(Config baselineConfig, List<Config> expConfigs) {
-        if (expConfigs == null || expConfigs.size() == 0) {
-            return baselineConfig;
-        }
-        Config resultConfig = expConfigs.get(0);
-        for (Config config : expConfigs) {
-            resultConfig = resultConfig.withFallback(config);
-        }
-        resultConfig = resultConfig.withFallback(baselineConfig);
-        return resultConfig;
-    }
-
-    public boolean load(String configFile) {
-        Config config = ConfigFactory.parseResources(configFile);
-        return load(config);
-    }
-
-    public boolean load(Config baselineConfig, List<Config> expConfigs) {
-        Config config = mergeConfig(baselineConfig, expConfigs);
-        return load(config);
-    }
-
-    public boolean load(Config config) {
-        Config scorerConfig = config.getConfig("scorer-config");
-
-        try {
-            loadScorers(scorerConfig);
-            int pos = 0;
-            for (ScorerConfigInfo scorerConfigInfo : configInfoList) {
-                LOGGER.debug("scorer at position [{}], priority [{}], scorer name [{}]",
-                        pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName());
-            }
-            LOGGER.debug("Load scorer config success");
-        } catch (Exception e) {
-            LOGGER.error("Load scorer config failed, [{}]", ExceptionUtils.getFullStackTrace(e));
-            return false;
-        }
-
-        return true;
-    }
-
-    public List<ScorerConfigInfo> getConfigInfoList() {
-        return configInfoList;
-    }
-
-
-    public String loadOptionStringConfig(Config config, String path) {
-        return config.hasPath(path) ? config.getString(path) : null;
-    }
-
-    private Config loadOptionConfig(Config config, String path) {
-        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
-    }
-
-    private void loadScorers(Config config) throws Exception {
-
-        ConfigObject confObj = config.root();
-        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
-            Config conf = ((ConfigObject) it.getValue()).toConfig();
-            // parse config
-            String configName = it.getKey();
-            String scorerName = conf.getString("scorer-name");
-            int scorerPriority = 0;
-            if (conf.hasPath("scorer-priority"))
-                scorerPriority = conf.getInt("scorer-priority");
-            Boolean disableSwitch = false;
-            if (conf.hasPath("disable-switch")) {
-                disableSwitch = conf.getBoolean("disable-switch");
-            }
-            Config paramConfig = loadOptionConfig(conf, "param-config");
-            // model path
-            String modelPath = loadOptionStringConfig(conf, "model-path");
-            if (modelPath == null) {
-                modelPath = loadOptionStringConfig(conf, "default-model-path");
-                LOGGER.debug("model-path is not exists in config file, use default-model-path instead, modelPath={}", modelPath);
-            }
-            // enable queues
-            Set<String> enableQueues = new HashSet<String>();
-            if (conf.hasPath("enable-queues")) {
-                enableQueues.addAll(conf.getStringList("enable-queues"));
-            }
-            ScorerConfigInfo configInfo = new ScorerConfigInfo(configName,
-                    scorerName,
-                    scorerPriority,
-                    disableSwitch,
-                    enableQueues,
-                    modelPath,
-                    paramConfig
-            );
-            LOGGER.debug("parse scorer config info [{}]", configInfo);
-            // add to ConfigInfoList
-            addConfigByPriority(configInfoList, configInfo);
-        }
-    }
-
-    private void addConfigByPriority(List<ScorerConfigInfo> configInfoList, ScorerConfigInfo addConfigInfo) {
-
-        int pos = 0;
-        for (; pos < configInfoList.size(); pos++) {
-            if (configInfoList.get(pos).getScorerPriority() <= addConfigInfo.getScorerPriority()) {
-                break;
-            }
-        }
-
-        configInfoList.add(pos, addConfigInfo);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerConfigInfo.java

@@ -1,78 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.google.gson.Gson;
-import com.typesafe.config.Config;
-
-import java.util.Set;
-
-
-public class ScorerConfigInfo {
-
-    private String configName;
-    private Integer scorerPriority;
-    private Boolean disableSwitch;
-    private String scorerName;
-    private Set<String> enableQueues;
-    private String modelPath;
-    private Config paramConfig; // param config
-
-    public ScorerConfigInfo(String configName,
-                            String scorerName,
-                            Integer scorerPriority,
-                            Boolean disableSwitch,
-                            Set<String> enableQueues,
-                            String modelPath,
-                            Config paramConfig) {
-
-        this.configName = configName;
-        this.scorerName = scorerName;
-        this.scorerPriority = scorerPriority;
-        this.disableSwitch = disableSwitch;
-        this.enableQueues = enableQueues;
-        this.modelPath = modelPath;
-        this.paramConfig = paramConfig;
-    }
-
-    public Config getParamConfig() {
-        return paramConfig;
-    }
-
-    public Set<String> getEnableQueues() {
-        return enableQueues;
-    }
-
-    public void setEnableQueues(Set<String> enableQueues) {
-        this.enableQueues = enableQueues;
-    }
-
-    public Integer getScorerPriority() {
-        return scorerPriority;
-    }
-
-    public String getScorerName() {
-        return scorerName;
-    }
-
-    public String getConfigName() {
-        return configName;
-    }
-
-    public boolean isQueueEnable(String queueName) {
-        return this.enableQueues == null ||
-                this.enableQueues.isEmpty() ||
-                this.enableQueues.contains(queueName);
-    }
-
-    public String getModelPath() {
-        return modelPath;
-    }
-
-    public Boolean getDisableSwitch() {
-        return disableSwitch;
-    }
-
-    @Override
-    public String toString() {
-        return new Gson().toJson(this);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerPipeline.java

@@ -1,187 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
-
-
-@Slf4j
-public class ScorerPipeline {
-    public static final int corePoolSize = 128;
-    public static final int SCORE_TIME_OUT = 400;
-    public static final Logger LOGGER = LoggerFactory.getLogger(ScorerPipeline.class);
-    public static final ExecutorService executorService = Executors.newFixedThreadPool(corePoolSize);
-
-    public List<AbstractScorer> scorers;
-
-    public ScorerPipeline(List<AbstractScorer> scorers) {
-        this.scorers = scorers;
-    }
-
-    /**
-     * scoring
-     *
-     * @return
-     */
-    public List<RankItem> scoring(final ScoreParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
-
-        if (CollectionUtils.isEmpty(scorers)) {
-            log.error("scorers is empty");
-            return rankItems;
-        }
-        List<RankItem> items = rankItems;
-
-        for (final AbstractScorer scorer : scorers) {
-            if (!scorer.isEnable()) {
-                continue;
-            }
-
-            final int beforeSize = rankItems.size();
-            final long startTime = System.currentTimeMillis();
-
-            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
-            String[] scorerNames = fullScorerName.split("\\.");
-            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
-
-            final List<RankItem> scoreRankerItems = items;
-            Callable<List<RankItem>> callable = () -> scorer.scoring(param, userFeature, scoreRankerItems);
-
-            // execute score use thread to protected score worst time
-            List<RankItem> scoredItems = new ArrayList<RankItem>();
-            try {
-                List<Future<List<RankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
-                for (Future<List<RankItem>> future : futures) {
-                    try {
-                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
-                            scoredItems.addAll(future.get());
-                        } else {
-                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
-                                    new Object[]{scorerName, scoreRankerItems.size()});
-                        }
-                    } catch (Exception e) {
-                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
-                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
-                    }
-                }
-            } catch (Exception e) {
-                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
-                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
-            }
-
-            //  变更item
-            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
-                items = scoreRankerItems;
-            } else {
-                items = new ArrayList<>(items);
-            }
-
-            int position = 0;
-            for (RankItem item : items) {
-                item.getRankerIndex().put(scorerName, position++);
-                item.getRankerScore().put(scorerName, item.getScore());
-            }
-
-            //
-            long spentTime = System.currentTimeMillis() - startTime;
-            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
-                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
-        }
-
-        int position = 0;
-        for (RankItem item : items) {
-            item.getRankerIndex().put("finalScore", position++);
-            item.getRankerScore().put("finalScore", item.getScore());
-        }
-
-        return items;
-    }
-
-    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
-                                  final Map<String, String> userFeatureMap,
-                                  final List<RankItem> rankItems) {
-
-        if (CollectionUtils.isEmpty(scorers)) {
-            log.error("scorers is empty");
-            return rankItems;
-        }
-        List<RankItem> items = rankItems;
-
-        for (final AbstractScorer scorer : scorers) {
-            if (!scorer.isEnable()) {
-                continue;
-            }
-
-            final int beforeSize = rankItems.size();
-            final long startTime = System.currentTimeMillis();
-
-            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
-            String[] scorerNames = fullScorerName.split("\\.");
-            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
-
-            final List<RankItem> scoreRankerItems = items;
-            Callable<List<RankItem>> callable = () -> scorer.scoring(sceneFeatureMap, userFeatureMap, scoreRankerItems);
-
-            // execute score use thread to protected score worst time
-            List<RankItem> scoredItems = new ArrayList<RankItem>();
-            try {
-                List<Future<List<RankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
-                for (Future<List<RankItem>> future : futures) {
-                    try {
-                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
-                            scoredItems.addAll(future.get());
-                        } else {
-                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
-                                    new Object[]{scorerName, scoreRankerItems.size()});
-                        }
-                    } catch (Exception e) {
-                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
-                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
-                    }
-                }
-            } catch (Exception e) {
-                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
-                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
-            }
-
-            //  变更item
-            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
-                items = scoreRankerItems;
-            } else {
-                items = new ArrayList<>(items);
-            }
-
-            int position = 0;
-            for (RankItem item : items) {
-                item.getRankerIndex().put(scorerName, position++);
-                if(scorerName.equals("finalScore")) {
-                    item.getRankerScore().put(scorerName, item.getScore());
-                }
-            }
-
-            //
-            long spentTime = System.currentTimeMillis() - startTime;
-            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
-                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
-        }
-
-        int position = 0;
-        for (RankItem item : items) {
-            item.getRankerIndex().put("finalScore", position++);
-            item.getRankerScore().put("finalScore", item.getScore());
-        }
-
-        return items;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerUtils.java

@@ -1,151 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-
-import com.typesafe.config.Config;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-
-@Slf4j
-public final class ScorerUtils {
-    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
-
-    private static final Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
-
-    public static String BASE_CONF_FEED = "feeds_score_config_new_baseline.conf";
-
-    public static String FLOWPOOL_CONF = "feeds_score_config_thompson_new.conf";
-
-
-    public static void warmUp() {
-        log.info("scorer warm up ");
-        ScorerUtils.init(BASE_CONF_FEED);
-        ScorerUtils.init(FLOWPOOL_CONF);
-    }
-
-    private ScorerUtils() {
-        // init(BASE_CONF);
-    }
-
-    public static void init(String configFile) {
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(configFile);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-
-        initLoadModel(scorers);
-        scorerPipelineCache.put(configFile, new ScorerPipeline(scorers));
-    }
-
-    /**
-     * init load model
-     *
-     * @param scorers
-     */
-    public static void initLoadModel(List<AbstractScorer> scorers) {
-        for (AbstractScorer scorer : scorers) {
-            if (scorer.isEnable()) {
-                scorer.loadModel();
-            }
-        }
-    }
-
-    public static void initLoadModel(String configFile) {
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(configFile);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-        initLoadModel(scorers);
-    }
-
-
-    public static void initLoadModel(Config config) {
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(config);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-        initLoadModel(scorers);
-    }
-
-    public static ScorerPipeline getScorerPipeline(String configFile) {
-        // 不需要保证严格意义的单例
-        if (scorerPipelineCache.containsKey(configFile)) {
-            return scorerPipelineCache.get(configFile);
-        }
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(configFile);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-        ScorerPipeline pipeline = new ScorerPipeline(scorers);
-        scorerPipelineCache.put(configFile, pipeline);
-        return pipeline;
-    }
-
-    public static ScorerPipeline getScorerPipeline(Config mergeConfig) {
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(mergeConfig);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-        return new ScorerPipeline(scorers);
-    }
-
-    public static ScorerPipeline getScorerPipeline(Config baselineConfig, List<Config> configList) {
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(baselineConfig, configList);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-        return new ScorerPipeline(scorers);
-    }
-
-    /**
-     * construct scorers
-     *
-     * @param scorerConfig
-     * @return
-     */
-    public static List<AbstractScorer> constructScorers(ScorerConfig scorerConfig) {
-
-        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
-        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
-            if (!configInfo.getDisableSwitch()) {
-                try {
-                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
-                            .getConstructor(ScorerConfigInfo.class)
-                            .newInstance(configInfo);
-                    scorers.add(scorer);
-                    LOGGER.debug("construct score [{}]", configInfo.getScorerName());
-                } catch (Exception e) {
-                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
-                }
-
-            }
-        }
-        return scorers;
-    }
-
-    /**
-     * construct scorers
-     *
-     * @param scorerConfig
-     * @return
-     */
-    public static List<AbstractScorer> constructQueueScorers(ScorerConfig scorerConfig, String queueName) {
-
-        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
-        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
-            if (!configInfo.getDisableSwitch() && configInfo.isQueueEnable(queueName)) {
-                try {
-                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
-                            .getConstructor(ScorerConfigInfo.class)
-                            .newInstance(configInfo);
-                    scorers.add(scorer);
-                    LOGGER.debug("construct queue scorer [{}] [{}]", queueName, configInfo.getScorerName());
-                } catch (Exception e) {
-                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        return scorers;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/AbstractUserAttentionExtractor.java

@@ -1,28 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.userattention;
-
-
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-public abstract class AbstractUserAttentionExtractor {
-    protected UserAttentionExtractorConfig extractorConfig;
-
-    public AbstractUserAttentionExtractor(UserAttentionExtractorConfig extractorConfig) {
-        this.extractorConfig = extractorConfig;
-    }
-
-    public String getExtractorName() {
-        String[] arr = this.extractorConfig.getName().split(".");
-        return arr.length > 0 ? arr[arr.length - 1] : "UnKnownExtractor";
-    }
-
-    public UserAttentionExtractorConfig getExtractorConfig() {
-        return extractorConfig;
-    }
-
-
-    public abstract void extractAttention(final RecommendRequest requestData, final User user);
-
-    public void loadModel() {
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorConfig.java

@@ -1,44 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.userattention;
-
-import com.typesafe.config.Config;
-
-
-
-public class UserAttentionExtractorConfig {
-    private final String name;  // attention extractor name
-    private final Integer priority;
-    private final String modelPath;
-    private final Config paramConfig; // param config
-
-    public UserAttentionExtractorConfig(String name,
-                                        Config paramConfig,
-                                        Integer priority,
-                                        String modelPath) {
-        this.name = name;
-        this.paramConfig = paramConfig;
-        this.priority = priority;
-        this.modelPath = modelPath;
-    }
-
-    public Integer getPriority() {
-        return priority;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    public Config getParamConfig() {
-        return paramConfig;
-    }
-
-    @Override
-    public String toString() {
-        return "AttentionExtractorConfig{" +
-                "name='" + name + '\'' +
-                ", priority=" + priority +
-                ", modelPath='" + modelPath + '\'' +
-                ", paramConfig=" + paramConfig +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorPipeline.java

@@ -1,36 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.userattention;
-
-
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.common.UserAttention;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class UserAttentionExtractorPipeline {
-    private List<AbstractUserAttentionExtractor> extractorList = new ArrayList<AbstractUserAttentionExtractor>();
-
-    public UserAttentionExtractorPipeline(List<AbstractUserAttentionExtractor> extractorList) {
-        this.extractorList = extractorList;
-    }
-
-    public void extractAttention(final RecommendRequest requestData, final User user) {
-        if (user.getUserAttention() == null) {
-            UserAttention userAttention = new UserAttention();
-            user.setUserAttention(userAttention);
-        }
-
-        long startTime = System.currentTimeMillis();
-
-        for (AbstractUserAttentionExtractor extractor : extractorList) {
-            try {
-                long currStartTime = System.currentTimeMillis();
-                extractor.extractAttention(requestData, user);
-                // log
-            } catch (Exception e) {
-                // log
-            }
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorUtils.java

@@ -1,134 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.userattention;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigObject;
-import com.typesafe.config.ConfigValue;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-
-public class UserAttentionExtractorUtils {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(UserAttentionExtractorUtils.class);
-
-    public static final String BASE_CONF = "";
-
-    /**
-     * 主调用函数
-     *
-     * @param config
-     */
-    public static void initLoadModel(Config config) {
-        initLoadModel(constructExtractorList(constructExtractorConfigList(config)));
-    }
-
-
-    public static UserAttentionExtractorPipeline getAtttentionPipeline(String configFile) {
-
-        Config attentionConfig = ConfigFactory.parseResources(configFile);
-        List<AbstractUserAttentionExtractor> attentionExtractors = constructExtractorList(constructExtractorConfigList(attentionConfig));
-        UserAttentionExtractorPipeline pipeline = new UserAttentionExtractorPipeline(attentionExtractors);
-
-        return pipeline;
-    }
-
-
-    public static List<UserAttentionExtractorConfig> constructExtractorConfigList(Config config) {
-        long startTime = System.currentTimeMillis();
-        List<UserAttentionExtractorConfig> configList = new ArrayList<UserAttentionExtractorConfig>();
-        try {
-            Config mergedConfig = config.getConfig("attention-extractor-config");
-            for (ConfigObject.Entry<String, ConfigValue> it : mergedConfig.root().entrySet()) {
-                LOGGER.debug("deal with base config, attention key: [{}]", it.getKey());
-                Config entryConf = ((ConfigObject) it.getValue()).toConfig();
-                UserAttentionExtractorConfig extractorConfig = parseExtractorConfig(entryConf);
-                addConfigByPriority(configList, extractorConfig);
-            }
-            // debug log
-            if (LOGGER.isDebugEnabled()) {
-                for (UserAttentionExtractorConfig extractorConfig : configList) {
-                    LOGGER.debug("attention extractor config [{}] [{}]", extractorConfig.getName(), extractorConfig);
-                }
-            }
-        } catch (Exception e) {
-            LOGGER.error("construct extractor config {} failed {}", config.toString(), ExceptionUtils.getFullStackTrace(e));
-        }
-        return configList;
-    }
-
-
-    private static List<AbstractUserAttentionExtractor> constructExtractorList(List<UserAttentionExtractorConfig> configList) {
-        long startTime = System.currentTimeMillis();
-        List<AbstractUserAttentionExtractor> extractorList = new ArrayList<AbstractUserAttentionExtractor>();
-        for (UserAttentionExtractorConfig config : configList) {
-            try {
-                AbstractUserAttentionExtractor extractor = (AbstractUserAttentionExtractor) Class.forName(config.getName())
-                        .getConstructor(UserAttentionExtractorConfig.class)
-                        .newInstance(config);
-                extractorList.add(extractor);
-            } catch (Exception e) {
-                LOGGER.error("instance extractor {} failed {}", config.getName(), ExceptionUtils.getFullStackTrace(e));
-            }
-        }
-        return extractorList;
-    }
-
-    private static String loadOptionStringConfig(Config config, String path) {
-        return config.hasPath(path) ? config.getString(path) : null;
-    }
-
-    private static Config loadOptionConfig(Config config, String path) {
-        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
-    }
-
-    private static int loadOptionIntConfig(Config config, String path) {
-        return config.hasPath(path) ? config.getInt(path) : 0;
-    }
-
-    /**
-     * attention层的配置化参数
-     */
-    private static UserAttentionExtractorConfig parseExtractorConfig(Config extractorConfig) {
-        String attentionExtractorName = extractorConfig.getString("name");
-        Integer priority = loadOptionIntConfig(extractorConfig, "attention-priority");
-        Config paramConfig = loadOptionConfig(extractorConfig, "param-config");
-        String modelPath = loadOptionStringConfig(extractorConfig, "model-path");
-        return new UserAttentionExtractorConfig(attentionExtractorName,
-                paramConfig,
-                priority,
-                modelPath);
-    }
-
-
-    /**
-     * attention支持模型获取
-     *
-     * @param attentionExtractors
-     */
-    private static void initLoadModel(List<AbstractUserAttentionExtractor> attentionExtractors) {
-        for (AbstractUserAttentionExtractor attentionExtractor : attentionExtractors) {
-            attentionExtractor.loadModel();
-        }
-    }
-
-    /**
-     * 按优先级处理逻辑
-     *
-     * @param configInfoList
-     * @param extractorConfig
-     */
-    private static void addConfigByPriority(List<UserAttentionExtractorConfig> configInfoList, UserAttentionExtractorConfig extractorConfig) {
-        int pos = 0;
-        for (; pos < configInfoList.size(); pos++) {
-            if (configInfoList.get(pos).getPriority() <= extractorConfig.getPriority()) {
-                break;
-            }
-        }
-        configInfoList.add(pos, extractorConfig);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/FixedThreadPoolHelper.java

@@ -1,11 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.utils;
-
-public class FixedThreadPoolHelper extends ThreadPoolHelper {
-    public FixedThreadPoolHelper(int nThread) {
-        super(ThreadPoolUtils.newFixedThreadPool(nThread));
-    }
-
-    public FixedThreadPoolHelper(int nThread, String name) {
-        super(ThreadPoolUtils.newFixedThreadPool(nThread, name), "FixedPool_" + name);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/RedisSmartClient.java

@@ -1,64 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.utils;
-
-
-import com.google.common.base.Function;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.stereotype.Service;
-import redis.clients.jedis.JedisCluster;
-import redis.clients.jedis.Tuple;
-
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-@Service
-public class RedisSmartClient {
-
-    private final static ExecutorService executorService = Executors.newCachedThreadPool();
-    @Qualifier("redisTemplate")
-    @Autowired
-    protected RedisTemplate<String, String> redisTemplate;
-
-
-    public RedisSmartClient() {
-    }
-    public void set(final String key, final String value) {
-        redisTemplate.opsForValue().set(key, value);
-    }
-
-    public String get(final String key) {
-        return redisTemplate.opsForValue().get(key);
-    }
-
-    // hget
-    public String hget(final String key, final String field) {
-        return redisTemplate.opsForHash().get(key, field).toString();
-    }
-
-
-    // getRandomValueFromSet
-    public List<String> getRandomList(final String key, long num) {
-        return redisTemplate.opsForSet().randomMembers(key, num);
-    }
-
-
-    // hgetall
-    public Map<Object, Object> hgetall(String key) {
-        return redisTemplate.opsForHash().entries(key);
-    }
-
-    // TODO 异步写入redis 作为用户的分发历史
-    public <T> T write(String key, String value) {
-        T t = null;
-        return t;
-    }
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/ThreadPoolHelper.java

@@ -1,64 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.utils;
-
-
-import lombok.Data;
-
-import java.util.Timer;
-import java.util.concurrent.ThreadPoolExecutor;
-
-/**
- * 线程池Helper
- *   可以命名线程池; 保存ThreadPoolExecutor实例, 方便监控线程池信息
- */
-@Data
-public class ThreadPoolHelper {
-    private String name;
-    private ThreadPoolExecutor threadPoolExecutor;
-    private Timer counterTimer;
-
-    ThreadPoolHelper(ThreadPoolExecutor threadPoolExecutor) {
-        this.threadPoolExecutor = threadPoolExecutor;
-        this.name = "";
-    }
-
-    ThreadPoolHelper(ThreadPoolExecutor threadPoolExecutor, String name) {
-        this.threadPoolExecutor = threadPoolExecutor;
-        this.name = name;
-    }
-
-
-    public long getTaskCount() {
-        return this.threadPoolExecutor.getTaskCount();
-    }
-
-    public long getActiveCount() {
-        return this.threadPoolExecutor.getActiveCount();
-    }
-
-    public long getCorePoolSize() {
-        return this.threadPoolExecutor.getCorePoolSize();
-    }
-
-    public long getPoolSize() {
-        return this.threadPoolExecutor.getPoolSize();
-    }
-
-    public long getMaximumPoolSize() {
-        return this.threadPoolExecutor.getMaximumPoolSize();
-    }
-
-    public long getLargestPoolSize() {
-        return this.threadPoolExecutor.getLargestPoolSize();
-    }
-
-    public long getTaskQueueSize() {
-        return this.threadPoolExecutor.getQueue().size();
-    }
-
-    public void cancelCounter() {
-        if (counterTimer != null) {
-            counterTimer.cancel();
-            counterTimer.purge();
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/ThreadPoolUtils.java

@@ -1,46 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.utils;
-
-
-import java.util.concurrent.*;
-
-/**
- * 命名线程池工场类
- * 最后创建的线程名称:
- *   name_{counter}
- */
-class NamedThreadFactory implements ThreadFactory {
-    private int counter = 0;
-    private String name;
-    NamedThreadFactory(String name) {
-        this.name = name;
-    }
-
-    @Override
-    public Thread newThread(Runnable r) {
-        counter += 1;
-        return new Thread(r, name + "_" + counter);
-    }
-}
-
-
-public class ThreadPoolUtils {
-    public static ThreadPoolExecutor newFixedThreadPool(int nThreads, String poolName) {
-        return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
-                new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("Fixed_" + poolName));
-    }
-    public static ThreadPoolExecutor newFixedThreadPool(int nThreads) {
-        return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
-                new LinkedBlockingQueue<Runnable>());
-    }
-
-    public static ThreadPoolExecutor newCachedThreadPool(String poolName) {
-        return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
-                new SynchronousQueue<Runnable>(), new NamedThreadFactory("Cached_" + poolName));
-    }
-
-    public static ThreadPoolExecutor newCachedThreadPool() {
-        return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
-                new SynchronousQueue<Runnable>());
-    }
-}
-

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/grpcservice/RecommendGrpcService.java

@@ -3,7 +3,7 @@ package com.tzld.piaoquan.recommend.server.grpcservice;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendResponse;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendServiceGrpc;
-import com.tzld.piaoquan.recommend.server.service.VideoRecommendService;
+import com.tzld.piaoquan.recommend.server.service.RecommendService;
 import io.grpc.stub.StreamObserver;
 import net.devh.boot.grpc.server.service.GrpcService;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -14,18 +14,18 @@ import org.springframework.beans.factory.annotation.Autowired;
 @GrpcService
 public class RecommendGrpcService extends RecommendServiceGrpc.RecommendServiceImplBase {
     @Autowired
-    private VideoRecommendService videoRecommendService;
+    private RecommendService recommendService;
 
     @Override
     public void homepageRecommend(RecommendRequest request, StreamObserver<RecommendResponse> responseObserver) {
-        RecommendResponse response = videoRecommendService.homepageRecommend(request);
+        RecommendResponse response = recommendService.homepageRecommend(request);
         responseObserver.onNext(response);
         responseObserver.onCompleted();
     }
 
     @Override
     public void relevantRecommend(RecommendRequest request, StreamObserver<RecommendResponse> responseObserver) {
-        RecommendResponse response = videoRecommendService.relevantRecommend(request);
+        RecommendResponse response = recommendService.relevantRecommend(request);
         responseObserver.onNext(response);
         responseObserver.onCompleted();
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/FlowPoolRecommendPipeline.java

@@ -1,208 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement;
-
-
-import com.alibaba.fastjson.JSONObject;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.MachineInfo;
-import com.tzld.piaoquan.recommend.server.model.RecommendParam;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.RecommendService;
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.FlowPoolWithLevelRecallStrategyTomson;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.FlowPoolWithLevelScoreRecallStrategy;
-import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import org.apache.commons.collections4.CollectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeansException;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.PostConstruct;
-import javax.annotation.Resource;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-@Service
-public class FlowPoolRecommendPipeline implements ApplicationContextAware {
-
-    private final Logger log = LoggerFactory.getLogger(FlowPoolRecommendPipeline.class);
-
-    private final static Map<String, RecallStrategy> strategyMap = new HashMap<>();
-
-    @Resource
-    private RecommendService recommendService;
-
-    private ApplicationContext applicationContext;
-
-    private final ExecutorService pool = ThreadPoolFactory.recallPool();
-
-    @PostConstruct
-    public void init() {
-        Map<String, RecallStrategy> type = applicationContext.getBeansOfType(RecallStrategy.class);
-        for (Map.Entry<String, RecallStrategy> entry : type.entrySet()) {
-            RecallStrategy value = entry.getValue();
-            strategyMap.put(value.getClass().getSimpleName(), value);
-        }
-    }
-
-
-    public List<Video> feedByRec(final RecommendRequest requestData,
-                                 final int recommendType, Boolean logPrint) {
-        List<RecallStrategy> strategies = new ArrayList<>();
-
-        RecommendParam param = recommendService.genRecommendParam(requestData, recommendType);
-        RecallParam recallParam = recommendService.convertToRecallParam(param);
-        RecallResult recallResult = getRecallResult(strategies, recallParam);
-        if (logPrint) {
-            log.info("traceId = {}, recallResult [{}]", requestData.getRequestId(),
-                    JSONObject.toJSONString(recallResult));
-        }
-
-        if (recallResult == null || CollectionUtils.isEmpty(recallResult.getData())) {
-            return Collections.emptyList();
-        }
-
-        RankParam rankParam = recommendService.convertToRankParam(param, recallResult);
-        List<Video> videoList = mergeAndRankFlowPoolRecall(rankParam);
-        if (logPrint) {
-            log.info("traceId = {}, videoList [{}]", requestData.getRequestId(),
-                    JSONObject.toJSONString(videoList));
-        }
-        return videoList;
-    }
-
-    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
-        return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
-    }
-
-    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
-
-        //初始化 userid
-        UserFeature userFeature = new UserFeature();
-        userFeature.setMid(param.getMid());
-
-        // 初始化RankItem
-        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
-                .filter(d -> d.getPushFrom().equals(pushFrom))
-                .findFirst();
-        if (!data.isPresent()) {
-            return Collections.emptyList();
-        }
-        List<Video> videoList = data.get().getVideos();
-        if (videoList == null) {
-            return Collections.emptyList();
-        }
-        List<RankItem> rankItems = new ArrayList<>();
-        for (int i = 0; i < videoList.size(); i++) {
-            RankItem rankItem = new RankItem(videoList.get(i));
-            rankItems.add(rankItem);
-        }
-
-        // 初始化上下文参数
-        ScoreParam scoreParam = convert(param);
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
-                .scoring(scoreParam, userFeature, rankItems);
-
-        if (rovRecallScore == null) {
-            return Collections.emptyList();
-        }
-
-        return CommonCollectionUtils.toList(rovRecallScore, i -> {
-            // hard code 将排序分数 赋值给video的sortScore
-            Video v = i.getVideo();
-            v.setSortScore(i.getScore());
-            return v;
-        });
-    }
-
-    protected ScoreParam convert(RankParam param) {
-        ScoreParam scoreParam = new ScoreParam();
-
-        scoreParam.setMid(param.getMid());
-
-        // TODO hardcode 为了兼容写入逻辑
-        RequestContext context = new RequestContext();
-        context.setApptype(param.getAppType() + "");
-
-        // TODO 地域转换
-        context.setRegion(param.getProvince());
-        context.setCity(param.getCity());
-
-        Calendar calendar = Calendar.getInstance();
-        context.setWeek((calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        context.setDay(new SimpleDateFormat("yyyyMMdd").format(calendar.getTime()));
-        context.setHour(new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        MachineInfo machineInfo = param.getMachineInfo();
-        if (machineInfo != null) {
-            context.setMachineinfo_brand(machineInfo.getBrand());
-            context.setMachineinfo_model(machineInfo.getModel());
-            context.setMachineinfo_platform(machineInfo.getPlatform());
-            context.setMachineinfo_sdkversion(machineInfo.getSdkVersion());
-            context.setMachineinfo_system(machineInfo.getSystem());
-            context.setMachineinfo_wechatversion(machineInfo.getWechatVersion());
-        }
-
-        scoreParam.setRequestContext(context);
-        return scoreParam;
-    }
-
-    private RecallResult getRecallResult(List<RecallStrategy> strategies, RecallParam param) {
-        if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL)) {
-            strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategyTomson.class.getSimpleName()));
-        } else if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL_SCORE)) {
-            strategies.add(strategyMap.get(FlowPoolWithLevelScoreRecallStrategy.class.getSimpleName()));
-        }
-
-        CountDownLatch cdl = new CountDownLatch(strategies.size());
-        List<Future<RecallResult.RecallData>> recallResultFutures = new ArrayList<>();
-        for (final RecallStrategy strategy : strategies) {
-            Future<RecallResult.RecallData> future = pool.submit(() -> {
-                List<Video> result = strategy.recall(param);
-                cdl.countDown();
-                return new RecallResult.RecallData(strategy.pushFrom(), result);
-            });
-            recallResultFutures.add(future);
-        }
-        try {
-            cdl.await(3000, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            log.error("recall error", e);
-            return null;
-        }
-
-        List<RecallResult.RecallData> results = new ArrayList<>();
-        for (Future<RecallResult.RecallData> f : recallResultFutures) {
-            try {
-                RecallResult.RecallData data = f.get();
-                results.add(data);
-            } catch (Exception e) {
-                log.error("future get error ", e);
-            }
-        }
-
-        return new RecallResult(results);
-    }
-
-
-    @Override
-    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
-        this.applicationContext = applicationContext;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/TopRecommendPipeline.java

@@ -1,627 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement;
-
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.base.Stopwatch;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.CandidateInfo;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.MergeUtils;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueue;
-import com.tzld.piaoquan.recommend.server.framework.recaller.BaseRecaller;
-import com.tzld.piaoquan.recommend.server.framework.recaller.provider.RedisBackedQueue;
-import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import org.apache.commons.collections4.CollectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.PostConstruct;
-import javax.annotation.Resource;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-@Service
-public class TopRecommendPipeline {
-
-    private static final Logger log = LoggerFactory.getLogger(TopRecommendPipeline.class);
-
-    public static final String MERGE_CONF = "merge_config.conf";
-
-    @Value("${recommend.recall.num:500}")
-    private int recallNum;
-    @ApolloJsonValue("${rank.score.merge.weightv547:}")
-    private Map<String, Double> mergeWeight;
-    @Resource
-    private RedisSmartClient client;
-    @Resource
-    private RedisTemplate<String, String> redisTemplate;
-
-    @Qualifier("featureRedisTemplate")
-    @Autowired
-    private RedisTemplate<String, String> featureRedisTemplate;
-
-    private RedisBackedQueue queueProvider;
-
-    @PostConstruct
-    public void init() {
-        queueProvider = new RedisBackedQueue(client, 15 * 60 * 1000L);
-        mergeWeight = mergeWeight == null ? new HashMap<>() : mergeWeight;
-    }
-
-    public List<Video> feeds(final RecommendRequest requestData,
-                             final int requestIndex,
-                             final User userInfo, Boolean logPrint,
-                             Map<String, String> timeLogMap) {
-        // Step 1: Attention extraction
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        stopwatch.reset().start();
-        timeLogMap.put("uid", userInfo.getUid());
-        timeLogMap.put("mid", userInfo.getMid());
-        timeLogMap.put("requestId", requestData.getRequestId());
-
-        List<RankItem> rankItems = feedByRec(requestData, requestIndex, userInfo, logPrint, timeLogMap);
-        timeLogMap.put("feedByRec", stopwatch.elapsed().toMillis() + "");
-        if (rankItems == null || rankItems.isEmpty()) {
-            return new ArrayList<>();
-        }
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, feeds rankItems = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(rankItems));
-        }
-        stopwatch.reset().start();
-        List<Video> videos = rankItem2Video(rankItems);
-        timeLogMap.put("rankItem2Video", stopwatch.elapsed().toMillis() + "");
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, videos = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(videos));
-        }
-        return videos;
-    }
-
-    private List<Video> rankItem2Video(List<RankItem> rankItems) {
-        List<Video> videos = new ArrayList<>();
-        for (RankItem item : rankItems) {
-            Video video = new Video();
-            video.setVideoId(Long.parseLong(item.getId()));
-            video.setPushFrom(item.getQueue());
-            video.setScore(item.getScore());
-            video.setSortScore(item.getScore());
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-
-            Map<String, List<String>> pushFromIndex = new HashMap<>();
-            pushFromIndex.put(item.getQueue(), item.getCandidateInfoList().stream()
-                    .map(CandidateInfo::getCandidateQueueName).collect(Collectors.toList()));
-            video.setPushFromIndex(pushFromIndex);
-            videos.add(video);
-        }
-        videos.sort(Comparator.comparing(Video::getScore).reversed());
-        return videos;
-    }
-
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
-
-    public Double calScoreWeight(List<Double> data){
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i=0; i<data.size(); ++i){
-            up += 1.0 / (i + 1) * data.get(i);
-            down += 1.0 / (i + 1);
-        }
-        return down > 1E-8? up / down: 0.0;
-    }
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
-        List<Double> data = new LinkedList<>();
-        for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
-        }
-        return data;
-    }
-
-    public List<RankItem> feedByRec(final RecommendRequest requestData,
-                                    final int requestIndex,
-                                    final User userInfo, Boolean logPrint,
-                                    Map<String, String> timeLogMap) {
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        // Step 2: create top queue
-        stopwatch.reset().start();
-        StrategyQueue topQueue = MergeUtils.createTopQueue(MERGE_CONF, "top-queue");
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, topQueue = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(topQueue));
-        }
-        timeLogMap.put("createTopQueue", stopwatch.elapsed().toMillis() + "");
-
-        // Step 3: Candidate
-        stopwatch.reset().start();
-        Map<String, Candidate> candidates = new HashMap<String, Candidate>();
-        topQueue.candidate(candidates, recallNum, userInfo, requestData, 0, 0);
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, candidates = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(candidates));
-        }
-        timeLogMap.put("topQueue-candidate-cost", stopwatch.elapsed().toMillis() + "");
-
-
-        // Step 4: Recalling & Basic Scoring
-        stopwatch.reset().start();
-        BaseRecaller recaller = new BaseRecaller(queueProvider);
-        List<RankItem> items = recaller.recalling(requestData, userInfo, new ArrayList<>(candidates.values()));
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, items = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(items));
-        }
-        timeLogMap.put("recalling-cost", stopwatch.elapsed().toMillis() + "");
-        timeLogMap.put("recalling-size", items == null ? "0" : items.size() + "");
-
-        if (CollectionUtils.isEmpty(items)) {
-            return new ArrayList<>();
-        }
-
-        // Step 4: Advance Scoring
-        stopwatch.reset().start();
-        List<RankItem> rankItemList = videoScoredByFeature(items, requestData);
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, rankItemList = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(rankItemList));
-        }
-
-        stopwatch.reset().start();
-        // Step 5: Merger
-        MergeUtils.distributeItemsToMultiQueues(topQueue, rankItemList);
-        topQueue.merge(recallNum * 3, userInfo, requestData, requestIndex, 0);
-
-        // 多样性融合
-        List<RankItem> mergeItems = topQueue.getItems();
-        if (CollectionUtils.isEmpty(mergeItems)) {
-            return new ArrayList<>();
-        }
-        duplicate(mergeItems);
-
-        timeLogMap.put("mergeItems-cost", stopwatch.elapsed().toMillis() + "");
-        timeLogMap.put("mergeItems-size", mergeItems.size() + "");
-
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, mergeItems = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(mergeItems));
-        }
-//        MergeUtils.diversityRerank(mergeItems, SimilarityUtils.getIsSameUserTagOrCategoryFunc(), recallNum, 6, 2);
-
-        // Step 6: Global Rank & subList
-        return mergeItems;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    private List<RankItem> videoScoredByFeature(List<RankItem> items, RecommendRequest recommendRequest) {
-        // 1 模型分
-        List<RankItem> rankItemList = model(items, recommendRequest);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分 3H
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 3; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : rankItemList) {
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // 平台回流ROV
-            List<Double> view2PreReturns = getRateData(preturns, views, 0.0, 0.0);
-            Double view2PreReturnsScore = calScoreWeightNoTimeDecay(view2PreReturns);
-            item.scoresMap.put("view2PreReturnsScore", view2PreReturnsScore);
-
-            // 平台回流ROS
-            List<Double> share2PreReturns = getRateData(preturns, shares, 1.0, 10.0);
-            Double share2PreReturnsScore = calScoreWeightNoTimeDecay(share2PreReturns);
-            item.scoresMap.put("share2PreReturnsScore", share2PreReturnsScore);
-
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-//        double a = mergeWeight.getOrDefault("a", 0.1);
-//        double b = mergeWeight.getOrDefault("b", 0.0);
-//        double c = mergeWeight.getOrDefault("c", 0.000001);
-//        double d = mergeWeight.getOrDefault("d", 1.0);
-//        double e = mergeWeight.getOrDefault("e", 1.0);
-//        double f = mergeWeight.getOrDefault("f", 0.8);
-//        double g = mergeWeight.getOrDefault("g", 2.0);
-//        double h = mergeWeight.getOrDefault("h", 240.0);
-//        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : rankItemList) {
-//            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-//                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-//            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-//                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-//            double strScore = item.getScoreStr();
-//            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-//            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-//            Double view2PreReturnsScore = item.scoresMap.getOrDefault("view2PreReturnsScore", 0.0);
-//            Double share2PreReturnsScore = item.scoresMap.getOrDefault("share2PreReturnsScore", 0.0);
-            // if NaN set 0
-            if (Double.isNaN(share2allreturnScore)) {
-                share2allreturnScore = 0.0;
-            }
-            if (Double.isNaN(view2allreturnScore)) {
-                view2allreturnScore = 0.0;
-            }
-            double score = share2allreturnScore + view2allreturnScore;
-//            if (ifAdd < 0.5) {
-//                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-//                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-//            } else {
-//                score = a * strScore + b * rosScore + c * preturnsScore +
-//                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-//
-//            }
-//            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-//            if (allreturnsScore > h) {
-//                score += (f * share2allreturnScore + g * view2allreturnScore);
-//            }
-            // 设置计算好的分数
-            item.setScore(score);
-        }
-        return rankItemList;
-    }
-
-    private List<RankItem> model(List<RankItem> items, RecommendRequest param) {
-        if (items.isEmpty()) {
-            return items;
-        }
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = featureRedisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.getClass().getSimpleName(), e));
-                }
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<String> videoIds = CommonCollectionUtils.toListDistinct(items, RankItem::getId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = featureRedisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    items.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    items.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.getClass().getSimpleName(), e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : items) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}",
-                            this.getClass().getSimpleName(), e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item : items) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}",
-                            this.getClass().getSimpleName(), e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, items);
-        return rovRecallScore;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    private void duplicate(List<RankItem> items) {
-        Set<String> ids = new HashSet<>();
-        List<RankItem> result = new ArrayList<>();
-        for (RankItem item : items) {
-            if (ids.contains(item.getId())) {
-                continue;
-            }
-            ids.add(item.getId());
-            result.add(item);
-        }
-        items.clear();
-        items.addAll(result);
-    }
-
-    private Map<String, String> getSceneFeature(RecommendRequest param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ){
-        }else{
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/attention/SimpleAttentionExtractor.java

@@ -1,20 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.attention;
-
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.userattention.AbstractUserAttentionExtractor;
-import com.tzld.piaoquan.recommend.server.framework.userattention.UserAttentionExtractorConfig;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-public class SimpleAttentionExtractor extends AbstractUserAttentionExtractor {
-
-    public SimpleAttentionExtractor(UserAttentionExtractorConfig config) {
-        super(config);
-    }
-
-    @Override
-    public void extractAttention(RecommendRequest recommendRequestData, User user) {
-        //TODO
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global1hHotCandidate.java

@@ -1,71 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Global1hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Global1hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
-
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-
-
-        QueueName queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "global1h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "global1h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_rov")
-                .addMatch("type", "global1h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "global1h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "combination")
-                .addMatch("type", "global1h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum, getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-
-    }
-
-    @Override
-    public String toString() {
-        return "Global24hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global24hHotCandidate.java

@@ -1,64 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Global24hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Global24hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<>();
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-
-        QueueName queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "combination")
-                .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-
-    }
-
-    @Override
-    public String toString() {
-        return "Global24hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global3hHotCandidate.java

@@ -1,66 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Global3hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Global3hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
-
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-
-
-        QueueName queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "global3h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_rov")
-                .addMatch("type", "global3h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "global3h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "global3h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-
-    }
-
-    @Override
-    public String toString() {
-        return "Global24hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/HotCandidateQueue.java

@@ -1,39 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.util.Map;
-
-/**
- * 对应 SimHotVideo 和 ReturnVideo
- *
- * @author sunxy
- */
-public class HotCandidateQueue extends IndexCandidateQueue {
-
-    public HotCandidateQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user,
-                               RecommendRequest requestData, int requestIndex, int expId) {
-
-        return 0;
-    }
-
-    @Override
-    public String toString() {
-        return "HotCandidateQueue{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region1hHotCandidate.java

@@ -1,81 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Region1hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Region1hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        String region = user.getRegion();
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-
-
-        QueueName queueName = new QueueName(ItemType, "short_rov")
-                .addMatch("type", "region1h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum,
-                getStrategyQueueInfo().getQueueName());
-
-
-        queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "region1h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum,
-                getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "region1h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
-                getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "combination")
-                .addMatch("type", "region1h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum,
-                getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "region1h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
-                getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-    }
-
-    @Override
-    public String toString() {
-        return "Region1hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region24hHotCandidate.java

@@ -1,77 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Region24hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Region24hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        String region = user.getRegion();
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
-
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
-        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-
-        // index key  video:queue:type=region24h:region=北京:ordering=rov
-        QueueName queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "region24h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
-
-        // index key  video:queue:type=region24h:region=北京:ordering=str
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "region24h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
-                getStrategyQueueInfo().getQueueName());
-
-        // index key  video:queue:type=region24h:region=北京:ordering=combination
-        queueName = new QueueName(ItemType, "combination")
-                .addMatch("type", "region24h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum,
-                getStrategyQueueInfo().getQueueName());
-
-        // index key  video:queue:type=region24h:region=北京:ordering=str
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "region24h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
-                getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-
-    }
-
-    @Override
-    public String toString() {
-        return "Region24hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region3hHotCandidate.java

@@ -1,77 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Region3hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Region3hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        String region = user.getRegion();
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<>();
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-
-
-        // index key video:queue:type=region3h:region=北京:ordering=short_rov
-        QueueName queueName = new QueueName(ItemType, "short_rov")
-                .addMatch("type", "region3h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum,
-                getStrategyQueueInfo().getQueueName());
-
-        // index key video:queue:type=region3h:region=北京:ordering=rov
-        queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "region3h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum,
-                getStrategyQueueInfo().getQueueName());
-
-        // index key video:queue:type=region3h:region=北京:ordering=str
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "region3h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
-                getStrategyQueueInfo().getQueueName());
-
-        // index key video:queue:type=region3h:region=北京:ordering=str
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "region3h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
-                getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-    }
-
-    @Override
-    public String toString() {
-        return "Region3hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/AllowListFilter.java

@@ -1,113 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.google.common.collect.Lists;
-import com.google.common.hash.Hashing;
-import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.PreViewedService;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.data.redis.core.RedisTemplate;
-
-import java.util.*;
-
-public class AllowListFilter extends AbstractFilter<Video> {
-
-    @Autowired
-    @Qualifier("longVideoRedisTemplate")
-    private RedisTemplate<String, String> redisTemplate;
-
-    private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX = "movie:videoid:allowSet:";
-    private static final String RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY = "mp:religion:allowlist:videoid:bitmap";
-
-    public AllowListFilter(FilterConfigInfo filterConfigInfo,
-                           RecommendRequest recommendRequest,
-                           User user) {
-        super(filterConfigInfo, recommendRequest, user);
-
-    }
-
-
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> list) {
-        if (CollectionUtils.isEmpty(list)) {
-            return;
-        }
-        List<Video> result = Lists.newArrayList();
-        for (Video video : list) {
-            if (!isAllowList(user, video)) {
-                result.add(video);
-            }
-        }
-        list.clear();
-        list.addAll(result);
-    }
-
-
-    public boolean isAllowList(User user, Video video) {
-        //不是新小程序 在白名单则不显示
-        Set<Long> retainVideoIds = new LinkedHashSet<>();
-        int appType = requestContext.getAppType();
-        if ( appType != AppTypeEnum.WAN_NENG_VIDEO.getCode()
-                && appType != AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
-                && appType != AppTypeEnum.ZUI_JING_QI.getCode()
-                && appType!= AppTypeEnum.H5.getCode()) {
-            // 如果不在新小程序白名单，则允许
-            return !isMemberOfVideoAllowList(video.getVideoId());
-
-        } else if (appType == AppTypeEnum.WAN_NENG_VIDEO.getCode()
-                || appType == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
-                || appType == AppTypeEnum.ZUI_JING_QI.getCode()
-                || appType == AppTypeEnum.H5.getCode()) {
-            return !isMemberOfReligionVideoAllowList(video.getVideoId());
-
-        }
-        return true;
-    }
-
-    private boolean isMemberOfVideoAllowList(Long videoId) {
-        if (Objects.isNull(videoId)) {
-            return false;
-        }
-        try {
-            int newIdx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 100;
-            String newPrefix = VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX + newIdx;
-            Boolean result = redisTemplate.opsForSet().isMember(newPrefix, String.valueOf(videoId));
-            return result;
-        } catch (Exception e) {
-            LOGGER.error("isMemberOfVideoAllowList error {}", videoId, e);
-        }
-        return false;
-    }
-
-    public boolean isMemberOfReligionVideoAllowList(Long videoId) {
-        if (Objects.isNull(videoId)) {
-            return false;
-        }
-        try {
-            return redisTemplate.opsForValue().getBit(RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY, videoId);
-        } catch (Exception e) {
-            LOGGER.error("isMemberOfReligionVideoAllowList error {}", e, videoId);
-        }
-        return false;
-    }
-
-
-
-
-
-}
-
-
-
-

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/HistoryLongPeriodFilter.java

@@ -1,56 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.RecommendParam;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.RecommendService;
-import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
-import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
-import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * TODO 有性能问题，暂时废弃
- *
- * @author sunxy
- */
-public class HistoryLongPeriodFilter extends AbstractFilter<Video> {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(HistoryLongPeriodFilter.class);
-
-    private final RegionFilterService filterService;
-    private final RecommendService recommendService;
-
-    public HistoryLongPeriodFilter(FilterConfigInfo filterConfigInfo, RecommendRequest requestContext, User user) {
-        super(filterConfigInfo, requestContext, user);
-        filterService = ServiceBeanFactory.getBean(RegionFilterService.class);
-        recommendService = ServiceBeanFactory.getBean(RecommendService.class);
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> list) {
-        if (list == null || list.isEmpty()) {
-            return;
-        }
-        RecommendParam recommendParam = recommendService.genRecommendParam(requestContext, 1);
-        RecallParam recallParam = recommendService.convertToRecallParam(recommendParam);
-        FilterParam filterParam = FilterParamFactory.create(recallParam,
-                list.stream().map(Video::getVideoId).collect(Collectors.toList()));
-        FilterResult filterResult = filterService.filter(filterParam);
-        if (filterResult != null && filterResult.getVideoIds() != null) {
-            list.removeIf(video -> !filterResult.getVideoIds().contains(video.getVideoId()));
-        }
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/PreViewedFilter.java

@@ -1,54 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.PreViewedService;
-import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-public class PreViewedFilter extends AbstractFilter<Video> {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(PreViewedFilter.class);
-
-    private final Set<Long> preViewedVideoIds;
-
-    public PreViewedFilter(FilterConfigInfo filterConfigInfo,
-                           RecommendRequest recommendRequest,
-                           User user) {
-        super(filterConfigInfo, recommendRequest, user);
-        PreViewedService preViewedService = ServiceBeanFactory.getBean(PreViewedService.class);
-        preViewedVideoIds = preViewedService.getVideoIds(this.requestContext.getAppType(), user.getMid());
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> list) {
-        if (StringUtils.isBlank(user.getMid())
-                || CollectionUtils.isEmpty(list)) {
-            return;
-        }
-
-        if (CollectionUtils.isEmpty(preViewedVideoIds)) {
-            return;
-        }
-        list.removeIf(video -> preViewedVideoIds.contains(video.getVideoId()));
-    }
-
-
-}
-
-
-
-

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/RecommendStatusFilter.java

@@ -1,47 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.SpringContextHolder;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
-import com.tzld.piaoquan.recommend.server.service.filter.strategy.RecommendStatusStrategy;
-import org.apache.commons.collections4.CollectionUtils;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-public class RecommendStatusFilter extends AbstractFilter<Video> {
-
-    private final RecommendStatusStrategy recommendStatusStrategy;
-
-    public RecommendStatusFilter(FilterConfigInfo filterConfigInfo,
-                                 RecommendRequest recommendRequest,
-                                 User user) {
-        super(filterConfigInfo, recommendRequest, user);
-        recommendStatusStrategy = SpringContextHolder.getBean(RecommendStatusStrategy.class);
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> videos) {
-        if (CollectionUtils.isEmpty(videos)) {
-            return;
-        }
-        FilterParam filterParam = new FilterParam();
-        filterParam.setVideoIds(videos.stream().map(Video::getVideoId).collect(Collectors.toList()));
-        List<Long> videoIdList = recommendStatusStrategy.filter(filterParam);
-        Set<Long> videoIdSet = new HashSet<>(videoIdList);
-        videos.removeIf(video -> !videoIdSet.contains(video.getVideoId()));
-    }
-
-}
-
-
-
-

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/RiskVideoFilter.java

@@ -1,28 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-
-import java.util.List;
-
-public class RiskVideoFilter extends AbstractFilter<Video> {
-
-
-    public RiskVideoFilter(FilterConfigInfo filterConfigInfo,
-                           RecommendRequest recommendRequest,
-                           User user) {
-        super(filterConfigInfo, recommendRequest, user);
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> t) {
-
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/TagFilter.java

@@ -1,105 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Lists;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
-import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.math.NumberUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-public class TagFilter extends AbstractFilter<Video> {
-
-    @Autowired
-    private WxVideoTagRelRepository repository;
-
-    @Value("${video.filter.tagids:}")
-    private String videoFilterTagIds;
-
-    private LoadingCache<Long, Set<Long>> videoTagCache = CacheBuilder.newBuilder()
-            .maximumSize(100)
-            .refreshAfterWrite(60, TimeUnit.SECONDS)
-            .expireAfterWrite(60, TimeUnit.SECONDS)
-            .expireAfterAccess(60, TimeUnit.SECONDS)
-            .build(new CacheLoader<Long, Set<Long>>() {
-                @Override
-                public Set<Long> load(Long tagId) {
-                    List<WxVideoTagRel> rels = repository.findAllByTagId(tagId);
-                    return CommonCollectionUtils.toSet(rels, WxVideoTagRel::getVideoId);
-                }
-            });
-
-
-    public TagFilter(FilterConfigInfo filterConfigInfo,
-                     RecommendRequest recommendRequest,
-                     User user) {
-        super(filterConfigInfo, recommendRequest, user);
-
-        if (org.apache.commons.lang.StringUtils.isNotBlank(this.videoFilterTagIds)) {
-            String[] tags = this.videoFilterTagIds.split(",");
-            for (String tag : tags) {
-                if (org.apache.commons.lang.StringUtils.isBlank(tag)) {
-                    continue;
-                }
-                this.videoTagCache.getUnchecked(NumberUtils.toLong(tag));
-            }
-        }
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> t) {
-        if (CollectionUtils.isEmpty(t)) {
-            return;
-        }
-        List<Video> videos = Lists.newArrayList();
-        for (Video video : t) {
-            if (!hasVideoRelTagId(video.getVideoId(), videoFilterTagIds)) {
-                videos.add(video);
-            }
-        }
-        t.removeAll(videos);
-    }
-
-    private boolean hasVideoRelTagId(Long videoId, String videoFilterTagIds) {
-        // TODO 主要是涉政标签
-        List<Long> tagIds = new ArrayList<>();
-
-        String[] tags = videoFilterTagIds.split(",");
-        for (String tag : tags) {
-            if (Objects.isNull(tag) || Objects.equals("", tag)) {
-                continue;
-            }
-            tagIds.add(Long.parseLong(tag));
-        }
-
-        for (Long tagId : tagIds) {
-            if (Objects.isNull(videoId) || videoId <= 0L || Objects.isNull(tagId) || tagId <= 0L) {
-                return false;
-            }
-            Set<Long> videos = this.videoTagCache.getUnchecked(tagId);
-            if (CollectionUtils.isEmpty(videos)) {
-                return false;
-            }
-            return videos.contains(videoId);
-        }
-        return false;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/ViewedHistoryFilter.java

@@ -1,115 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
-import com.tzld.piaoquan.recommend.server.service.SpringContextHolder;
-import com.tzld.piaoquan.recommend.server.service.filter.VideoCityFilterService;
-import com.tzld.piaoquan.recommend.server.service.filter.strategy.VideoView;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.time.StopWatch;
-import org.springframework.data.mongodb.core.MongoTemplate;
-import org.springframework.data.mongodb.core.query.Criteria;
-import org.springframework.data.mongodb.core.query.Query;
-import org.springframework.data.redis.core.RedisTemplate;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-public class ViewedHistoryFilter extends AbstractFilter<Video> {
-
-    private final VideoCityFilterService videoCityFilterService;
-
-    protected Set<String> historySet;
-
-    private final RedisTemplate<String, String> redisTemplate;
-
-    private final MongoTemplate mongoTemplate;
-
-    private final String keyFormat = "user:exclude:videoidset:%s";
-
-    public ViewedHistoryFilter(FilterConfigInfo filterConfigInfo,
-                               RecommendRequest recommendRequest,
-                               User user) {
-        super(filterConfigInfo, recommendRequest, user);
-        mongoTemplate = ServiceBeanFactory.getBean(MongoTemplate.class);
-        redisTemplate = SpringContextHolder.getBean("filterRedisTemplate", RedisTemplate.class);
-        historySet = this.loadUserHistory(user);
-        if (historySet == null) {
-            historySet = new HashSet<>();
-        }
-        videoCityFilterService = SpringContextHolder.getBean(VideoCityFilterService.class);
-
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> videoList) {
-        if (CollectionUtils.isEmpty(videoList)) {
-            return;
-        }
-        videoList.removeIf(video -> this.historySet.contains(String.valueOf(video.getVideoId())));
-        if (CollectionUtils.isEmpty(videoList)) {
-            return;
-        }
-        List<Long> filterVideosByCity = videoCityFilterService.filterVideosByCity(videoList.stream().map(Video::getVideoId).collect(Collectors.toList()),
-                new HashSet<>(requestContext.getAbExpCodeList()), requestContext.getHotSceneType(), requestContext.getCityCode());
-        Set<Long> filterVideosSet = new HashSet<>(filterVideosByCity);
-        videoList.removeIf(video -> !filterVideosSet.contains(video.getVideoId()));
-    }
-
-
-    public Set<String> loadUserHistory(User user) {
-        String userid = StringUtils.isNotBlank(user.getUid()) ? user.getUid() : user.getMid();
-        if (StringUtils.isBlank(userid)) {
-            return new HashSet<>();
-        }
-
-        String key = String.format(keyFormat, userid);
-        Set<String> viewedVideoIds = redisTemplate.opsForSet().members(key);
-        if (CollectionUtils.isEmpty(viewedVideoIds)) {
-            viewedVideoIds = new HashSet<>();
-            StopWatch stopWatch = new StopWatch();
-            stopWatch.start();
-            // 从mongodb中取曝光记录
-            Criteria criteriatime = new Criteria();
-            criteriatime.and("uid").is(userid);
-            Query query = new Query();
-            query.addCriteria(criteriatime);
-            //查新库
-            List<VideoView> list = mongoTemplate.find(query, VideoView.class);
-            //限制最多10000条
-            if (Objects.nonNull(list) && list.size() > 10000) {
-                list = list.subList(list.size() - 10000, list.size());
-            }
-            if (CollectionUtils.isNotEmpty(list)) {
-                viewedVideoIds.addAll(list.stream().map(VideoView::getVideoId).map(String::valueOf).collect(Collectors.toSet()));
-            } else {
-                // 避免缓存击穿
-                viewedVideoIds.add("-1");
-            }
-            stopWatch.stop();
-
-            redisTemplate.opsForSet().add(key, viewedVideoIds.toArray(new String[0]));
-            redisTemplate.expire(key, 1, TimeUnit.DAYS);
-        }
-
-        return viewedVideoIds;
-    }
-
-
-
-
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/BaseLRModelScorer.java

@@ -1,22 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.score;
-
-import com.tzld.piaoquan.recommend.server.framework.score.AbstractScorer;
-import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
-import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public abstract class BaseLRModelScorer extends AbstractScorer {
-
-    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRModelScorer.class);
-
-    public BaseLRModelScorer(ScorerConfigInfo scorerConfigInfo) {
-        super(scorerConfigInfo);
-    }
-
-    @Override
-    public void loadModel() {
-        doLoadModel(LRModel.class);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogShareLRScorer.java

@@ -1,328 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.score;
-
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
-import com.tzld.piaoquan.recommend.feature.domain.video.feature.VlogShareLRFeatureExtractor;
-import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
-import com.tzld.piaoquan.recommend.server.service.rank.strategy.OfflineVlogShareLRFeatureExtractor;
-import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-
-public class VlogShareLRScorer extends BaseLRModelScorer {
-
-    private final static int CORE_POOL_SIZE = 64;
-
-    private static final int LOCAL_TIME_OUT = 150;
-    private final static Logger LOGGER = LoggerFactory.getLogger(VlogShareLRScorer.class);
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
-    private static final double defaultUserCtrGroupNumber = 10.0;
-    private static final int enterFeedsScoreRatio = 10;
-    private static final int enterFeedsScoreNum = 20;
-
-
-    public VlogShareLRScorer(ScorerConfigInfo configInfo) {
-        super(configInfo);
-    }
-
-
-    @Override
-    public List<RankItem> scoring(final ScoreParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
-
-        if (CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<RankItem> result = rankItems;
-        result = rankByJava(rankItems, param.getRequestContext(),
-                userFeature == null ? UserFeature.defaultInstance(param.getMid()) : userFeature);
-
-        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<RankItem> rankByJava(final List<RankItem> items,
-                                      final RequestContext requestContext,
-                                      final UserFeature user) {
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        // userBytes
-        UserBytesFeature userInfoBytes = null;
-        userInfoBytes = new UserBytesFeature(user);
-
-        // 所有都参与打分，按照ctr排序
-        multipleCtrScore(items, userInfoBytes, requestContext, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
-            }
-        }
-
-        Collections.sort(items);
-
-        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-
-    /**
-     * 计算 predict ctr
-     */
-    public double calcScore(final LRModel lrModel,
-                            final RankItem item,
-                            final UserBytesFeature userInfoBytes,
-                            final RequestContext requestContext) {
-
-        LRSamples lrSamples = null;
-        VlogShareLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new VlogShareLRFeatureExtractor();
-
-        try {
-            VideoBytesFeature newsInfoBytes = new VideoBytesFeature(item.getItemFeature() == null
-                    ? ItemFeature.defaultInstance(item.getVideoId() + "")
-                    : item.getItemFeature());
-            lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
-                    new RequestContextBytesFeature(requestContext));
-        } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e));
-        }
-
-
-        double pro = 0.0;
-        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
-            try {
-                pro = lrModel.score(lrSamples);
-            } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
-            }
-            // 增加实时特征后打开在线存储日志逻辑
-            //
-            // CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
-            // samples.setLr_samples(lrSamples);
-            // item.setSamples(samples);
-            //
-        }
-        item.setScore(pro);
-        return pro;
-    }
-
-
-    /**
-     * 并行打分
-     *
-     * @param items
-     * @param userInfoBytes
-     * @param requestContext
-     * @param model
-     */
-    private void multipleCtrScore(final List<RankItem> items,
-                                  final UserBytesFeature userInfoBytes,
-                                  final RequestContext requestContext,
-                                  final LRModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
-                    } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", requestContext.getRequest_id(),
-                            ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
-    }
-    @Override
-    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
-                                           final Map<String, String> userFeatureMap,
-                                           final List<RankItem> rankItems){
-        if (CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<RankItem> result = rankItems;
-        result = rankByJava(
-                sceneFeatureMap, userFeatureMap, rankItems
-        );
-
-        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
-                                      final Map<String, String> userFeatureMap,
-                                      final List<RankItem> items) {
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-        // userBytes
-        Map<String, byte[]> userFeatureMapByte = new HashMap<>();
-        for(Map.Entry<String, String> entry: userFeatureMap.entrySet()){
-            userFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
-        }
-        //sceneBytes
-        Map<String, byte[]> sceneFeatureMapByte = new HashMap<>();
-        for(Map.Entry<String, String> entry: sceneFeatureMap.entrySet()){
-            sceneFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
-        }
-
-        // 所有都参与打分，按照ctr排序
-        multipleCtrScore(items, userFeatureMapByte, sceneFeatureMapByte, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
-            }
-        }
-
-        Collections.sort(items);
-
-        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-    private void multipleCtrScore(final List<RankItem> items,
-                                  final Map<String, byte[]> userFeatureMapByte,
-                                  final Map<String, byte[]> sceneFeatureMapByte,
-                                  final LRModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-            // items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex), userFeatureMapByte, sceneFeatureMapByte);
-                    } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", sceneFeatureMapByte.size(),
-                            ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
-    }
-
-    public double calcScore(final LRModel lrModel,
-                            final RankItem item,
-                            final Map<String, byte[]> userFeatureMapByte,
-                            final Map<String, byte[]> sceneFeatureMapByte) {
-
-        LRSamples lrSamples = null;
-        OfflineVlogShareLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new OfflineVlogShareLRFeatureExtractor();
-
-        try {
-
-            Map<String, byte[]> itemFeatureByte = new HashMap<>();
-            for (Map.Entry<String, String> entry: item.getFeatureMap().entrySet()){
-                itemFeatureByte.put(entry.getKey(), entry.getValue().getBytes());
-            }
-            lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
-        } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e));
-        }
-
-
-        double pro = 0.0;
-        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
-            try {
-                pro = lrModel.score(lrSamples);
-            } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
-            }
-        }
-        item.setScoreStr(pro);
-        return pro;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogShareLRScorer4Ros.java

@@ -1,328 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.score;
-
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
-import com.tzld.piaoquan.recommend.feature.domain.video.feature.VlogShareLRFeatureExtractor;
-import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
-import com.tzld.piaoquan.recommend.server.service.rank.strategy.OfflineVlogShareLRFeatureExtractor;
-import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-
-public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
-
-    private final static int CORE_POOL_SIZE = 64;
-
-    private static final int LOCAL_TIME_OUT = 150;
-    private final static Logger LOGGER = LoggerFactory.getLogger(VlogShareLRScorer4Ros.class);
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
-    private static final double defaultUserCtrGroupNumber = 10.0;
-    private static final int enterFeedsScoreRatio = 10;
-    private static final int enterFeedsScoreNum = 20;
-
-
-    public VlogShareLRScorer4Ros(ScorerConfigInfo configInfo) {
-        super(configInfo);
-    }
-
-
-    @Override
-    public List<RankItem> scoring(final ScoreParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
-
-        if (CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<RankItem> result = rankItems;
-        result = rankByJava(rankItems, param.getRequestContext(),
-                userFeature == null ? UserFeature.defaultInstance(param.getMid()) : userFeature);
-
-        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<RankItem> rankByJava(final List<RankItem> items,
-                                      final RequestContext requestContext,
-                                      final UserFeature user) {
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        // userBytes
-        UserBytesFeature userInfoBytes = null;
-        userInfoBytes = new UserBytesFeature(user);
-
-        // 所有都参与打分，按照ctr排序
-        multipleCtrScore(items, userInfoBytes, requestContext, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
-            }
-        }
-
-        Collections.sort(items);
-
-        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-
-    /**
-     * 计算 predict ctr
-     */
-    public double calcScore(final LRModel lrModel,
-                            final RankItem item,
-                            final UserBytesFeature userInfoBytes,
-                            final RequestContext requestContext) {
-
-        LRSamples lrSamples = null;
-        VlogShareLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new VlogShareLRFeatureExtractor();
-
-        try {
-            VideoBytesFeature newsInfoBytes = new VideoBytesFeature(item.getItemFeature() == null
-                    ? ItemFeature.defaultInstance(item.getVideoId() + "")
-                    : item.getItemFeature());
-            lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
-                    new RequestContextBytesFeature(requestContext));
-        } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e));
-        }
-
-
-        double pro = 0.0;
-        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
-            try {
-                pro = lrModel.score(lrSamples);
-            } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
-            }
-            // 增加实时特征后打开在线存储日志逻辑
-            //
-            // CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
-            // samples.setLr_samples(lrSamples);
-            // item.setSamples(samples);
-            //
-        }
-        item.setScore(pro);
-        return pro;
-    }
-
-
-    /**
-     * 并行打分
-     *
-     * @param items
-     * @param userInfoBytes
-     * @param requestContext
-     * @param model
-     */
-    private void multipleCtrScore(final List<RankItem> items,
-                                  final UserBytesFeature userInfoBytes,
-                                  final RequestContext requestContext,
-                                  final LRModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
-                    } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", requestContext.getRequest_id(),
-                            ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
-    }
-    @Override
-    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
-                                  final Map<String, String> userFeatureMap,
-                                  final List<RankItem> rankItems){
-        if (CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<RankItem> result = rankItems;
-        result = rankByJava(
-                sceneFeatureMap, userFeatureMap, rankItems
-        );
-
-        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
-                                      final Map<String, String> userFeatureMap,
-                                      final List<RankItem> items) {
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-        // userBytes
-        Map<String, byte[]> userFeatureMapByte = new HashMap<>();
-        for(Map.Entry<String, String> entry: userFeatureMap.entrySet()){
-            userFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
-        }
-        //sceneBytes
-        Map<String, byte[]> sceneFeatureMapByte = new HashMap<>();
-        for(Map.Entry<String, String> entry: sceneFeatureMap.entrySet()){
-            sceneFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
-        }
-
-        // 所有都参与打分，按照ctr排序
-        multipleCtrScore(items, userFeatureMapByte, sceneFeatureMapByte, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
-            }
-        }
-
-        Collections.sort(items);
-
-        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-    private void multipleCtrScore(final List<RankItem> items,
-                                  final Map<String, byte[]> userFeatureMapByte,
-                                  final Map<String, byte[]> sceneFeatureMapByte,
-                                  final LRModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-//            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex), userFeatureMapByte, sceneFeatureMapByte);
-                    } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", sceneFeatureMapByte.size(),
-                            ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
-    }
-
-    public double calcScore(final LRModel lrModel,
-                            final RankItem item,
-                            final Map<String, byte[]> userFeatureMapByte,
-                            final Map<String, byte[]> sceneFeatureMapByte) {
-
-        LRSamples lrSamples = null;
-        OfflineVlogShareLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new OfflineVlogShareLRFeatureExtractor();
-
-        try {
-
-            Map<String, byte[]> itemFeatureByte = new HashMap<>();
-            for (Map.Entry<String, String> entry: item.getFeatureMap().entrySet()){
-                itemFeatureByte.put(entry.getKey(), entry.getValue().getBytes());
-            }
-            lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
-        } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e));
-        }
-
-
-        double pro = 0.0;
-        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
-            try {
-                pro = lrModel.score(lrSamples);
-            } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
-            }
-        }
-        item.setScoreRos(pro);
-        return pro;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogThompsonScorer.java

@@ -1,143 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.score;
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.score.BaseThompsonSamplingScorer;
-import com.tzld.piaoquan.recommend.server.framework.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
-import com.tzld.piaoquan.recommend.server.service.score.model.ThompsonSamplingModel;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.*;
-
-
-//@Service
-public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
-
-    private static final int LOCAL_TIME_OUT = 150;
-    private final static Logger LOGGER = LoggerFactory.getLogger(VlogThompsonScorer.class);
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
-
-    public VlogThompsonScorer(ScorerConfigInfo configInfo) {
-        super(configInfo);
-    }
-
-    @Override
-    public List<RankItem> scoring(final ScoreParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
-
-        if (userFeature == null || CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<RankItem> result = rankItems;
-        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
-
-        LOGGER.debug("thompson sampling ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<RankItem> rankByJava(final List<RankItem> items,
-                                      final RequestContext requestContext,
-                                      final UserFeature user) {
-        long startTime = System.currentTimeMillis();
-        ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        // 所有都参与打分，按照ROV Thompson排序
-        multipleCtrScore(items, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
-            }
-        }
-
-        LOGGER.debug("thompson ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[thompson ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-
-    /**
-     * 计算 predict ROV
-     */
-    public double calcScore(final ThompsonSamplingModel model,
-                            final RankItem item) {
-        double score = 0d;
-        try {
-            score = model.score(item);
-        } catch (Exception e) {
-            LOGGER.error("score error for doc={} exception={}", item.getVideo(), ExceptionUtils.getFullStackTrace(e));
-        }
-        item.setScore(score);
-        return score;
-    }
-
-
-    /**
-     * 并行打分 Thompson ROV
-     *
-     * @param items
-     * @param model
-     */
-    private void multipleCtrScore(final List<RankItem> items,
-                                  final ThompsonSamplingModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-            items.get(fIndex).setScore(0.0);   //设置为原始值为0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex));
-                    } catch (Exception e) {
-                        LOGGER.error("thompson exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("ROV-Thompson Score {}, Total: {}, Cancel: {}", items.size(), cancel);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/SpringContextHolder.java

@@ -1,33 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service;
-
-import org.springframework.beans.BeansException;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
-import org.springframework.stereotype.Service;
-
-@Service
-public class SpringContextHolder implements ApplicationContextAware {
-
-    private static ApplicationContext applicationContext;
-
-    @Override
-    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
-        SpringContextHolder.applicationContext = applicationContext;
-    }
-
-    public static ApplicationContext getApplicationContext() {
-        return applicationContext;
-    }
-
-    public static Object getBean(String beanName) {
-        return applicationContext.getBean(beanName);
-    }
-
-    public static <T> T getBean(String beanName, Class<T> clazz) {
-        return applicationContext.getBean(beanName, clazz);
-    }
-
-    public static <T> T getBean(Class<T> clazz) {
-        return applicationContext.getBean(clazz);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/VideoRecommendService.java

@@ -1,222 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service;
-
-import com.alibaba.fastjson.JSONObject;
-import com.google.common.base.Stopwatch;
-import com.google.common.base.Strings;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.common.Result;
-import com.tzld.piaoquan.recommend.server.gen.recommend.PushFromIndex;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendResponse;
-import com.tzld.piaoquan.recommend.server.gen.recommend.VideoProto;
-import com.tzld.piaoquan.recommend.server.implement.FlowPoolRecommendPipeline;
-import com.tzld.piaoquan.recommend.server.implement.TopRecommendPipeline;
-import com.tzld.piaoquan.recommend.server.model.RecommendParam;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.strategy.RankStrategy4RegionMergeModelV547;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.Resource;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author sunxy
- */
-@Service
-@Slf4j
-public class VideoRecommendService {
-
-    @Value("${recommend.new.log.print:true}")
-    private Boolean newLogPrint;
-    @Resource
-    private RecommendService recommendService;
-    @Resource
-    private TopRecommendPipeline topRecommendPipeline;
-    @Resource
-    private FlowPoolRecommendPipeline flowPoolRecommendPipeline;
-    @Resource
-    private RankStrategy4RegionMergeModelV547 rankStrategy4RegionMergeModelV547;
-    @Resource
-    private TimerLogService timerLogService;
-    @Autowired
-    @Qualifier("redisTemplate")
-    private RedisTemplate<String, String> redisTemplate;
-
-    @Value("${recall.framework.exp:code:610}")
-    private String recallFrameworkExpCode;
-
-
-    public RecommendResponse homepageRecommend(RecommendRequest request) {
-        boolean hit = CommonCollectionUtils.contains(request.getAbExpCodeList(), recallFrameworkExpCode);
-        if (hit) {
-            RecommendParam recommendParam = recommendService.genRecommendParam(request, 0);
-            return recommendNew(request, recommendParam, 0);
-        }
-        return recommend(request, 0);
-    }
-
-
-    public RecommendResponse relevantRecommend(RecommendRequest request) {
-        boolean hit = CommonCollectionUtils.contains(request.getAbExpCodeList(), recallFrameworkExpCode);
-        if (hit) {
-            RecommendParam recommendParam = recommendService.genRecommendParam(request, 1);
-            return recommendNew(request, recommendParam, 1);
-        }
-        return recommend(request, 1);
-    }
-
-    private RecommendResponse recommendNew(RecommendRequest request, RecommendParam param, int recommendType) {
-        if (request == null) {
-            return RecommendResponse.newBuilder()
-                    .setResult(Result.newBuilder().setCode(1).setMessage("success"))
-                    .build();
-        }
-        if (request.getVersionAuditStatus() == 1) {
-            return recommendService.specialMidRecommend(request);
-        }
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        if (StringUtils.isNotBlank(request.getMid())
-                && redisTemplate.opsForSet().isMember("special:mid", request.getMid())) {
-            return recommendService.specialMidRecommend(request);
-        }
-        stopwatch.reset().start();
-
-        log.info("genRecommendParam={},genRecommendParam cost={}", JSONUtils.toJson(param),
-                stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
-        stopwatch.reset().start();
-        List<Video> videos = videoRecommend(request, param, recommendType);
-        recommendService.updateCache(request, param, videos);
-
-        // 更新position
-        List<VideoProto> vps = new ArrayList<>();
-        for (int i = 0; i < videos.size(); i++) {
-            Map<String, List<String>> pushFromIndex = videos.get(i).getPushFromIndex();
-            List<PushFromIndex> pushFromIndexList = new ArrayList<>();
-            if (MapUtils.isNotEmpty(pushFromIndex)) {
-                pushFromIndex.forEach((k, v) -> {
-                    PushFromIndex.Builder builder = PushFromIndex.newBuilder().setPushFrom(k);
-                    v.forEach(builder::addIndex);
-                    pushFromIndexList.add(builder.build());
-                });
-            }
-            vps.add(VideoProto.newBuilder()
-                    .setPosition(i + 1)
-                    .setPushFrom(Strings.nullToEmpty(videos.get(i).getPushFrom()))
-                    .setAbCode(Strings.nullToEmpty(videos.get(i).getAbCode()))
-                    .setVideoId(videos.get(i).getVideoId())
-                    .setRovScore(videos.get(i).getRovScore())
-                    .setSortScore(videos.get(i).getSortScore())
-                    .setFlowPool(Strings.nullToEmpty(videos.get(i).getFlowPool()))
-                    .setIsInFlowPool(videos.get(i).isInFlowPool() ? 1 : 0)
-                    .setRand(videos.get(i).getRand())
-                    .addAllPushFromIndex(pushFromIndexList)
-                    .build());
-        }
-
-        return RecommendResponse.newBuilder()
-                .setResult(Result.newBuilder().setCode(1).setMessage("success"))
-                .addAllVideo(vps)
-                .build();
-    }
-
-    private List<Video> videoRecommend(RecommendRequest request, RecommendParam param, int recommendType) {
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        User user = getUser(request);
-        List<Video> topRecommendVideoList = new ArrayList<>();
-        List<Video> flowPoolRecommendVideoList = new ArrayList<>();
-        Map<String, String> timeLogMap = new HashMap<>(32);
-        try {
-            topRecommendVideoList = topRecommendPipeline.feeds(request, 0, user, newLogPrint, timeLogMap);
-            if (newLogPrint) {
-                log.info("traceId = {}, cost = {}, topRecommendVideoList [{}]", request.getRequestId(),
-                        stopwatch.elapsed().toMillis(), JSONObject.toJSONString(topRecommendVideoList));
-            }
-
-        } catch (Exception e) {
-            log.error("traceId = {}, topRecommendVideoList error", request.getRequestId(), e);
-        } finally {
-        }
-        try {
-            stopwatch.reset().start();
-            flowPoolRecommendVideoList = flowPoolRecommendPipeline.feedByRec(request, recommendType, newLogPrint);
-            if (newLogPrint) {
-                log.info("traceId = {}, cost = {}, flowPoolRecommendVideoList [{}]", request.getRequestId(),
-                        stopwatch.elapsed().toMillis(), JSONObject.toJSONString(flowPoolRecommendVideoList));
-            }
-        } catch (Exception e) {
-            log.error("traceId = {}, flowPoolRecommendVideoList error", request.getRequestId(), e);
-        }
-
-        stopwatch.reset().start();
-        RankParam rankParam = recommendService.convertToRankParam(param, null);
-        RankResult rankResult = rankStrategy4RegionMergeModelV547.mergeAndSort(rankParam, topRecommendVideoList,
-                flowPoolRecommendVideoList);
-        if (newLogPrint) {
-            log.info("traceId = {}, cost = {}, final rankResult [{}]", request.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONObject.toJSONString(rankResult));
-        }
-
-        // 只返回size条数据
-        List<Video> videos = rankResult.getVideos();
-        if (param.getSize() < rankResult.getVideos().size()) {
-            videos = rankResult.getVideos().subList(0, param.getSize());
-        }
-
-        // 更新position
-        List<VideoProto> vps = new ArrayList<>();
-        for (int i = 0; i < videos.size(); i++) {
-            Map<String, List<String>> pushFromIndex = videos.get(i).getPushFromIndex();
-            List<PushFromIndex> pushFromIndexList = new ArrayList<>();
-            if (MapUtils.isNotEmpty(pushFromIndex)) {
-                pushFromIndex.forEach((k, v) -> {
-                    PushFromIndex.Builder builder = PushFromIndex.newBuilder().setPushFrom(k);
-                    v.forEach(builder::addIndex);
-                    pushFromIndexList.add(builder.build());
-                });
-            }
-            vps.add(VideoProto.newBuilder()
-                    .setPosition(i + 1)
-                    .setPushFrom(Strings.nullToEmpty(videos.get(i).getPushFrom()))
-                    .setAbCode(Strings.nullToEmpty(videos.get(i).getAbCode()))
-                    .setVideoId(videos.get(i).getVideoId())
-                    .setRovScore(videos.get(i).getRovScore())
-                    .setSortScore(videos.get(i).getSortScore())
-                    .setFlowPool(Strings.nullToEmpty(videos.get(i).getFlowPool()))
-                    .setIsInFlowPool(videos.get(i).isInFlowPool() ? 1 : 0)
-                    .setRand(videos.get(i).getRand())
-                    .addAllPushFromIndex(pushFromIndexList)
-                    .build());
-        }
-        return videos;
-    }
-
-
-    private User getUser(RecommendRequest request) {
-        User user = new User();
-        user.setMid(request.getMid());
-        user.setUid(request.getUid());
-        user.setRegion(request.getProvince());
-        return user;
-    }
-
-    private RecommendResponse recommend(RecommendRequest request, int recommendType) {
-        return recommendService.recommend(request, recommendType);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/ViewedService.java

@@ -1,115 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service;
-
-import com.google.common.collect.Lists;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.HttpClientFactory;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.PostConstruct;
-import java.util.*;
-
-/**
- * @author dyp
- */
-@Service
-@Slf4j
-public class ViewedService {
-    private final Map<Integer, List<Integer>> viewedTypesMap = new HashMap<>();
-    private final List<Integer> defaultViewedTypes = Lists.newArrayList(1, 6, 7, 5, 9);
-
-    @Value("${video.filter.url:}")
-    private String videoFilterUrl;
-    @Value("${securityAbExpCode:}")
-    private String securityAbExpCode;
-
-    private final CloseableHttpClient client = HttpClientFactory.create(1000, 1000, 200, 200, 0, 1000);
-
-
-    @PostConstruct
-    public void init() {
-//        1-已观看 2-视频状态 3-是否进入老年人社区 4-话题状态 5-推荐状态 6-白名单过滤 7-涉政视频过滤
-        viewedTypesMap.put(0, Lists.newArrayList(1, 6, 5, 9));
-        viewedTypesMap.put(4, Lists.newArrayList(1, 6, 5, 9));
-        viewedTypesMap.put(13, Lists.newArrayList(1, 5, 9));
-    }
-
-
-    // TODO 如果过滤失败，那么认为所有视频都被过滤掉
-    public List<Long> filterViewedVideo(int appType, String mid, String uid, List<Long> videoIds, String cityCode,
-                                        Set<String> abExpCodes, Long hotSceneType, String clientIp) {
-
-        List<Integer> viewedTypes = new ArrayList<>(viewedTypesMap.getOrDefault(appType, defaultViewedTypes));
-        if (CommonCollectionUtils.contains(abExpCodes, securityAbExpCode)) {
-            viewedTypes.add(8);
-        }
-        CloseableHttpResponse chr = null;
-        try {
-            HttpPost post = new HttpPost(videoFilterUrl);
-            post.addHeader("Connection", "close");
-            post.addHeader("content-type", "application/json");
-            Map<String, Object> param = new HashMap<>();
-            param.put("appType", appType);
-            param.put("mid", mid);
-            param.put("uid", uid);
-            param.put("types", viewedTypes);
-            param.put("videoIds", videoIds);
-            param.put("cityCode", cityCode);
-            param.put("hotSenceType", hotSceneType);
-            param.put("clientIp", clientIp);
-            param.put("abExpCodes", abExpCodes);
-            List<Integer> recommendStatus = new ArrayList<>();
-            recommendStatus.add(-6);
-            param.put("recommendStatus", recommendStatus);
-            post.setEntity(new StringEntity(JSONUtils.toJson(param)));
-
-            chr = client.execute(post);
-            if (chr == null
-                    || chr.getStatusLine() == null
-                    || chr.getStatusLine().getStatusCode() != 200) {
-                log.error("filterViewedVideo failed http status exception!");
-                return Collections.emptyList();
-            }
-            HttpEntity entity = chr.getEntity();
-            if (entity == null) {
-                return Collections.emptyList();
-            }
-            String content = EntityUtils.toString(entity);
-            Map<String, String> data = JSONUtils.fromJson(content,
-                    new TypeToken<Map<String, String>>() {
-                    },
-                    Collections.emptyMap());
-            if (data.get("code") == null
-                    || !data.get("code").equals("0")) {
-                log.error("filterViewedVideo failed videoFilterUrl return code exception! data={}",
-                        JSONUtils.toJson(data));
-                return Collections.emptyList();
-            }
-            return JSONUtils.fromJson(data.get("data"),
-                    new TypeToken<List<Long>>() {
-                    }, Collections.emptyList());
-
-        } catch (Exception e) {
-            log.error("invoke http filterViewedVideo error", e);
-        } finally {
-            try {
-                if (chr != null) {
-                    chr.close();
-                }
-            } catch (Exception ex) {
-                log.error("close CloseableHttpResponse error", ex);
-            }
-        }
-        return Collections.emptyList();
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/WarmUpService.java

@@ -2,7 +2,6 @@ package com.tzld.piaoquan.recommend.server.service;
 
 import com.tzld.piaoquan.recommend.server.repository.WxVideoStatusRepository;
 import com.tzld.piaoquan.recommend.server.util.SimilarityUtils;
-import com.tzld.piaoquan.recommend.similarity.word2vec.Segment;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.core.annotation.Order;
@@ -35,7 +34,6 @@ public class WarmUpService {
         featureRedisTemplate.opsForValue().get("");
         longVideoRedisTemplate.opsForValue().get("");
         com.tzld.piaoquan.recommend.server.service.score.ScorerUtils.warmUp();
-        com.tzld.piaoquan.recommend.server.framework.score.ScorerUtils.warmUp();
         wxVideoStatusRepository.count();
         SimilarityUtils.init();
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/web/RecommendController.java

@@ -1,94 +0,0 @@
-package com.tzld.piaoquan.recommend.server.web;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.tzld.piaoquan.recommend.server.client.ProtobufUtils;
-import com.tzld.piaoquan.recommend.server.client.RecommendHttpRequest;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendResponse;
-import com.tzld.piaoquan.recommend.server.service.FeatureService;
-import com.tzld.piaoquan.recommend.server.service.VideoRecommendService;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.collections4.MapUtils;
-import org.slf4j.MDC;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-/**
- * @author dyp
- */
-@RestController
-@Slf4j
-@Deprecated
-public class RecommendController {
-    @Autowired
-    private VideoRecommendService videoRecommendService;
-
-    @Autowired
-    private FeatureService featureService;
-
-    @RequestMapping("/demo")
-    public String demo() {
-//        return JSONUtils.toJson(featureService.getFeature(Lists.newArrayList("10054148", "20637161"), "0", "1"));
-        return "";
-    }
-
-    @RequestMapping("/homepage/recommend")
-    public String homepageRecommend(@RequestBody RecommendHttpRequest httpRequest) {
-        MDC.put("appType", String.valueOf(httpRequest.getAppType()));
-        RecommendResponse response = videoRecommendService.homepageRecommend(generate(httpRequest));
-        String result = "";
-        try {
-            result = ProtobufUtils.toJson(response);
-        } catch (InvalidProtocolBufferException e) {
-            log.error("homepageRecommend ProtobufUtils.toJson", e);
-        }
-        return result;
-    }
-
-    @RequestMapping("/relevant/recommend")
-    public String relevantRecommend(@RequestBody RecommendHttpRequest httpRequest) {
-        MDC.put("appType", String.valueOf(httpRequest.getAppType()));
-
-        RecommendResponse response = videoRecommendService.relevantRecommend(generate(httpRequest));
-        String result = "";
-        try {
-            result = ProtobufUtils.toJson(response);
-        } catch (InvalidProtocolBufferException e) {
-            log.error("relevantRecommend ProtobufUtils.toJson", e);
-        }
-        return result;
-    }
-
-    private RecommendRequest generate(RecommendHttpRequest httpRequest) {
-        if (httpRequest == null) {
-            return null;
-        }
-        RecommendRequest.Builder builder = RecommendRequest.newBuilder();
-
-        builder.setRequestId(Strings.nullToEmpty(httpRequest.getRequestId()))
-                .setMid(Strings.nullToEmpty(httpRequest.getMid()))
-                .setUid(Strings.nullToEmpty(httpRequest.getUid()))
-                .setSize(httpRequest.getSize())
-                .setAppType(httpRequest.getAppType())
-                .setCityCode(Strings.nullToEmpty(httpRequest.getCityCode()))
-                .setProvinceCode(Strings.nullToEmpty(httpRequest.getProvinceCode()))
-                .setVersionAuditStatus(httpRequest.getVersionAuditStatus())
-                .setRecommendTraceId(Strings.nullToEmpty(httpRequest.getRecommendTraceId()))
-                .setVideoId(httpRequest.getVideoId());
-
-        if (CollectionUtils.isNotEmpty(httpRequest.getAbExpCodes())) {
-            builder.addAllAbExpCode(httpRequest.getAbExpCodes());
-        }
-        if (MapUtils.isNotEmpty(httpRequest.getEventIdMap())) {
-            builder.putAllEventId(httpRequest.getEventIdMap());
-        }
-
-        return builder.build();
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/web/RecommendV2Controller.java

@@ -4,7 +4,7 @@ import com.google.protobuf.InvalidProtocolBufferException;
 import com.tzld.piaoquan.recommend.server.client.ProtobufUtils;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendResponse;
-import com.tzld.piaoquan.recommend.server.service.VideoRecommendService;
+import com.tzld.piaoquan.recommend.server.service.RecommendService;
 import lombok.extern.slf4j.Slf4j;
 import org.slf4j.MDC;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -19,18 +19,12 @@ import org.springframework.web.bind.annotation.RestController;
 @Slf4j
 public class RecommendV2Controller {
     @Autowired
-    private VideoRecommendService videoRecommendService;
-//
-//    @Resource
-//    TopRecommendPipeline topRecommendPipeline;
-//
-//    @Resource
-//    FlowPoolRecommendPipeline flowPoolRecommendPipeline;
+    private RecommendService recommendService;
 
     @RequestMapping("/homepage/recommend/v2")
     public String homepageRecommend(@RequestBody RecommendRequest httpRequest) {
         MDC.put("appType", String.valueOf(httpRequest.getAppType()));
-        RecommendResponse response = videoRecommendService.homepageRecommend(httpRequest);
+        RecommendResponse response = recommendService.homepageRecommend(httpRequest);
         String result = "";
         try {
             result = ProtobufUtils.toJson(response);
@@ -44,7 +38,7 @@ public class RecommendV2Controller {
     public String relevantRecommend(@RequestBody RecommendRequest httpRequest) {
         MDC.put("appType", String.valueOf(httpRequest.getAppType()));
 
-        RecommendResponse response = videoRecommendService.relevantRecommend(httpRequest);
+        RecommendResponse response = recommendService.relevantRecommend(httpRequest);
         String result = "";
         try {
             result = ProtobufUtils.toJson(response);
@@ -54,33 +48,4 @@ public class RecommendV2Controller {
         return result;
     }
 
-//    @RequestMapping("/test/feedByRec")
-//    public String feedByRec(@RequestBody RecommendRequest httpRequest) {
-//        MDC.put("appType", String.valueOf(httpRequest.getAppType()));
-//
-//        User user = new User();
-//        user.setRegion(httpRequest.getCityCode());
-//        List<Video> videoList = topRecommendPipeline.feeds(httpRequest, 0, user, true);
-//        String result = "";
-//        try {
-//            result = JSONObject.toJSONString(videoList);
-//        } catch (Exception e) {
-//            log.error("feedByRec ProtobufUtils.toJson", e);
-//        }
-//        return result;
-//    }
-//
-//    @RequestMapping("/test/flowPool")
-//    public String flowPool(@RequestBody RecommendRequest httpRequest) {
-//        MDC.put("appType", String.valueOf(httpRequest.getAppType()));
-//
-//        List<Video> videoList = flowPoolRecommendPipeline.feedByRec(httpRequest, 0, true);
-//        String result = "";
-//        try {
-//            result = JSONObject.toJSONString(videoList);
-//        } catch (Exception e) {
-//            log.error("flowPool ProtobufUtils.toJson", e);
-//        }
-//        return result;
-//    }
 }

recommend-server-service/src/main/resources/attention_config.conf

@@ -1,6 +0,0 @@
-attention-extractor-config = {
-  categories-by-ctr-extractor = {
-    name = "com.tzld.piaoquan.recommend.server.implement.attention.SimpleAttentionExtractor"
-  }
-
-}

recommend-server-service/src/main/resources/filter_config.conf

@@ -1,13 +0,0 @@
-recall-config = {
-  filter-config = {
-    viewed-filter-config = {
-      filter-name = "com.tzld.piaoquan.recommend.server.implement.recall.ViewedHistoryFilter"
-    }
-    pre-viewed-filter-config = {
-      filter-name = "com.tzld.piaoquan.recommend.server.implement.recall.PreViewedFilter"
-    }
-    recommend-filter-config = {
-      filter-name = "com.tzld.piaoquan.recommend.server.implement.recall.RecommendStatusFilter"
-    }
-  }
-}

recommend-server-service/src/main/resources/merge_config.conf

@@ -1,91 +0,0 @@
-queue-config = {
-  top-queue = {
-    class = "com.tzld.piaoquan.recommend.server.framework.merger.SimpleMergeQueue"
-    children = {
-      region-hot-queue = {
-        class = "com.tzld.piaoquan.recommend.server.framework.merger.SimpleMergeQueue"
-        children = {
-          region1h-index = {
-            class = "com.tzld.piaoquan.recommend.server.implement.candidate.Region1hHotCandidate"
-          }
-          region3h-index = {
-            class = "com.tzld.piaoquan.recommend.server.implement.candidate.Region3hHotCandidate"
-          }
-          region24h-index = {
-            class = "com.tzld.piaoquan.recommend.server.implement.candidate.Region24hHotCandidate"
-          }
-        }
-      }
-      global-hot-queue = {
-          class = "com.tzld.piaoquan.recommend.server.framework.merger.SimpleMergeQueue"
-          children = {
-            global1h-index = {
-                class = "com.tzld.piaoquan.recommend.server.implement.candidate.Global1hHotCandidate"
-            }
-            global3h-index = {
-                class = "com.tzld.piaoquan.recommend.server.implement.candidate.Global3hHotCandidate"
-            }
-            global24h-index = {
-                class = "com.tzld.piaoquan.recommend.server.implement.candidate.Global24hHotCandidate"
-            }
-         }
-      }
-    }
-  }
-}
-
-rule-config = {
-  // 顶层队列
-  top-queue = {
-    merge-rule = {
-      global-hot-queue = {
-        recall-percentage = 0.5
-        max-merge-num = 400
-        priority = 1
-      }
-      region-hot-queue = {
-        recall-percentage = 0.5
-        max-merge-num = 400
-        priority = 1
-      }
-    }
-  }
-
-  // 全局热门队列
-  global-hot-queue = {
-    merge-rule = {
-      global1h-index = {
-        recall-percentage = 0.3
-        max-merge-num = 200
-      }
-      global3h-index = {
-        recall-percentage = 0.3
-        max-merge-num = 200
-      }
-      global24h-index = {
-        recall-percentage = 0.3
-        max-merge-num = 200
-      }
-    }
-  }
-
-
-  region-hot-queue = {
-    merge-rule = {
-      region1h-index = {
-        recall-percentage = 0.3
-        max-merge-num = 200
-
-      }
-      region3h-index = {
-        recall-percentage = 0.3
-        max-merge-num = 200
-      }
-      region24h-index = {
-        recall-percentage = 0.3
-        max-merge-num = 200
-      }
-    }
-  }
-
-}