|
|
@@ -2,6 +2,7 @@ package com.tzld.piaoquan.recommend.server.implement;
|
|
|
|
|
|
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
import com.google.common.reflect.TypeToken;
|
|
|
import com.tzld.piaoquan.recommend.server.common.base.RankItem;
|
|
|
import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
|
|
|
@@ -13,6 +14,8 @@ import com.tzld.piaoquan.recommend.server.framework.recaller.provider.RedisBacke
|
|
|
import com.tzld.piaoquan.recommend.server.framework.score.ScorerUtils;
|
|
|
import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
|
|
|
import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
|
|
|
+import com.tzld.piaoquan.recommend.server.model.Video;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
|
|
|
import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
|
|
|
@@ -30,6 +33,10 @@ import java.util.stream.Collectors;
|
|
|
@Service
|
|
|
public class TopRecommendPipeline {
|
|
|
|
|
|
+
|
|
|
+ @ApolloJsonValue("${rank.score.merge.weight:}")
|
|
|
+ private Map<String, Double> mergeWeight;
|
|
|
+
|
|
|
private static final Logger log = LoggerFactory.getLogger(TopRecommendPipeline.class);
|
|
|
|
|
|
public static final String FILTER_CONF = "filter_config.conf";
|
|
|
@@ -37,15 +44,158 @@ public class TopRecommendPipeline {
|
|
|
|
|
|
public static final String PREFIX = "";
|
|
|
|
|
|
+
|
|
|
@Resource
|
|
|
private RedisSmartClient client;
|
|
|
@Resource
|
|
|
public RedisTemplate<String, String> redisTemplate;
|
|
|
|
|
|
+ public List<Video> feeds(final RecommendRequest requestData,
|
|
|
+ final int requestIndex,
|
|
|
+ final User userInfo) {
|
|
|
+ // Step 1: Attention extraction
|
|
|
+ List<RankItem> rankItems = feedByRec(requestData, requestIndex, userInfo);
|
|
|
+ if (rankItems == null || rankItems.isEmpty()) {
|
|
|
+ return new ArrayList<>();
|
|
|
+ }
|
|
|
+ List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
|
|
|
+ List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
|
|
|
+ Calendar calendar = Calendar.getInstance();
|
|
|
+ String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
|
|
|
+ String hour = new SimpleDateFormat("HH").format(calendar.getTime());
|
|
|
+ String rtFeaPart1h = date + hour;
|
|
|
+ if (rtFeaPartKeyResult != null) {
|
|
|
+ if (rtFeaPartKeyResult.get(1) != null) {
|
|
|
+ rtFeaPart1h = rtFeaPartKeyResult.get(1);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 2 统计分
|
|
|
+ String cur = rtFeaPart1h;
|
|
|
+ List<String> datehours = new LinkedList<>();
|
|
|
+ for (int i = 0; i < 24; ++i) {
|
|
|
+ datehours.add(cur);
|
|
|
+ cur = ExtractorUtils.subtractHours(cur, 1);
|
|
|
+ }
|
|
|
+ for (RankItem item : rankItems) {
|
|
|
+ Map<String, String> itemBasicMap = item.getItemBasicFeature();
|
|
|
+ Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
|
|
|
+ List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
|
|
|
+ List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
|
|
|
+ List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
|
|
|
+ List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
|
|
|
+ List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
|
|
|
+
|
|
|
+ List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
|
|
|
+ Double share2returnScore = calScoreWeight(share2return);
|
|
|
+ List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
|
|
|
+ Double view2returnScore = calScoreWeight(view2return);
|
|
|
+ List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
|
|
|
+ Double view2playScore = calScoreWeight(view2play);
|
|
|
+ List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
|
|
|
+ Double play2shareScore = calScoreWeight(play2share);
|
|
|
+ item.scoresMap.put("share2returnScore", share2returnScore);
|
|
|
+ item.scoresMap.put("view2returnScore", view2returnScore);
|
|
|
+ item.scoresMap.put("view2playScore", view2playScore);
|
|
|
+ item.scoresMap.put("play2shareScore", play2shareScore);
|
|
|
+
|
|
|
+ Double allreturnsScore = calScoreWeight(allreturns);
|
|
|
+ item.scoresMap.put("allreturnsScore", allreturnsScore);
|
|
|
+
|
|
|
+ // rov的趋势
|
|
|
+ double trendScore = calTrendScore(view2return);
|
|
|
+ item.scoresMap.put("trendScore", trendScore);
|
|
|
+
|
|
|
+ // 新视频提取
|
|
|
+ double newVideoScore = calNewVideoScore(itemBasicMap);
|
|
|
+ item.scoresMap.put("newVideoScore", newVideoScore);
|
|
|
+
|
|
|
+ }
|
|
|
+ // 3 融合公式
|
|
|
+ List<Video> result = new ArrayList<>();
|
|
|
+ double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
|
|
|
+ double beta = this.mergeWeight.getOrDefault("beta", 1.0);
|
|
|
+ for (RankItem item : rankItems) {
|
|
|
+ double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
|
|
|
+ item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
|
|
|
+ double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 0.0 ?
|
|
|
+ item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
|
|
|
+ double score = item.getScoreStr() *
|
|
|
+ item.scoresMap.getOrDefault("share2returnScore", 0.0)
|
|
|
+ + alpha * trendScore
|
|
|
+ + beta * newVideoScore;
|
|
|
+ Video video = new Video();
|
|
|
+ video.setVideoId(Long.parseLong(item.getId()));
|
|
|
+ video.setScore(score);
|
|
|
+ video.setSortScore(score);
|
|
|
+ video.setScoreStr(item.getScoreStr());
|
|
|
+ video.setScoresMap(item.getScoresMap());
|
|
|
+ result.add(video);
|
|
|
+ }
|
|
|
+ Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
|
|
|
+ List<String> datehours, String key){
|
|
|
+ List<Double> views = new LinkedList<>();
|
|
|
+ Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
|
|
|
+ for (String dh : datehours){
|
|
|
+ views.add(tmp.getOrDefault(dh, 0.0D) +
|
|
|
+ (views.isEmpty() ? 0.0: views.get(views.size()-1))
|
|
|
+ );
|
|
|
+ }
|
|
|
+ return views;
|
|
|
+ }
|
|
|
+
|
|
|
+ public double calNewVideoScore(Map<String, String> itemBasicMap){
|
|
|
+ double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
|
|
|
+ if (existenceDays > 8){
|
|
|
+ return 0.0;
|
|
|
+ }
|
|
|
+ double score = 1.0 / (existenceDays + 5.0);
|
|
|
+ return score;
|
|
|
+ }
|
|
|
+ public double calTrendScore(List<Double> data){
|
|
|
+ double sum = 0.0;
|
|
|
+ int size = data.size();
|
|
|
+ for (int i=0; i<size-4; ++i){
|
|
|
+ sum += data.get(i) - data.get(i+4);
|
|
|
+ }
|
|
|
+ if (sum * 10 > 0.6){
|
|
|
+ sum = 0.6;
|
|
|
+ }else{
|
|
|
+ sum = sum * 10;
|
|
|
+ }
|
|
|
+ if (sum > 0){
|
|
|
+ // 为了打断点
|
|
|
+ sum = sum;
|
|
|
+ }
|
|
|
+ return sum;
|
|
|
+ }
|
|
|
+
|
|
|
+ public Double calScoreWeight(List<Double> data){
|
|
|
+ Double up = 0.0;
|
|
|
+ Double down = 0.0;
|
|
|
+ for (int i=0; i<data.size(); ++i){
|
|
|
+ up += 1.0 / (i + 1) * data.get(i);
|
|
|
+ down += 1.0 / (i + 1);
|
|
|
+ }
|
|
|
+ return down > 1E-8? up / down: 0.0;
|
|
|
+ }
|
|
|
+ public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
|
|
|
+ List<Double> data = new LinkedList<>();
|
|
|
+ for(int i=0; i<ups.size(); ++i){
|
|
|
+ data.add(
|
|
|
+ (ups.get(i) + up) / (downs.get(i) + down)
|
|
|
+ );
|
|
|
+ }
|
|
|
+ return data;
|
|
|
+ }
|
|
|
+
|
|
|
public List<RankItem> feedByRec(final RecommendRequest requestData,
|
|
|
final int requestIndex,
|
|
|
final User userInfo) {
|
|
|
- int recallNum = 200;
|
|
|
+ int recallNum = 200;
|
|
|
|
|
|
// Step 1: Attention extraction
|
|
|
// long timestamp = System.currentTimeMillis();
|
|
|
@@ -86,7 +236,7 @@ public class TopRecommendPipeline {
|
|
|
// TODO 前置和后置处理逻辑 hardcode，后续优化
|
|
|
Map<String, String> sceneFeatureMap = getSceneFeature(requestData);
|
|
|
Map<String, String> userFeatureMap = getUserFeatureMap(requestData, items);
|
|
|
- List<RankItem> rovRecallRankNewallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
|
|
|
+ List<RankItem> rovRecallRankNewallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF_NEW_FEED)
|
|
|
.scoring(sceneFeatureMap, userFeatureMap, items);
|
|
|
|
|
|
return rovRecallRankNewallScore;
|
|
|
@@ -213,6 +363,9 @@ public class TopRecommendPipeline {
|
|
|
int j = 0;
|
|
|
for (RankItem item: rankItems){
|
|
|
++j;
|
|
|
+ if (j >= rankItems.size()) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
Map<String, String> vfMap = new HashMap<>();
|
|
|
Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
|
|
|
try {
|
|
|
@@ -244,6 +397,9 @@ public class TopRecommendPipeline {
|
|
|
}
|
|
|
for (RankItem item: rankItems){
|
|
|
++j;
|
|
|
+ if (j >= rankItems.size()) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
Map<String, String> vfMap = new HashMap<>();
|
|
|
Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
|
|
|
try {
|
sunxy