|
|
@@ -0,0 +1,586 @@
|
|
|
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
|
|
|
+
|
|
|
+
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
+import com.google.common.reflect.TypeToken;
|
|
|
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
|
|
|
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
|
|
|
+import com.tzld.piaoquan.recommend.server.model.Video;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeatureV2;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeatureV2;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
|
|
|
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
|
|
|
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.collections4.CollectionUtils;
|
|
|
+import org.apache.commons.lang3.RandomUtils;
|
|
|
+import org.springframework.data.redis.connection.RedisConnectionFactory;
|
|
|
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
|
|
|
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
|
|
|
+import org.springframework.data.redis.core.RedisTemplate;
|
|
|
+import org.springframework.data.redis.serializer.StringRedisSerializer;
|
|
|
+import org.springframework.stereotype.Service;
|
|
|
+
|
|
|
+import java.text.SimpleDateFormat;
|
|
|
+import java.util.*;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+
|
|
|
+/**
|
|
|
+ * @author zhangbo
|
|
|
+ * @desc 地域召回融合
|
|
|
+ */
|
|
|
+@Service
|
|
|
+@Slf4j
|
|
|
+public class RankStrategy4RegionMergeModelV562 extends RankService {
|
|
|
+ @ApolloJsonValue("${rank.score.merge.weightv562:}")
|
|
|
+ private Map<String, Double> mergeWeight;
|
|
|
+ @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
|
|
|
+ private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
|
|
|
+ final private String CLASS_NAME = this.getClass().getSimpleName();
|
|
|
+
|
|
|
+ public void duplicate(Set<Long> setVideo, List<Video> videos){
|
|
|
+ Iterator<Video> iterator = videos.iterator();
|
|
|
+ while(iterator.hasNext()){
|
|
|
+ Video v = iterator.next();
|
|
|
+ if (setVideo.contains(v.getVideoId())){
|
|
|
+ iterator.remove();
|
|
|
+ }else{
|
|
|
+ setVideo.add(v.getVideoId());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ @Override
|
|
|
+ public List<Video> mergeAndRankRovRecall(RankParam param) {
|
|
|
+ Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
|
|
|
+ //-------------------融-------------------
|
|
|
+ //-------------------合-------------------
|
|
|
+ //-------------------逻-------------------
|
|
|
+ //-------------------辑-------------------
|
|
|
+
|
|
|
+ //-------------------地域相关召回 融合+去重-------------------
|
|
|
+ List<Video> rovRecallRank = new ArrayList<>();
|
|
|
+ List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
|
|
|
+ List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
|
|
|
+ List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
|
|
|
+ List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
|
|
|
+ Set<Long> setVideo = new HashSet<>();
|
|
|
+ this.duplicate(setVideo, v1);
|
|
|
+ this.duplicate(setVideo, v2);
|
|
|
+ this.duplicate(setVideo, v3);
|
|
|
+ this.duplicate(setVideo, v4);
|
|
|
+ //-------------------相关性召回 融合+去重-------------------
|
|
|
+ List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
|
|
|
+ List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
|
|
|
+ this.duplicate(setVideo, v5);
|
|
|
+ this.duplicate(setVideo, v6);
|
|
|
+ //-------------------节日扶持召回 融合+去重-------------------
|
|
|
+ List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
|
|
|
+ this.duplicate(setVideo, v7);
|
|
|
+
|
|
|
+ rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
|
|
|
+ rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
|
|
|
+ rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
|
|
|
+ rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
|
|
|
+ rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
|
|
|
+ rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
|
|
|
+ rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
|
|
|
+
|
|
|
+ //-------------------排-------------------
|
|
|
+ //-------------------序-------------------
|
|
|
+ //-------------------逻-------------------
|
|
|
+ //-------------------辑-------------------
|
|
|
+
|
|
|
+ // 1 模型分
|
|
|
+ List<RankItem> items = model(rovRecallRank, param);
|
|
|
+ List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
|
|
|
+ List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
|
|
|
+ Calendar calendar = Calendar.getInstance();
|
|
|
+ String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
|
|
|
+ String hour = new SimpleDateFormat("HH").format(calendar.getTime());
|
|
|
+ String rtFeaPart1h = date + hour;
|
|
|
+ if (rtFeaPartKeyResult != null){
|
|
|
+ if (rtFeaPartKeyResult.get(1) != null){
|
|
|
+ rtFeaPart1h = rtFeaPartKeyResult.get(1);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 2 统计分
|
|
|
+ String cur = rtFeaPart1h;
|
|
|
+ List<String> datehours = new LinkedList<>(); // 时间是倒叙的
|
|
|
+ for (int i=0; i<24; ++i){
|
|
|
+ datehours.add(cur);
|
|
|
+ cur = ExtractorUtils.subtractHours(cur, 1);
|
|
|
+ }
|
|
|
+ for (RankItem item : items){
|
|
|
+ Map<String, String> itemBasicMap = item.getItemBasicFeature();
|
|
|
+ Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
|
|
|
+ List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
|
|
|
+ List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
|
|
|
+ List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
|
|
|
+ List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
|
|
|
+ List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
|
|
|
+
|
|
|
+ List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
|
|
|
+ Double share2returnScore = calScoreWeight(share2return);
|
|
|
+ List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
|
|
|
+ Double view2returnScore = calScoreWeight(view2return);
|
|
|
+ List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
|
|
|
+ Double view2playScore = calScoreWeight(view2play);
|
|
|
+ List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
|
|
|
+ Double play2shareScore = calScoreWeight(play2share);
|
|
|
+ item.scoresMap.put("share2returnScore", share2returnScore);
|
|
|
+ item.scoresMap.put("view2returnScore", view2returnScore);
|
|
|
+ item.scoresMap.put("view2playScore", view2playScore);
|
|
|
+ item.scoresMap.put("play2shareScore", play2shareScore);
|
|
|
+
|
|
|
+ // 全部回流
|
|
|
+ Double allreturnsScore = calScoreWeight(allreturns);
|
|
|
+ item.scoresMap.put("allreturnsScore", allreturnsScore);
|
|
|
+
|
|
|
+ // 平台回流
|
|
|
+ Double preturnsScore = calScoreWeight(returns);
|
|
|
+ item.scoresMap.put("preturnsScore", preturnsScore);
|
|
|
+
|
|
|
+ // rov的趋势
|
|
|
+ double trendScore = calTrendScore(view2return);
|
|
|
+ item.scoresMap.put("trendScore", trendScore);
|
|
|
+
|
|
|
+ // 新视频提取
|
|
|
+ double newVideoScore = calNewVideoScore(itemBasicMap);
|
|
|
+ item.scoresMap.put("newVideoScore", newVideoScore);
|
|
|
+
|
|
|
+ }
|
|
|
+ // 3 融合公式
|
|
|
+ List<Video> result = new ArrayList<>();
|
|
|
+ double a = mergeWeight.getOrDefault("a", 1.0);
|
|
|
+ double b = mergeWeight.getOrDefault("b", 1.0);
|
|
|
+ double c = mergeWeight.getOrDefault("c", 0.0002);
|
|
|
+ double d = mergeWeight.getOrDefault("d", 1.0);
|
|
|
+ double e = mergeWeight.getOrDefault("e", 1.0);
|
|
|
+ double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
|
|
|
+ for (RankItem item : items){
|
|
|
+ double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
|
|
|
+ item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
|
|
|
+ double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
|
|
|
+ item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
|
|
|
+ double strScore = item.getScoreStr();
|
|
|
+ double rosScoreModel = item.getScoreRos();
|
|
|
+ double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
|
|
|
+ double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
|
|
|
+ double score = 0.0;
|
|
|
+ if (ifAdd < 0.5){
|
|
|
+ score = Math.pow(strScore, a) * Math.pow(rosScoreModel, b) + c * preturnsScore +
|
|
|
+ (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
|
|
|
+ }else {
|
|
|
+ score = a * strScore + b * rosScoreModel + c * preturnsScore +
|
|
|
+ (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
|
|
|
+ }
|
|
|
+ Video video = item.getVideo();
|
|
|
+ video.setScore(score);
|
|
|
+ video.setSortScore(score);
|
|
|
+ video.setScoreStr(item.getScoreStr());
|
|
|
+ video.setScoreRos(item.getScoreRos());
|
|
|
+ video.setScoresMap(item.getScoresMap());
|
|
|
+ result.add(video);
|
|
|
+ }
|
|
|
+ Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ public double calNewVideoScore(Map<String, String> itemBasicMap){
|
|
|
+ double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
|
|
|
+ if (existenceDays > 5){
|
|
|
+ return 0.0;
|
|
|
+ }
|
|
|
+ double score = 1.0 / (existenceDays + 10.0);
|
|
|
+ return score;
|
|
|
+ }
|
|
|
+ public double calTrendScore(List<Double> data){
|
|
|
+ double sum = 0.0;
|
|
|
+ int size = data.size();
|
|
|
+ for (int i=0; i<size-4; ++i){
|
|
|
+ sum += data.get(i) - data.get(i+4);
|
|
|
+ }
|
|
|
+ if (sum * 10 > 0.6){
|
|
|
+ sum = 0.6;
|
|
|
+ }else{
|
|
|
+ sum = sum * 10;
|
|
|
+ }
|
|
|
+ if (sum > 0){
|
|
|
+ // 为了打断点
|
|
|
+ sum = sum;
|
|
|
+ }
|
|
|
+ return sum;
|
|
|
+ }
|
|
|
+ public Double calScoreWeight(List<Double> data){
|
|
|
+ Double up = 0.0;
|
|
|
+ Double down = 0.0;
|
|
|
+ for (int i=0; i<data.size(); ++i){
|
|
|
+ up += 1.0 / (i + 1) * data.get(i);
|
|
|
+ down += 1.0 / (i + 1);
|
|
|
+ }
|
|
|
+ return down > 1E-8? up / down: 0.0;
|
|
|
+ }
|
|
|
+ public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
|
|
|
+ List<Double> data = new LinkedList<>();
|
|
|
+ for(int i=0; i<ups.size(); ++i){
|
|
|
+ data.add(
|
|
|
+ (ups.get(i) + up) / (downs.get(i) + down)
|
|
|
+ );
|
|
|
+ }
|
|
|
+ return data;
|
|
|
+ }
|
|
|
+ public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
|
|
|
+ List<String> datehours, String key){
|
|
|
+ List<Double> views = new LinkedList<>();
|
|
|
+ Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
|
|
|
+ for (String dh : datehours){
|
|
|
+ views.add(tmp.getOrDefault(dh, 0.0D) +
|
|
|
+ (views.isEmpty() ? 0.0: views.get(views.size()-1))
|
|
|
+ );
|
|
|
+ }
|
|
|
+ return views;
|
|
|
+ }
|
|
|
+ public List<RankItem> model(List<Video> videos, RankParam param){
|
|
|
+ List<RankItem> result = new ArrayList<>();
|
|
|
+ if (videos.isEmpty()){
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
|
|
|
+ redisSC.setPort(6379);
|
|
|
+ redisSC.setPassword("Wqsd@2019");
|
|
|
+ redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
|
|
|
+ RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
|
|
|
+ RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
|
|
|
+ redisTemplate.setConnectionFactory(connectionFactory);
|
|
|
+ redisTemplate.setDefaultSerializer(new StringRedisSerializer());
|
|
|
+ redisTemplate.afterPropertiesSet();
|
|
|
+
|
|
|
+ // 0: 场景特征处理
|
|
|
+ Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
|
|
|
+
|
|
|
+ // 1: user特征处理
|
|
|
+ Map<String, String> userFeatureMap = new HashMap<>();
|
|
|
+ if (param.getMid() != null && !param.getMid().isEmpty()){
|
|
|
+ String midKey = "user_info_4video_" + param.getMid();
|
|
|
+ String userFeatureStr = redisTemplate.opsForValue().get(midKey);
|
|
|
+ if (userFeatureStr != null){
|
|
|
+ try{
|
|
|
+ userFeatureMap = JSONUtils.fromJson(userFeatureStr,
|
|
|
+ new TypeToken<Map<String, String>>() {},
|
|
|
+ userFeatureMap);
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
|
|
|
+ }
|
|
|
+ }else{
|
|
|
+ JSONObject obj = new JSONObject();
|
|
|
+ obj.put("name", "user_key_in_model_is_null");
|
|
|
+ obj.put("class", this.CLASS_NAME);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
|
|
|
+ "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
|
|
|
+ "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
|
|
|
+ "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
|
|
|
+ "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
|
|
|
+ ));
|
|
|
+ Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
|
|
|
+ while (iterator.hasNext()) {
|
|
|
+ Map.Entry<String, String> entry = iterator.next();
|
|
|
+ if (!userFeatureSet.contains(entry.getKey())) {
|
|
|
+ iterator.remove();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ Map<String, String> f1 = RankExtractorUserFeatureV2.getOriginFeature(userFeatureMap,
|
|
|
+ new HashSet<String>(Arrays.asList(
|
|
|
+ "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
|
|
|
+ ))
|
|
|
+ );
|
|
|
+ Map<String, Double> f2__ = RankExtractorUserFeatureV2.getUserRateFeature(userFeatureMap);
|
|
|
+ Map<String, String> f2 = RankExtractorUserFeatureV2.rateFeatureChange(f2__);
|
|
|
+ Map<String, String> f3 = RankExtractorUserFeatureV2.cntFeatureChange(userFeatureMap,
|
|
|
+ new HashSet<String>(Arrays.asList(
|
|
|
+ "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
|
|
|
+ "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
|
|
|
+ "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
|
|
|
+ ))
|
|
|
+ );
|
|
|
+ f1.putAll(f2);
|
|
|
+ f1.putAll(f3);
|
|
|
+
|
|
|
+ // 2-1: item特征处理
|
|
|
+ final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
|
|
|
+ "total_time", "play_count_total",
|
|
|
+ "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
|
|
|
+ "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
|
|
|
+ "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"
|
|
|
+ ));
|
|
|
+
|
|
|
+ List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
|
|
|
+ List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
|
|
|
+ List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
|
|
|
+ if (videoFeatures != null){
|
|
|
+ for (int i=0; i<videoFeatures.size(); ++i){
|
|
|
+ String vF = videoFeatures.get(i);
|
|
|
+ Map<String, String> vfMap = new HashMap<>();
|
|
|
+ if (vF == null){
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ try{
|
|
|
+ vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
|
|
|
+ Map<String, String> vfMapCopy = new HashMap<>(vfMap);
|
|
|
+ rankItems.get(i).setItemBasicFeature(vfMapCopy);
|
|
|
+ Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
|
|
|
+ while (iteratorIn.hasNext()) {
|
|
|
+ Map.Entry<String, String> entry = iteratorIn.next();
|
|
|
+ if (!itemFeatureSet.contains(entry.getKey())) {
|
|
|
+ iteratorIn.remove();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ Map<String, Double> f4__ = RankExtractorItemFeatureV2.getItemRateFeature(vfMap);
|
|
|
+ Map<String, String> f4 = RankExtractorItemFeatureV2.rateFeatureChange(f4__);
|
|
|
+ Map<String, String> f5 = RankExtractorItemFeatureV2.cntFeatureChange(vfMap,
|
|
|
+ new HashSet<String>(Arrays.asList(
|
|
|
+ "total_time", "play_count_total",
|
|
|
+ "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
|
|
|
+ "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
|
|
|
+ "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"))
|
|
|
+ );
|
|
|
+ f4.putAll(f5);
|
|
|
+ rankItems.get(i).setFeatureMap(f4);
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 2-2: item 实时特征处理
|
|
|
+ List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
|
|
|
+ List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
|
|
|
+ Calendar calendar = Calendar.getInstance();
|
|
|
+ String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
|
|
|
+ String hour = new SimpleDateFormat("HH").format(calendar.getTime());
|
|
|
+ String rtFeaPart1day = date + hour;
|
|
|
+ String rtFeaPart1h = date + hour;
|
|
|
+ if (rtFeaPartKeyResult != null){
|
|
|
+ if (rtFeaPartKeyResult.get(0) != null){
|
|
|
+ rtFeaPart1day = rtFeaPartKeyResult.get(0);
|
|
|
+ }
|
|
|
+ if (rtFeaPartKeyResult.get(1) != null){
|
|
|
+ rtFeaPart1h = rtFeaPartKeyResult.get(1);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ videoRtKeys1.addAll(videoRtKeys2);
|
|
|
+ List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
|
|
|
+
|
|
|
+ if (videoRtFeatures != null){
|
|
|
+ int j = 0;
|
|
|
+ for (RankItem item: rankItems){
|
|
|
+ String vF = videoRtFeatures.get(j);
|
|
|
+ ++j;
|
|
|
+ if (vF == null){
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ Map<String, String> vfMap = new HashMap<>();
|
|
|
+ Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
|
|
|
+ try{
|
|
|
+ vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
|
|
|
+ for (Map.Entry<String, String> entry : vfMap.entrySet()){
|
|
|
+ String value = entry.getValue();
|
|
|
+ if (value == null){
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ String [] var1 = value.split(",");
|
|
|
+ Map<String, Double> tmp = new HashMap<>();
|
|
|
+ for (String var2 : var1){
|
|
|
+ String [] var3 = var2.split(":");
|
|
|
+ tmp.put(var3[0], Double.valueOf(var3[1]));
|
|
|
+ }
|
|
|
+ vfMapNew.put(entry.getKey(), tmp);
|
|
|
+ }
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
|
|
|
+ }
|
|
|
+ Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
|
|
|
+ Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
|
|
|
+ item.getFeatureMap().putAll(f8);
|
|
|
+ }
|
|
|
+ for (RankItem item: rankItems){
|
|
|
+ String vF = videoRtFeatures.get(j);
|
|
|
+ ++j;
|
|
|
+ if (vF == null){
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ Map<String, String> vfMap = new HashMap<>();
|
|
|
+ Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
|
|
|
+ try{
|
|
|
+ vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
|
|
|
+
|
|
|
+ for (Map.Entry<String, String> entry : vfMap.entrySet()){
|
|
|
+ String value = entry.getValue();
|
|
|
+ if (value == null){
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ String [] var1 = value.split(",");
|
|
|
+ Map<String, Double> tmp = new HashMap<>();
|
|
|
+ for (String var2 : var1){
|
|
|
+ String [] var3 = var2.split(":");
|
|
|
+ tmp.put(var3[0], Double.valueOf(var3[1]));
|
|
|
+ }
|
|
|
+ vfMapNew.put(entry.getKey(), tmp);
|
|
|
+ }
|
|
|
+ item.setItemRealTimeFeature(vfMapNew);
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
|
|
|
+ }
|
|
|
+ Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
|
|
|
+ Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
|
|
|
+ item.getFeatureMap().putAll(f8);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline("feeds_score_config_20240228.conf")
|
|
|
+ .scoring(sceneFeatureMap, userFeatureMap, rankItems);
|
|
|
+ JSONObject obj = new JSONObject();
|
|
|
+ obj.put("name", "user_key_in_model_is_not_null");
|
|
|
+ obj.put("class", this.CLASS_NAME);
|
|
|
+ return rovRecallScore;
|
|
|
+ }
|
|
|
+ private Map<String, String> getSceneFeature(RankParam param) {
|
|
|
+ Map<String, String> sceneFeatureMap = new HashMap<>();
|
|
|
+ String provinceCn = param.getProvince();
|
|
|
+ provinceCn = provinceCn.replaceAll("省$", "");
|
|
|
+ sceneFeatureMap.put("ctx_region", provinceCn);
|
|
|
+ String city = param.getCity();
|
|
|
+ if ("台北市".equals(city) |
|
|
|
+ "高雄市".equals(city) |
|
|
|
+ "台中市".equals(city) |
|
|
|
+ "桃园市".equals(city) |
|
|
|
+ "新北市".equals(city) |
|
|
|
+ "台南市".equals(city) |
|
|
|
+ "基隆市".equals(city) |
|
|
|
+ "吉林市".equals(city) |
|
|
|
+ "新竹市".equals(city) |
|
|
|
+ "嘉义市".equals(city)
|
|
|
+ ){
|
|
|
+ ;
|
|
|
+ }else{
|
|
|
+ city = city.replaceAll("市$", "");
|
|
|
+ }
|
|
|
+ sceneFeatureMap.put("ctx_city", city);
|
|
|
+
|
|
|
+ Calendar calendar = Calendar.getInstance();
|
|
|
+ sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
|
|
|
+ sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
|
|
|
+
|
|
|
+ return sceneFeatureMap;
|
|
|
+ }
|
|
|
+ @Override
|
|
|
+ public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
|
|
|
+
|
|
|
+ //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
|
|
|
+ if (CollectionUtils.isEmpty(rovVideos)) {
|
|
|
+ if (param.getSize() < flowVideos.size()) {
|
|
|
+ return new RankResult(flowVideos.subList(0, param.getSize()));
|
|
|
+ } else {
|
|
|
+ return new RankResult(flowVideos);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ //2 根据实验号解析阿波罗参数。
|
|
|
+ String abCode = param.getAbCode();
|
|
|
+ Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
|
|
|
+
|
|
|
+ //3 标签读取
|
|
|
+ if (rulesMap != null && !rulesMap.isEmpty()){
|
|
|
+ RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
|
|
|
+ extractorItemTags.processor(rovVideos, flowVideos);
|
|
|
+ }
|
|
|
+ //6 合并结果时间卡控
|
|
|
+ if (rulesMap != null && !rulesMap.isEmpty()){
|
|
|
+ RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
|
|
|
+ }
|
|
|
+
|
|
|
+ //4 rov池提权功能
|
|
|
+ RankProcessorBoost.boostByTag(rovVideos, rulesMap);
|
|
|
+
|
|
|
+ //5 rov池强插功能
|
|
|
+ RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
|
|
|
+
|
|
|
+ //7 流量池按比例强插
|
|
|
+ List<Video> result = new ArrayList<>();
|
|
|
+ for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
|
|
|
+ result.add(rovVideos.get(i));
|
|
|
+ }
|
|
|
+ double flowPoolP = getFlowPoolP(param);
|
|
|
+ int flowPoolIndex = 0;
|
|
|
+ int rovPoolIndex = param.getTopK();
|
|
|
+ for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
|
|
|
+ double rand = RandomUtils.nextDouble(0, 1);
|
|
|
+ log.info("rand={}, flowPoolP={}", rand, flowPoolP);
|
|
|
+ if (rand < flowPoolP) {
|
|
|
+ if (flowPoolIndex < flowVideos.size()) {
|
|
|
+ result.add(flowVideos.get(flowPoolIndex++));
|
|
|
+ } else {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ if (rovPoolIndex < rovVideos.size()) {
|
|
|
+ result.add(rovVideos.get(rovPoolIndex++));
|
|
|
+ } else {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (rovPoolIndex >= rovVideos.size()) {
|
|
|
+ for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
|
|
|
+ result.add(flowVideos.get(i));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (flowPoolIndex >= flowVideos.size()) {
|
|
|
+ for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
|
|
|
+ result.add(rovVideos.get(i));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ //8 合并结果密度控制
|
|
|
+ Map<String, Integer> densityRules = new HashMap<>();
|
|
|
+ if (rulesMap != null && !rulesMap.isEmpty()) {
|
|
|
+ for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
|
|
|
+ String key = entry.getKey();
|
|
|
+ Map<String, String> value = entry.getValue();
|
|
|
+ if (value.containsKey("density")) {
|
|
|
+ densityRules.put(key, Integer.valueOf(value.get("density")));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
|
|
|
+ List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
+ List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
+ List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
|
|
|
+ rovRecallRankNew, flowPoolRankNew, densityRules);
|
|
|
+
|
|
|
+ return new RankResult(resultWithDensity);
|
|
|
+ }
|
|
|
+
|
|
|
+}
|
