|
@@ -1,21 +1,30 @@
|
|
|
package com.tzld.piaoquan.recommend.server.service.rank.strategy;
|
|
|
|
|
|
+import com.alibaba.fastjson.JSON;
|
|
|
import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
|
|
|
import com.tzld.piaoquan.recommend.server.common.base.RankItem;
|
|
|
+import com.tzld.piaoquan.recommend.server.model.MachineInfo;
|
|
|
import com.tzld.piaoquan.recommend.server.model.Video;
|
|
|
import com.tzld.piaoquan.recommend.server.service.FeatureService;
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.bo.UserSRBO;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.bo.UserShareReturnProfile;
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractVideoMergeCate;
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.tansform.FeatureV6;
|
|
|
import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
|
|
|
import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
|
|
|
import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
|
|
|
+import com.tzld.piaoquan.recommend.server.util.FeatureBucketUtils;
|
|
|
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.collections4.CollectionUtils;
|
|
|
import org.apache.commons.collections4.MapUtils;
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
import org.apache.commons.math3.util.Pair;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.beans.factory.annotation.Value;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
import java.util.*;
|
|
@@ -29,18 +38,36 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
@ApolloJsonValue("${rank.score.merge.weightv562:}")
|
|
|
private Map<String, Double> mergeWeight;
|
|
|
|
|
|
+ @Value("#{'${rootsessionid.tails.562:}'.split(',')}")
|
|
|
+ private Set<String> rootSessionIdTails;
|
|
|
+
|
|
|
@Autowired
|
|
|
private FeatureService featureService;
|
|
|
|
|
|
+ @Autowired
|
|
|
+ private RankStrategy4RegionMergeModelV564 rank564;
|
|
|
+
|
|
|
@Override
|
|
|
public List<Video> mergeAndRankRovRecall(RankParam param) {
|
|
|
+
|
|
|
+ String rootSessionId = param.getRootSessionId();
|
|
|
+
|
|
|
+ if (StringUtils.isBlank(rootSessionId) || CollectionUtils.isEmpty(rootSessionIdTails)) {
|
|
|
+ return rank564.mergeAndRankRovRecall(param);
|
|
|
+ }
|
|
|
+
|
|
|
+ String tail = rootSessionId.substring(rootSessionId.length() - 1);
|
|
|
+ if (!rootSessionIdTails.contains(tail)) {
|
|
|
+ return rank564.mergeAndRankRovRecall(param);
|
|
|
+ }
|
|
|
Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
|
|
|
+
|
|
|
//-------------------融-------------------
|
|
|
//-------------------合-------------------
|
|
|
//-------------------逻-------------------
|
|
|
//-------------------辑-------------------
|
|
|
|
|
|
- //-------------------老地域召回------------------
|
|
|
+ long currentMs = System.currentTimeMillis();
|
|
|
List<Video> oldRovs = new ArrayList<>();
|
|
|
oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
|
|
|
oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
|
|
@@ -49,281 +76,93 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
|
|
|
removeDuplicate(oldRovs);
|
|
|
int sizeReturn = param.getSize();
|
|
|
- List<Video> v0 = oldRovs.size() <= sizeReturn ? oldRovs : oldRovs.subList(0, sizeReturn);
|
|
|
+ List<Video> v0 = oldRovs.size() <= sizeReturn
|
|
|
+ ? oldRovs
|
|
|
+ : oldRovs.subList(0, sizeReturn);
|
|
|
Set<Long> setVideo = new HashSet<>();
|
|
|
this.duplicate(setVideo, v0);
|
|
|
setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
- List<Video> recallVideos4Rank = new ArrayList<>(v0);
|
|
|
+ List<Video> rovRecallRank = new ArrayList<>(v0);
|
|
|
//-------------------return相似召回------------------
|
|
|
List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
|
|
|
v6 = v6.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
|
|
|
- recallVideos4Rank.addAll(v6);
|
|
|
+ rovRecallRank.addAll(v6);
|
|
|
setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
//-------------------新地域召回------------------
|
|
|
List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
|
|
|
v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
|
|
|
- recallVideos4Rank.addAll(v1);
|
|
|
+ rovRecallRank.addAll(v1);
|
|
|
setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
//-------------------scene cf rovn------------------
|
|
|
List<Video> sceneCFRovn = extractAndSort(param, SceneCFRovnRecallStrategy.PUSH_FORM);
|
|
|
sceneCFRovn = sceneCFRovn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
sceneCFRovn = sceneCFRovn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), sceneCFRovn.size()));
|
|
|
- recallVideos4Rank.addAll(sceneCFRovn);
|
|
|
+ rovRecallRank.addAll(sceneCFRovn);
|
|
|
setVideo.addAll(sceneCFRovn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
//-------------------scene cf rosn------------------
|
|
|
List<Video> sceneCFRosn = extractAndSort(param, SceneCFRosnRecallStrategy.PUSH_FORM);
|
|
|
sceneCFRosn = sceneCFRosn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
|
|
|
- recallVideos4Rank.addAll(sceneCFRosn);
|
|
|
+ rovRecallRank.addAll(sceneCFRosn);
|
|
|
setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
+ // -------------------cate1------------------
|
|
|
+ int cate1RecallN = mergeWeight.getOrDefault("cate1RecallN", 5.0).intValue();
|
|
|
+ addRecall(param, cate1RecallN, UserCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+ // -------------------cate2------------------
|
|
|
+ int cate2RecallN = mergeWeight.getOrDefault("cate2RecallN", 5.0).intValue();
|
|
|
+ addRecall(param, cate2RecallN, UserCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+ // -------------------head province cate1------------------
|
|
|
+ int headCate1RecallN = mergeWeight.getOrDefault("headCate1RecallN", 3.0).intValue();
|
|
|
+ addRecall(param, headCate1RecallN, HeadProvinceCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+ // -------------------head province cate2------------------
|
|
|
+ int headCate2RecallN = mergeWeight.getOrDefault("headCate2RecallN", 3.0).intValue();
|
|
|
+ addRecall(param, headCate2RecallN, HeadProvinceCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
|
|
|
//-------------------排-------------------
|
|
|
//-------------------序-------------------
|
|
|
//-------------------逻-------------------
|
|
|
//-------------------辑-------------------
|
|
|
|
|
|
- // 1 批量获取特征 省份参数要对齐 headvid 要传递过来!
|
|
|
- List<String> vids = CommonCollectionUtils.toListDistinct(recallVideos4Rank, v -> String.valueOf(v.getVideoId()));
|
|
|
-
|
|
|
+ // 1. 批量获取特征 省份参数要对齐 headvid 要传递过来!
|
|
|
// k1:视频、k2:表、k3:特征、v:特征值
|
|
|
- String provinceCn = param.getProvince().replaceAll("省$", "");
|
|
|
+ List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
|
|
|
String headVid = String.valueOf(param.getHeadVid());
|
|
|
- FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
|
|
|
- String.valueOf(param.getAppType()), provinceCn, headVid);
|
|
|
+ Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo(headVid, vids);
|
|
|
+ FeatureService.Feature feature = featureService.getFeatureV3(param, videoBaseInfoMap, vids);
|
|
|
Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
|
|
|
Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
|
|
|
+ Map<String, String> headVideoInfo = videoBaseInfoMap.getOrDefault(headVid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
|
|
|
+ // 2. 用户信息预处理
|
|
|
+ Map<String, Map<String, String[]>> newC7Map = FeatureV6.parseUCFScore(featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>()));
|
|
|
+ Map<String, Map<String, String[]>> newC8Map = FeatureV6.parseUCFScore(featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>()));
|
|
|
+ UserShareReturnProfile userProfile = parseUserProfile(featureOriginUser);
|
|
|
+ Map<String, Map<String, String>> userBehaviorVideoMap = getUserBehaviorVideoMap(userProfile);
|
|
|
|
|
|
- // 2 特征处理
|
|
|
- Map<String, Double> userFeatureMapDouble = new HashMap<>();
|
|
|
- // String mid = param.getMid();
|
|
|
- Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
|
|
|
- Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
|
|
|
- Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
|
|
|
- Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
|
|
|
- Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
|
|
|
- Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
|
|
|
- Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
|
|
|
- Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
|
|
|
- Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
|
|
|
-
|
|
|
- if (!c1.isEmpty()) {
|
|
|
- userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
|
|
|
- userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
|
|
|
- userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
|
|
|
- userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
|
|
|
- }
|
|
|
- if (!c2.isEmpty()) {
|
|
|
- userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
|
|
|
- userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
|
|
|
- userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
|
|
|
- userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
|
|
|
- userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
|
|
|
- userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
|
|
|
- userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
|
|
|
- userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
|
|
|
- }
|
|
|
-
|
|
|
- Map<String, String> c34567Map = new HashMap<>(15);
|
|
|
- List<Tuple2> tmpList0 = Arrays.asList(
|
|
|
- new Tuple2(c3, "c3_feature"),
|
|
|
- new Tuple2(c4, "c4_feature"),
|
|
|
- new Tuple2(c5, "c5_feature"),
|
|
|
- new Tuple2(c6, "c6_feature"),
|
|
|
- new Tuple2(c7, "c7_feature")
|
|
|
- );
|
|
|
- for (Tuple2 tuple2 : tmpList0) {
|
|
|
- for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
|
|
|
- String tags = tuple2.first.getOrDefault(key_time, "");
|
|
|
- if (!tags.isEmpty()) {
|
|
|
- c34567Map.put(tuple2.name + "_" + key_time, tags);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
|
|
|
- List<Tuple2> tmpList1 = Arrays.asList(
|
|
|
- new Tuple2(c8, "c8_feature"),
|
|
|
- new Tuple2(c9, "c9_feature")
|
|
|
- );
|
|
|
- for (Tuple2 tuple2 : tmpList1) {
|
|
|
- for (String key_action : Arrays.asList("share", "return")) {
|
|
|
- String cfListStr = tuple2.first.getOrDefault(key_action, "");
|
|
|
- if (!cfListStr.isEmpty()) {
|
|
|
- Map<String, String[]> cfMap = new HashMap<>();
|
|
|
- String[] entries = cfListStr.split(",");
|
|
|
- for (String entry : entries) {
|
|
|
- String[] rList = entry.split(":");
|
|
|
- if (rList.length >= 4) { // 确保分割后有四个元素
|
|
|
- String key = rList[0];
|
|
|
- String value1 = rList[1];
|
|
|
- String value2 = rList[2];
|
|
|
- String value3 = rList[3];
|
|
|
- String[] strs = {value1, value2, value3};
|
|
|
- cfMap.put(key, strs);
|
|
|
- }
|
|
|
- }
|
|
|
- c89Map.put(tuple2.name + "_" + key_action, cfMap);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- List<RankItem> rankItems = CommonCollectionUtils.toList(recallVideos4Rank, RankItem::new);
|
|
|
- for (RankItem item : rankItems) {
|
|
|
- Map<String, Double> featureMap = new HashMap<>();
|
|
|
- String vid = item.getVideoId() + "";
|
|
|
- Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
|
|
|
- Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
|
|
|
- Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
|
|
|
- Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
|
|
|
- Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
|
|
|
-
|
|
|
- Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
|
|
|
- Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
|
|
|
- Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
|
|
|
- Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
|
|
|
- Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
|
|
|
- Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
|
|
|
- Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
|
|
|
- Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
|
|
|
- Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
|
|
|
-
|
|
|
- List<Tuple4> originData = Arrays.asList(
|
|
|
- new Tuple4(b1, b2, b3, "b123"),
|
|
|
- new Tuple4(b1, b6, b7, "b167"),
|
|
|
- new Tuple4(b8, b9, b10, "b8910"),
|
|
|
- new Tuple4(b11, b12, b13, "b111213"),
|
|
|
- new Tuple4(b17, b18, b19, "b171819")
|
|
|
- );
|
|
|
-
|
|
|
- for (Tuple4 tuple4 : originData) {
|
|
|
- for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
|
|
|
- double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
|
|
|
- double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
|
|
|
- double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
|
|
|
-
|
|
|
- double f1 = ExtractorUtils.calDiv(share, exp);
|
|
|
- double f2 = ExtractorUtils.calLog(share);
|
|
|
- double f3 = ExtractorUtils.calDiv(returns, exp);
|
|
|
- double f4 = ExtractorUtils.calLog(returns);
|
|
|
- double f5 = f3 * f4;
|
|
|
-
|
|
|
- String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
|
|
|
- String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
|
|
|
- String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
|
|
|
- String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
|
|
|
- String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
|
|
|
-
|
|
|
- featureMap.put(key1, f1);
|
|
|
- featureMap.put(key2, f2);
|
|
|
- featureMap.put(key3, f3);
|
|
|
- featureMap.put(key4, f4);
|
|
|
- featureMap.put(key5, f5);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
- featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
|
|
|
- featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
|
|
|
-
|
|
|
- String title = videoInfo.getOrDefault("title", "");
|
|
|
- if (!title.isEmpty()) {
|
|
|
- List<Future<Pair<String, Double[]>>> futures = new ArrayList<>();
|
|
|
- for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
|
|
|
- for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
|
|
|
- String key = name + "_" + key_time;
|
|
|
- String tags = c34567Map.getOrDefault(key, "");
|
|
|
- if (!tags.isEmpty()) {
|
|
|
- Future<Pair<String, Double[]>> future = ThreadPoolFactory.defaultPool().submit(() -> {
|
|
|
- Double[] doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
|
|
|
- return Pair.create(key, doubles);
|
|
|
- });
|
|
|
- futures.add(future);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- try {
|
|
|
- for (Future<Pair<String, Double[]>> future : futures) {
|
|
|
- Pair<String, Double[]> pair = future.get(1000, TimeUnit.MILLISECONDS);
|
|
|
- featureMap.put(pair.getFirst() + "_matchnum", pair.getSecond()[0]);
|
|
|
- featureMap.put(pair.getFirst() + "_maxscore", pair.getSecond()[1]);
|
|
|
- featureMap.put(pair.getFirst() + "_avgscore", pair.getSecond()[2]);
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("concurrent similarity error", e);
|
|
|
- }
|
|
|
- }
|
|
|
+ // 3. 特征处理
|
|
|
+ List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
|
|
|
+ Map<String, String> userFeatureMap = getUserFeature(currentMs, param, headVideoInfo, userProfile, featureOriginUser);
|
|
|
+ batchGetVideoFeature(currentMs, userProfile, headVideoInfo, videoBaseInfoMap,
|
|
|
+ newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, rankItems);
|
|
|
|
|
|
- if (!vid.isEmpty()) {
|
|
|
- for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
|
|
|
- for (String key_action : Arrays.asList("share", "return")) {
|
|
|
- Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
|
|
|
- if (cfMap.containsKey(vid)) {
|
|
|
- String[] scores = cfMap.get(vid);
|
|
|
- Double score1 = Double.parseDouble(scores[0]);
|
|
|
- Double score2 = Double.parseDouble(scores[1]);
|
|
|
- Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
|
|
|
- featureMap.put(key_feature + "_" + key_action + "_score", score1);
|
|
|
- featureMap.put(key_feature + "_" + key_action + "_num", score2);
|
|
|
- featureMap.put(key_feature + "_" + key_action + "_rank", score3);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
|
|
|
- if (!d1.isEmpty()) {
|
|
|
- featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
|
|
|
- featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
|
|
|
- featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
|
|
|
- }
|
|
|
- item.featureMapDouble = featureMap;
|
|
|
- }
|
|
|
-
|
|
|
- // 3 连续值特征分桶
|
|
|
- readBucketFile();
|
|
|
- Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
|
|
|
- for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
|
|
|
- String name = entry.getKey();
|
|
|
- Double score = entry.getValue();
|
|
|
- // 注意:0值、不在分桶文件中的特征,会被过滤掉。
|
|
|
- if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
|
|
|
- Double bucketNum = this.bucketsLen.get(name);
|
|
|
- double[] buckets = this.bucketsMap.get(name);
|
|
|
- Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
|
|
|
- userFeatureMap.put(name, String.valueOf(scoreNew));
|
|
|
- }
|
|
|
- }
|
|
|
- for (RankItem item : rankItems) {
|
|
|
- Map<String, String> featureMap = new HashMap<>();
|
|
|
- Map<String, Double> featureMapDouble = item.featureMapDouble;
|
|
|
-
|
|
|
- for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
|
|
|
- String name = entry.getKey();
|
|
|
- Double score = entry.getValue();
|
|
|
- // 注意:0值、不在分桶文件中的特征,会被过滤掉。
|
|
|
- if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
|
|
|
- Double bucketNum = this.bucketsLen.get(name);
|
|
|
- double[] buckets = this.bucketsMap.get(name);
|
|
|
- Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
|
|
|
- featureMap.put(name, String.valueOf(scoreNew));
|
|
|
- }
|
|
|
- }
|
|
|
- item.featureMap = featureMap;
|
|
|
- }
|
|
|
- // 4 排序模型计算
|
|
|
+ // 4. 排序模型计算
|
|
|
Map<String, String> sceneFeatureMap = new HashMap<>(0);
|
|
|
- List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240807.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
|
|
|
- // 5 排序公式特征
|
|
|
- Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
|
|
|
- // Ros增强传播因子
|
|
|
- Map<String, Map<String, String>> rosSpreadDivMap = this.getVideoRedisFeature(vids, "vid_for_spread:");
|
|
|
+ List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250317.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
|
|
|
+
|
|
|
+ // 5. 排序公式特征
|
|
|
double alpha = mergeWeight.getOrDefault("alpha", 0.1);
|
|
|
double beta = mergeWeight.getOrDefault("beta", 0.0);
|
|
|
double gamma = mergeWeight.getOrDefault("gamma", 0.5);
|
|
|
- double vorMod = mergeWeight.getOrDefault("vorMod", 4.0);
|
|
|
+ double vorMod = mergeWeight.getOrDefault("vorMod", 4.0);
|
|
|
+ double xgbNorPowerWeight = mergeWeight.getOrDefault("xgbNorPowerWeight", 1.22);
|
|
|
+ double xgbNorPowerExp = mergeWeight.getOrDefault("xgbNorPowerExp", 1.15);
|
|
|
+
|
|
|
+ Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
|
|
|
+
|
|
|
+ Map<String, Map<String, String>> rosSpreadDivMap = this.getVideoRedisFeature(vids, "vid_for_spread:");
|
|
|
+
|
|
|
List<Video> result = new ArrayList<>();
|
|
|
for (RankItem item : items) {
|
|
|
double fmRovOrigin = item.getScoreRov();
|
|
@@ -343,9 +182,14 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
double spreadRate = Double.parseDouble(rosSpreadDivMap.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>(0)).getOrDefault("head_video_recommend_fission_rate", "0.01"));
|
|
|
item.getScoresMap().put("spreadRate", spreadRate);
|
|
|
|
|
|
+ double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);
|
|
|
+ double newNorXGBScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norXGBScore);
|
|
|
+ item.getScoresMap().put("norXGBScore", norXGBScore);
|
|
|
+ item.getScoresMap().put("newNorXGBScore", newNorXGBScore);
|
|
|
+
|
|
|
// 最终融合公式计算
|
|
|
double scoreRov = strTransfor * (alpha + rosTransfor);
|
|
|
- double scoreVov = scoreRov * (beta + vorTransfor);
|
|
|
+ double scoreVov = scoreRov * (beta + newNorXGBScore);
|
|
|
double score = scoreVov + gamma * spreadRate;
|
|
|
|
|
|
item.getScoresMap().put("scoreRov", scoreRov);
|
|
@@ -379,16 +223,179 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
return result;
|
|
|
}
|
|
|
|
|
|
- private String indexCoverKey(double index) {
|
|
|
- switch ((int) index) {
|
|
|
- case 1:
|
|
|
- return "head_video_rov1";
|
|
|
- case 3:
|
|
|
- return "head_video_recommend_rovn";
|
|
|
- case 4:
|
|
|
- return "head_video_recommend_fission_rate";
|
|
|
- default:
|
|
|
- return "recommend_123_depth_fission_rate";
|
|
|
+ private UserShareReturnProfile parseUserProfile(Map<String, Map<String, String>> userOriginInfo) {
|
|
|
+ if (null != userOriginInfo) {
|
|
|
+ Map<String, String> c9 = userOriginInfo.get("alg_recsys_feature_user_share_return_stat");
|
|
|
+ if (null != c9 && !c9.isEmpty()) {
|
|
|
+ String c9Str = JSONUtils.toJson(c9);
|
|
|
+ if (!c9Str.isEmpty()) {
|
|
|
+ try {
|
|
|
+ return JSON.parseObject(c9Str, UserShareReturnProfile.class);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("parseObject user profile error! value=[{}]", c9Str, e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ private Map<String, Map<String, String>> getUserBehaviorVideoMap(UserShareReturnProfile userProfile) {
|
|
|
+ Set<String> vidSet = new HashSet<>();
|
|
|
+ if (null != userProfile) {
|
|
|
+ for (List<UserSRBO> list : Arrays.asList(userProfile.getM_s_s(), userProfile.getM_r_s(), userProfile.getL_s_s(), userProfile.getL_r_s())) {
|
|
|
+ if (null != list) {
|
|
|
+ for (UserSRBO u : list) {
|
|
|
+ if (null != u) {
|
|
|
+ vidSet.add(u.getId() + "");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ Map<String, Map<String, String>> historyVideoMap = new HashMap<>();
|
|
|
+ if (!vidSet.isEmpty()) {
|
|
|
+ Map<String, Map<String, Map<String, String>>> videoMap = featureService.getVideoBaseInfo("", new ArrayList<>(vidSet));
|
|
|
+ if (null != videoMap && !videoMap.isEmpty()) {
|
|
|
+ for (Map.Entry<String, Map<String, Map<String, String>>> entry : videoMap.entrySet()) {
|
|
|
+ String vid = entry.getKey();
|
|
|
+ Map<String, Map<String, String>> map = entry.getValue();
|
|
|
+ if (null != map && map.containsKey("alg_vid_feature_basic_info")) {
|
|
|
+ historyVideoMap.put(vid, map.get("alg_vid_feature_basic_info"));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return historyVideoMap;
|
|
|
+ }
|
|
|
+
|
|
|
+ private Map<String, String> getUserFeature(long currentMs, RankParam param, Map<String, String> headInfo, UserShareReturnProfile userProfile, Map<String, Map<String, String>> userOriginInfo) {
|
|
|
+ Map<String, Double> featMap = new HashMap<>();
|
|
|
+ // context feature
|
|
|
+ String appType = String.valueOf(param.getAppType());
|
|
|
+ String hotSceneType = String.valueOf(param.getHotSceneType());
|
|
|
+ FeatureV6.getContextFeature(currentMs, appType, hotSceneType, featMap);
|
|
|
+
|
|
|
+ // head video feature
|
|
|
+ FeatureV6.getVideoBaseFeature("h", currentMs, headInfo, featMap);
|
|
|
+
|
|
|
+ // user feature
|
|
|
+ Map<String, String> baseInfo = getUserBaseInfo(param);
|
|
|
+ FeatureV6.getUserFeature(userOriginInfo, featMap);
|
|
|
+ FeatureV6.getUserProfileFeature(userProfile, baseInfo, featMap);
|
|
|
+
|
|
|
+ return FeatureBucketUtils.noBucketFeature(featMap);
|
|
|
+ }
|
|
|
+
|
|
|
+ private Map<String, String> getVideoFeature(long currentMs, String vid,
|
|
|
+ UserShareReturnProfile userProfile,
|
|
|
+ Map<String, String> headInfo, Map<String, String> rankInfo,
|
|
|
+ Map<String, Map<String, String[]>> c7Map,
|
|
|
+ Map<String, Map<String, String[]>> c8Map,
|
|
|
+ Map<String, Map<String, String>> userOriginInfo,
|
|
|
+ Map<String, Map<String, String>> historyVideoMap,
|
|
|
+ Map<String, Map<String, Map<String, String>>> videoOriginInfo) {
|
|
|
+ Map<String, Double> featMap = new HashMap<>();
|
|
|
+ // user & video feature
|
|
|
+ FeatureV6.getUserTagsCrossVideoFeature("c5", rankInfo, userOriginInfo.get("alg_mid_feature_return_tags"), featMap);
|
|
|
+ FeatureV6.getUserTagsCrossVideoFeature("c6", rankInfo, userOriginInfo.get("alg_mid_feature_share_tags"), featMap);
|
|
|
+ FeatureV6.getUserCFFeature("c7", vid, c7Map, featMap);
|
|
|
+ FeatureV6.getUserCFFeature("c8", vid, c8Map, featMap);
|
|
|
+
|
|
|
+ // rank video feature
|
|
|
+ FeatureV6.getVideoBaseFeature("r", currentMs, rankInfo, featMap);
|
|
|
+ FeatureV6.getVideoFeature(vid, videoOriginInfo, featMap);
|
|
|
+
|
|
|
+ // head&rank cross feature
|
|
|
+ FeatureV6.getHeadRankVideoCrossFeature(headInfo, rankInfo, featMap);
|
|
|
+
|
|
|
+ // user profile & rank cross
|
|
|
+ FeatureV6.getProfileVideoCrossFeature(currentMs, userProfile, rankInfo, historyVideoMap, featMap);
|
|
|
+
|
|
|
+ return FeatureBucketUtils.noBucketFeature(featMap);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void batchGetVideoFeature(long currentMs,
|
|
|
+ UserShareReturnProfile userProfile,
|
|
|
+ Map<String, String> headInfo,
|
|
|
+ Map<String, Map<String, Map<String, String>>> videoBaseInfoMap,
|
|
|
+ Map<String, Map<String, String[]>> c7Map,
|
|
|
+ Map<String, Map<String, String[]>> c8Map,
|
|
|
+ Map<String, Map<String, String>> userOriginInfo,
|
|
|
+ Map<String, Map<String, String>> historyVideoMap,
|
|
|
+ Map<String, Map<String, Map<String, String>>> videoOriginInfo,
|
|
|
+ List<RankItem> rankItems) {
|
|
|
+ if (null != rankItems && !rankItems.isEmpty()) {
|
|
|
+ List<Future<Integer>> futures = new ArrayList<>();
|
|
|
+ for (RankItem item : rankItems) {
|
|
|
+ String vid = item.getVideoId() + "";
|
|
|
+ Map<String, String> rankInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
+ Future<Integer> future = ThreadPoolFactory.defaultPool().submit(() -> {
|
|
|
+ item.featureMap = getVideoFeature(currentMs, vid, userProfile, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
|
|
|
+ item.norFeatureMap = item.featureMap;
|
|
|
+ return 1;
|
|
|
+ });
|
|
|
+ futures.add(future);
|
|
|
+ }
|
|
|
+
|
|
|
+ try {
|
|
|
+ for (Future<Integer> future : futures) {
|
|
|
+ future.get(1000, TimeUnit.MILLISECONDS);
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("get feature error", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private Map<String, String> getUserBaseInfo(RankParam param) {
|
|
|
+ Map<String, String> baseInfo = new HashMap<>();
|
|
|
+ String province = param.getProvince();
|
|
|
+ if (null != province && !province.isEmpty()) {
|
|
|
+ baseInfo.put("province", province.replaceAll("省$", ""));
|
|
|
+ }
|
|
|
+
|
|
|
+ String city = param.getCity();
|
|
|
+ if (null != city && !city.isEmpty()) {
|
|
|
+ baseInfo.put("city", city.replaceAll("市$", ""));
|
|
|
+ }
|
|
|
+
|
|
|
+ MachineInfo machineInfo = param.getMachineInfo();
|
|
|
+ if (null != machineInfo) {
|
|
|
+ String model = machineInfo.getModel();
|
|
|
+ if (null != model && !model.isEmpty()) {
|
|
|
+ baseInfo.put("model", model);
|
|
|
+ }
|
|
|
+ String brand = machineInfo.getBrand();
|
|
|
+ if (null != brand && !brand.isEmpty()) {
|
|
|
+ baseInfo.put("brand", brand);
|
|
|
+ }
|
|
|
+ String system = machineInfo.getSystem();
|
|
|
+ if (null != system && !system.isEmpty()) {
|
|
|
+ baseInfo.put("system", system);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return baseInfo;
|
|
|
+ }
|
|
|
+
|
|
|
+ private double norPowerCalibration(double weight, double exp, double score) {
|
|
|
+ double newScore = weight * Math.pow(score, exp);
|
|
|
+ if (newScore > 100) {
|
|
|
+ newScore = 100;
|
|
|
+ } else if (newScore < score) {
|
|
|
+ newScore = score;
|
|
|
+ }
|
|
|
+ return newScore;
|
|
|
+ }
|
|
|
+
|
|
|
+ private void addRecall(RankParam param, int recallNum, String recallName, Set<Long> setVideo, List<Video> rovRecallRank) {
|
|
|
+ if (recallNum > 0) {
|
|
|
+ List<Video> list = extractAndSort(param, recallName);
|
|
|
+ list = list.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
+ list = list.subList(0, Math.min(recallNum, list.size()));
|
|
|
+ rovRecallRank.addAll(list);
|
|
|
+ setVideo.addAll(list.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
}
|
|
|
}
|
|
|
|