|
|
@@ -1,27 +1,14 @@
|
|
|
package com.tzld.piaoquan.recommend.server.service.rank.strategy;
|
|
|
|
|
|
import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
|
|
|
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
|
|
|
import com.tzld.piaoquan.recommend.server.model.Video;
|
|
|
-import com.tzld.piaoquan.recommend.server.service.FeatureService;
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
|
|
|
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
|
|
|
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
|
|
|
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
|
|
|
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
|
|
|
-import com.tzld.piaoquan.recommend.server.util.FeatureBucketUtils;
|
|
|
-import com.tzld.piaoquan.recommend.server.util.SimilarityUtils;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
-import org.apache.commons.collections4.MapUtils;
|
|
|
import org.apache.commons.math3.util.Pair;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
import java.util.*;
|
|
|
-import java.util.concurrent.Future;
|
|
|
-import java.util.concurrent.TimeUnit;
|
|
|
-import java.util.stream.Collectors;
|
|
|
|
|
|
@Service
|
|
|
@Slf4j
|
|
|
@@ -30,429 +17,80 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
private Map<String, Double> mergeWeight;
|
|
|
|
|
|
@Autowired
|
|
|
- private FeatureService featureService;
|
|
|
-
|
|
|
- private static final List<String> shortPeriod = Arrays.asList("1h", "2h", "4h", "6h", "12h", "24h", "7d");
|
|
|
- private static final List<String> middlePeriod = Arrays.asList("14d", "30d");
|
|
|
- private static final List<String> longPeriod = Arrays.asList("7d", "35d", "90d", "365d");
|
|
|
- private static final List<String> cfRosList = Collections.singletonList("rosn");
|
|
|
- private static final List<String> cfRovList = Collections.singletonList("rovn");
|
|
|
- private static final List<String> videoSimAttrs = Arrays.asList("cate1_list", "cate2", "cate2_list",
|
|
|
- "keywords", "style", "theme", "title", "topic", "user_value");
|
|
|
+ private RankStrategy4RegionMergeModelV563 modelV563Service;
|
|
|
|
|
|
@Override
|
|
|
public List<Video> mergeAndRankRovRecall(RankParam param) {
|
|
|
Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
|
|
|
- //-------------------融-------------------
|
|
|
- //-------------------合-------------------
|
|
|
- //-------------------逻-------------------
|
|
|
- //-------------------辑-------------------
|
|
|
-
|
|
|
- List<Video> oldRovs = new ArrayList<>();
|
|
|
- oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
|
|
|
- oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
|
|
|
- oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
|
|
|
- oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
|
|
|
- oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
|
|
|
- removeDuplicate(oldRovs);
|
|
|
- int sizeReturn = param.getSize();
|
|
|
- List<Video> v0 = oldRovs.size() <= sizeReturn
|
|
|
- ? oldRovs
|
|
|
- : oldRovs.subList(0, sizeReturn);
|
|
|
- Set<Long> setVideo = new HashSet<>();
|
|
|
- this.duplicate(setVideo, v0);
|
|
|
- setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
- List<Video> rovRecallRank = new ArrayList<>(v0);
|
|
|
- //-------------------return相似召回------------------
|
|
|
- List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
|
|
|
- v6 = v6.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
- v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
|
|
|
- rovRecallRank.addAll(v6);
|
|
|
- setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
- //-------------------新地域召回------------------
|
|
|
- List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
|
|
|
- v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
- v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
|
|
|
- rovRecallRank.addAll(v1);
|
|
|
- setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
-
|
|
|
- //-------------------排-------------------
|
|
|
- //-------------------序-------------------
|
|
|
- //-------------------逻-------------------
|
|
|
- //-------------------辑-------------------
|
|
|
-
|
|
|
- // 1 批量获取特征 省份参数要对齐 headvid 要传递过来！
|
|
|
- List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
|
|
|
-
|
|
|
- // k1:视频、k2:表、k3:特征、v:特征值
|
|
|
- String provinceCn = param.getProvince().replaceAll("省$", "");
|
|
|
- String headVid = String.valueOf(param.getHeadVid());
|
|
|
- String sceneType = String.valueOf(param.getHotSceneType());
|
|
|
- Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo(headVid, vids);
|
|
|
- FeatureService.Feature feature = featureService.getNewFeature(provinceCn, param.getMid(), sceneType, headVid, videoBaseInfoMap, vids);
|
|
|
- Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
|
|
|
- Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
|
|
|
- Map<String, String> headVideoInfo = videoBaseInfoMap.getOrDefault(headVid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
-
|
|
|
- // 2 特征处理
|
|
|
- Map<String, Double> userFeatureMapDouble = new HashMap<>();
|
|
|
- String mid = param.getMid();
|
|
|
- Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
|
|
|
- Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
|
|
|
- Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
|
|
|
- Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
|
|
|
- Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
|
|
|
- Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
|
|
|
- Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
|
|
|
- Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
|
|
|
- Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
|
|
|
-
|
|
|
- if (!c1.isEmpty()) {
|
|
|
- userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
|
|
|
- userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
|
|
|
- userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
|
|
|
- userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
|
|
|
- }
|
|
|
- if (!c2.isEmpty()) {
|
|
|
- userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
|
|
|
- userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
|
|
|
- userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
|
|
|
- userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
|
|
|
- userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
|
|
|
- userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
|
|
|
- userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
|
|
|
- userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
|
|
|
- }
|
|
|
-
|
|
|
- Map<String, String> c34567Map = new HashMap<>(15);
|
|
|
- List<Tuple2> tmpList0 = Arrays.asList(
|
|
|
- new Tuple2(c3, "c3_feature"),
|
|
|
- new Tuple2(c4, "c4_feature"),
|
|
|
- new Tuple2(c5, "c5_feature"),
|
|
|
- new Tuple2(c6, "c6_feature"),
|
|
|
- new Tuple2(c7, "c7_feature")
|
|
|
- );
|
|
|
- for (Tuple2 tuple2 : tmpList0) {
|
|
|
- for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
|
|
|
- String tags = tuple2.first.getOrDefault(key_time, "");
|
|
|
- if (!tags.isEmpty()) {
|
|
|
- c34567Map.put(tuple2.name + "_" + key_time, tags);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
|
|
|
- List<Tuple2> tmpList1 = Arrays.asList(
|
|
|
- new Tuple2(c8, "c8_feature"),
|
|
|
- new Tuple2(c9, "c9_feature")
|
|
|
- );
|
|
|
- for (Tuple2 tuple2 : tmpList1) {
|
|
|
- for (String key_action : Arrays.asList("share", "return")) {
|
|
|
- String cfListStr = tuple2.first.getOrDefault(key_action, "");
|
|
|
- if (!cfListStr.isEmpty()) {
|
|
|
- Map<String, String[]> cfMap = new HashMap<>();
|
|
|
- String[] entries = cfListStr.split(",");
|
|
|
- for (String entry : entries) {
|
|
|
- String[] rList = entry.split(":");
|
|
|
- if (rList.length >= 4) { // 确保分割后有四个元素
|
|
|
- String key = rList[0];
|
|
|
- String value1 = rList[1];
|
|
|
- String value2 = rList[2];
|
|
|
- String value3 = rList[3];
|
|
|
- String[] strs = {value1, value2, value3};
|
|
|
- cfMap.put(key, strs);
|
|
|
- }
|
|
|
- }
|
|
|
- c89Map.put(tuple2.name + "_" + key_action, cfMap);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
|
|
|
- for (RankItem item : rankItems) {
|
|
|
- Map<String, Double> featureMap = new HashMap<>();
|
|
|
- String vid = item.getVideoId() + "";
|
|
|
- Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
|
|
|
- Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
|
|
|
- Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
|
|
|
- Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
|
|
|
- Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
|
|
|
-
|
|
|
- Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
|
|
|
- Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
|
|
|
- Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
|
|
|
- Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
|
|
|
- Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
|
|
|
- Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
|
|
|
- Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
|
|
|
- Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
|
|
|
- Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
|
|
|
-
|
|
|
- List<Tuple4> originData = Arrays.asList(
|
|
|
- new Tuple4(b1, b2, b3, "b123"),
|
|
|
- new Tuple4(b1, b6, b7, "b167"),
|
|
|
- new Tuple4(b8, b9, b10, "b8910"),
|
|
|
- new Tuple4(b11, b12, b13, "b111213"),
|
|
|
- new Tuple4(b17, b18, b19, "b171819")
|
|
|
- );
|
|
|
-
|
|
|
- for (Tuple4 tuple4 : originData) {
|
|
|
- for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
|
|
|
- double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
|
|
|
- double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
|
|
|
- double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
|
|
|
-
|
|
|
- double f1 = ExtractorUtils.calDiv(share, exp);
|
|
|
- double f2 = ExtractorUtils.calLog(share);
|
|
|
- double f3 = ExtractorUtils.calDiv(returns, exp);
|
|
|
- double f4 = ExtractorUtils.calLog(returns);
|
|
|
- double f5 = f3 * f4;
|
|
|
- double f6 = ExtractorUtils.calDiv(returns, share);
|
|
|
-
|
|
|
- String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
|
|
|
- String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
|
|
|
- String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
|
|
|
- String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
|
|
|
- String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
|
|
|
- String key6 = tuple4.name + "_" + prefix2 + "_" + "ROS";
|
|
|
-
|
|
|
- featureMap.put(key1, f1);
|
|
|
- featureMap.put(key2, f2);
|
|
|
- featureMap.put(key3, f3);
|
|
|
- featureMap.put(key4, f4);
|
|
|
- featureMap.put(key5, f5);
|
|
|
- featureMap.put(key6, f6);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- Map<String, String> videoInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
- featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
|
|
|
- featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
|
|
|
-
|
|
|
- String title = videoInfo.getOrDefault("title", "");
|
|
|
- if (!title.isEmpty()) {
|
|
|
- List<Future<Pair<String, Double[]>>> futures = new ArrayList<>();
|
|
|
- for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
|
|
|
- for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
|
|
|
- String key = name + "_" + key_time;
|
|
|
- String tags = c34567Map.getOrDefault(key, "");
|
|
|
- if (!tags.isEmpty()) {
|
|
|
- Future<Pair<String, Double[]>> future = ThreadPoolFactory.defaultPool().submit(() -> {
|
|
|
- Double[] doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
|
|
|
- return Pair.create(key, doubles);
|
|
|
- });
|
|
|
- futures.add(future);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- try {
|
|
|
- for (Future<Pair<String, Double[]>> future : futures) {
|
|
|
- Pair<String, Double[]> pair = future.get(1000, TimeUnit.MILLISECONDS);
|
|
|
- featureMap.put(pair.getFirst() + "_matchnum", pair.getSecond()[0]);
|
|
|
- featureMap.put(pair.getFirst() + "_maxscore", pair.getSecond()[1]);
|
|
|
- featureMap.put(pair.getFirst() + "_avgscore", pair.getSecond()[2]);
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("concurrent similarity error", e);
|
|
|
- }
|
|
|
- }
|
|
|
|
|
|
- if (!vid.isEmpty()) {
|
|
|
- for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
|
|
|
- for (String key_action : Arrays.asList("share", "return")) {
|
|
|
- Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
|
|
|
- if (cfMap.containsKey(vid)) {
|
|
|
- String[] scores = cfMap.get(vid);
|
|
|
- Double score1 = Double.parseDouble(scores[0]);
|
|
|
- Double score2 = Double.parseDouble(scores[1]);
|
|
|
- Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
|
|
|
- featureMap.put(key_feature + "_" + key_action + "_score", score1);
|
|
|
- featureMap.put(key_feature + "_" + key_action + "_num", score2);
|
|
|
- featureMap.put(key_feature + "_" + key_action + "_rank", score3);
|
|
|
- }
|
|
|
- }
|
|
|
+ List<Video> result = modelV563Service.mergeAndRankRovRecall(param);
|
|
|
+ if (null != result && !result.isEmpty()) {
|
|
|
+ try {
|
|
|
+ Integer versionCode = param.getVersionCode();
|
|
|
+ int orderVersionCode = mergeWeight.getOrDefault("orderVersionCode", 1500D).intValue();
|
|
|
+ if (null != versionCode && versionCode == orderVersionCode) {
|
|
|
+ int keepTopN = mergeWeight.getOrDefault("keepTopN", 1D).intValue();
|
|
|
+ boolean firstRovFlag = mergeWeight.getOrDefault("firstRovFlag", 1D).intValue() > 0;
|
|
|
+ return resort(keepTopN, firstRovFlag, result);
|
|
|
}
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("diversity resort error", e);
|
|
|
}
|
|
|
- Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
|
|
|
- if (!d1.isEmpty()) {
|
|
|
- featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
|
|
|
- featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
|
|
|
- featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
|
|
|
- }
|
|
|
- // ******************** new feature ********************
|
|
|
- addVideoStatFeature(vid, featureOriginVideo, featureMap);
|
|
|
- //addVideoCFFeature(vid, featureOriginVideo, featureMap);
|
|
|
- addVideoSimFeature(headVideoInfo, videoInfo, featureMap);
|
|
|
-
|
|
|
- item.featureMapDouble = featureMap;
|
|
|
- }
|
|
|
-
|
|
|
- // 3 连续值特征分桶
|
|
|
- Map<String, String> userFeatureMap = FeatureBucketUtils.bucketFeature("20241209_rov_bucket.txt", userFeatureMapDouble);
|
|
|
- Map<String, String> norUserFeatureMap = FeatureBucketUtils.bucketFeature("20241209_nor_bucket.txt", userFeatureMapDouble);
|
|
|
- for (RankItem item : rankItems) {
|
|
|
- Map<String, Double> featureMapDouble = item.featureMapDouble;
|
|
|
- item.featureMap = FeatureBucketUtils.bucketFeature("20241209_rov_bucket.txt", featureMapDouble);
|
|
|
- item.norFeatureMap = FeatureBucketUtils.bucketFeature("20241209_nor_bucket.txt", featureMapDouble);
|
|
|
- }
|
|
|
- // 4 排序模型计算
|
|
|
- double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.02);
|
|
|
- double xgbNorValExp = mergeWeight.getOrDefault("xgbNorValExp", 1.1);
|
|
|
- Map<String, String> sceneFeatureMap = new HashMap<>(0);
|
|
|
- List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_xgb_20250109.conf").scoring(sceneFeatureMap, userFeatureMap, norUserFeatureMap, rankItems);
|
|
|
- // 5 排序公式特征
|
|
|
- Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
|
|
|
- List<Video> result = new ArrayList<>();
|
|
|
- for (RankItem item : items) {
|
|
|
- double score;
|
|
|
- double fmRovOrigin = item.getScoreRov();
|
|
|
- item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
|
|
|
- double fmRov = restoreScore(fmRovOrigin, xgbRovNegRate);
|
|
|
- item.getScoresMap().put("fmRov", fmRov);
|
|
|
- item.getScoresMap().put("xgbRovNegRate", xgbRovNegRate);
|
|
|
- item.getScoresMap().put("xgbNorValExp", xgbNorValExp);
|
|
|
- double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
|
|
|
- item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
|
|
|
- double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);
|
|
|
- double vor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
|
|
|
- item.getScoresMap().put("vor", vor);
|
|
|
- score = fmRov * (0.1 + Math.pow(norXGBScore, xgbNorValExp)) * (0.1 + vor);
|
|
|
- Video video = item.getVideo();
|
|
|
- video.setScore(score);
|
|
|
- video.setSortScore(score);
|
|
|
- video.setScoresMap(item.getScoresMap());
|
|
|
- video.setAllFeatureMap(item.getAllFeatureMap());
|
|
|
- if (MapUtils.isNotEmpty(feature.getVideoFeature()) && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
|
|
|
- video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
|
|
|
- }
|
|
|
- if (MapUtils.isNotEmpty(videoBaseInfoMap) && MapUtils.isNotEmpty(videoBaseInfoMap.get(item.getVideoId() + ""))) {
|
|
|
- video.getMetaFeatureMap().putAll(videoBaseInfoMap.get(item.getVideoId() + ""));
|
|
|
- }
|
|
|
- if (MapUtils.isNotEmpty(headVideoInfo)) {
|
|
|
- video.getMetaFeatureMap().put("head_video", headVideoInfo);
|
|
|
- }
|
|
|
- if (MapUtils.isNotEmpty(feature.getUserFeature())) {
|
|
|
- video.getMetaFeatureMap().putAll(feature.getUserFeature());
|
|
|
- }
|
|
|
- result.add(video);
|
|
|
}
|
|
|
- result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
|
|
|
return result;
|
|
|
}
|
|
|
|
|
|
- private Map<String, String> getVideoOneTypeInfo(String vid, String name,
|
|
|
- Map<String, Map<String, Map<String, String>>> videoAllInfoMap) {
|
|
|
- if (null == videoAllInfoMap) {
|
|
|
- return new HashMap<>();
|
|
|
+ private List<Video> resort(int keepTopN, boolean firstRovFlag, List<Video> videos) {
|
|
|
+ Map<Long, Video> videoMap = new HashMap<>();
|
|
|
+ List<Pair<Long, Double>> rovList = new ArrayList<>();
|
|
|
+ List<Pair<Long, Double>> rosList = new ArrayList<>();
|
|
|
+ for (Video video : videos) {
|
|
|
+ long vid = video.getVideoId();
|
|
|
+ videoMap.put(vid, video);
|
|
|
+
|
|
|
+ Map<String, Double> scoreMap = video.getScoresMap();
|
|
|
+ rovList.add(Pair.create(vid, scoreMap.getOrDefault("fmRov", 0d)));
|
|
|
+ rosList.add(Pair.create(vid, scoreMap.getOrDefault("hasReturnRovScore", 0d)));
|
|
|
}
|
|
|
- return videoAllInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault(name, new HashMap<>());
|
|
|
- }
|
|
|
+ rovList.sort(Comparator.comparingDouble(o -> -o.getSecond()));
|
|
|
+ rosList.sort(Comparator.comparingDouble(o -> -o.getSecond()));
|
|
|
|
|
|
- private double getVideoOneInfo(String name, Map<String, String> infoMap) {
|
|
|
- if (null == infoMap) {
|
|
|
- return 0.0;
|
|
|
- }
|
|
|
- return infoMap.isEmpty() ? 0 : Double.parseDouble(infoMap.getOrDefault(name, "0.0"));
|
|
|
- }
|
|
|
-
|
|
|
- private void addVideoStatFeature(String vid, Map<String, Map<String, Map<String, String>>> videoAllInfoMap,
|
|
|
- Map<String, Double> featureMap) {
|
|
|
- List<Tuple3> vidStatInfo = Arrays.asList(
|
|
|
- new Tuple3("b20", shortPeriod, getVideoOneTypeInfo(vid, "alg_cate2_feature", videoAllInfoMap)),
|
|
|
- new Tuple3("b21", shortPeriod, getVideoOneTypeInfo(vid, "alg_cate1_feature", videoAllInfoMap)),
|
|
|
- new Tuple3("b22", shortPeriod, getVideoOneTypeInfo(vid, "alg_vid_source_feature", videoAllInfoMap)),
|
|
|
- new Tuple3("b28", shortPeriod, getVideoOneTypeInfo(vid, "alg_sence_type_feature", videoAllInfoMap)),
|
|
|
- new Tuple3("b29", shortPeriod, getVideoOneTypeInfo(vid, "alg_videoid_feature", videoAllInfoMap)),
|
|
|
- new Tuple3("b23", middlePeriod, getVideoOneTypeInfo(vid, "alg_cate2_feature_day", videoAllInfoMap)),
|
|
|
- new Tuple3("b24", middlePeriod, getVideoOneTypeInfo(vid, "alg_cate1_feature_day", videoAllInfoMap)),
|
|
|
- new Tuple3("b25", middlePeriod, getVideoOneTypeInfo(vid, "alg_video_source_feature_day", videoAllInfoMap)),
|
|
|
- new Tuple3("b26", longPeriod, getVideoOneTypeInfo(vid, "alg_video_unionid_feature_day", videoAllInfoMap)),
|
|
|
- new Tuple3("b27", longPeriod, getVideoOneTypeInfo(vid, "alg_vid_feature_day", videoAllInfoMap))
|
|
|
- );
|
|
|
- for (Tuple3 tuple3 : vidStatInfo) {
|
|
|
- String infoType = tuple3.first;
|
|
|
- List<String> infoPeriod = tuple3.second;
|
|
|
- Map<String, String> infoMap = tuple3.third;
|
|
|
- for (String period : infoPeriod) {
|
|
|
- double share = getVideoOneInfo("share_" + period, infoMap);
|
|
|
- double return_ = getVideoOneInfo("return_" + period, infoMap);
|
|
|
- double view_hasreturn = getVideoOneInfo("view_hasreturn_" + period, infoMap);
|
|
|
- double share_hasreturn = getVideoOneInfo("share_hasreturn_" + period, infoMap);
|
|
|
- double ros = getVideoOneInfo("ros_" + period, infoMap);
|
|
|
- double rov = getVideoOneInfo("rov_" + period, infoMap);
|
|
|
- double r_cnt = getVideoOneInfo("r_cnt_" + period, infoMap);
|
|
|
- double r_rate = getVideoOneInfo("r_rate_" + period, infoMap);
|
|
|
- double r_cnt4s = getVideoOneInfo("r_cnt4s_" + period, infoMap);
|
|
|
- double str = getVideoOneInfo("str_" + period, infoMap);
|
|
|
-
|
|
|
- featureMap.put(infoType + "_" + period + "_" + "share", ExtractorUtils.calLog(share));
|
|
|
- featureMap.put(infoType + "_" + period + "_" + "return", ExtractorUtils.calLog(return_));
|
|
|
- featureMap.put(infoType + "_" + period + "_" + "view_hasreturn", ExtractorUtils.calLog(view_hasreturn));
|
|
|
- featureMap.put(infoType + "_" + period + "_" + "share_hasreturn", ExtractorUtils.calLog(share_hasreturn));
|
|
|
- featureMap.put(infoType + "_" + period + "_" + "ros", ros);
|
|
|
- featureMap.put(infoType + "_" + period + "_" + "rov", rov);
|
|
|
- featureMap.put(infoType + "_" + period + "_" + "r_cnt", r_cnt);
|
|
|
- featureMap.put(infoType + "_" + period + "_" + "r_rate", r_rate);
|
|
|
- featureMap.put(infoType + "_" + period + "_" + "r_cnt4s", r_cnt4s);
|
|
|
- featureMap.put(infoType + "_" + period + "_" + "str", str);
|
|
|
- }
|
|
|
+ // top
|
|
|
+ Set<Long> hit = new HashSet<>();
|
|
|
+ List<Video> result = new ArrayList<>();
|
|
|
+ for (int i = 0; i < keepTopN && i < videos.size(); i++) {
|
|
|
+ Video video = videos.get(i);
|
|
|
+ hit.add(video.getVideoId());
|
|
|
+ result.add(video);
|
|
|
}
|
|
|
- }
|
|
|
-
|
|
|
- private void addVideoCFFeature(String vid, Map<String, Map<String, Map<String, String>>> videoAllInfoMap,
|
|
|
- Map<String, Double> featureMap) {
|
|
|
- List<Tuple3> vidCFInfo = Arrays.asList(
|
|
|
- new Tuple3("d2", cfRosList, getVideoOneTypeInfo(vid, "alg_recsys_feature_weak_cf_i2i_scene_ros", videoAllInfoMap)),
|
|
|
- new Tuple3("d3", cfRosList, getVideoOneTypeInfo(vid, "alg_recsys_feature_cf_i2i_scene_ros", videoAllInfoMap)),
|
|
|
- new Tuple3("d4", cfRovList, getVideoOneTypeInfo(vid, "alg_recsys_feature_weak_cf_i2i_scene_rov", videoAllInfoMap)),
|
|
|
- new Tuple3("d5", cfRovList, getVideoOneTypeInfo(vid, "alg_recsys_feature_cf_i2i_scene_rov", videoAllInfoMap))
|
|
|
- );
|
|
|
- for (Tuple3 tuple3 : vidCFInfo) {
|
|
|
- String infoType = tuple3.first;
|
|
|
- List<String> valTypeList = tuple3.second;
|
|
|
- Map<String, String> infoMap = tuple3.third;
|
|
|
- if (!infoMap.isEmpty()) {
|
|
|
- for (String valType : valTypeList) {
|
|
|
- String expKey = "exp";
|
|
|
- if (valType.equals("rosn")) {
|
|
|
- expKey = "share";
|
|
|
- }
|
|
|
- double exp = getVideoOneInfo(expKey, infoMap);
|
|
|
- double return_n = getVideoOneInfo("return_n", infoMap);
|
|
|
- double value = getVideoOneInfo(valType, infoMap);
|
|
|
|
|
|
- featureMap.put(infoType + "_exp", ExtractorUtils.calLog(exp));
|
|
|
- featureMap.put(infoType + "_return_n", ExtractorUtils.calLog(return_n));
|
|
|
- featureMap.put(infoType + "_" + valType, value);
|
|
|
- }
|
|
|
+ // alternate
|
|
|
+ int rovIndex = 0;
|
|
|
+ int rosIndex = 0;
|
|
|
+ boolean flag = firstRovFlag;
|
|
|
+ for (int i = keepTopN; i < videos.size(); i++) {
|
|
|
+ if (flag) {
|
|
|
+ rovIndex = fillVideo(rovIndex, rovList, videoMap, hit, result);
|
|
|
+ } else {
|
|
|
+ rosIndex = fillVideo(rosIndex, rosList, videoMap, hit, result);
|
|
|
}
|
|
|
+ flag = !flag;
|
|
|
}
|
|
|
+ return result;
|
|
|
}
|
|
|
|
|
|
- private void addVideoSimFeature(Map<String, String> headInfo, Map<String, String> rankInfo, Map<String, Double> featureMap) {
|
|
|
- if (!headInfo.isEmpty() && !rankInfo.isEmpty()) {
|
|
|
- List<Future<Pair<String, Double>>> futures = new ArrayList<>();
|
|
|
- for (String attr : videoSimAttrs) {
|
|
|
- String headAttr = headInfo.getOrDefault(attr, "");
|
|
|
- String rankAttr = rankInfo.getOrDefault(attr, "");
|
|
|
- if (!"".equals(headAttr) && !"unknown".equals(headAttr) && !"".equals(rankAttr) && !"unknown".equals(rankAttr)) {
|
|
|
- String key = "video_sim_" + attr;
|
|
|
- Future<Pair<String, Double>> future = ThreadPoolFactory.defaultPool().submit(() -> {
|
|
|
- double simScore = SimilarityUtils.word2VecSimilarity(headAttr, rankAttr);
|
|
|
- return Pair.create(key, simScore);
|
|
|
- });
|
|
|
- futures.add(future);
|
|
|
- }
|
|
|
- }
|
|
|
- try {
|
|
|
- for (Future<Pair<String, Double>> future : futures) {
|
|
|
- Pair<String, Double> pair = future.get(1000, TimeUnit.MILLISECONDS);
|
|
|
- featureMap.put(pair.getFirst(), pair.getSecond());
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("video attr similarity error", e);
|
|
|
+ private int fillVideo(int start, List<Pair<Long, Double>> list, Map<Long, Video> videoMap, Set<Long> hit, List<Video> result) {
|
|
|
+ for (int i = start; i < list.size(); i++) {
|
|
|
+ start++;
|
|
|
+ Pair<Long, Double> pair = list.get(i);
|
|
|
+ long vid = pair.getFirst();
|
|
|
+ if (!hit.contains(vid)) {
|
|
|
+ Video video = videoMap.get(vid);
|
|
|
+ hit.add(vid);
|
|
|
+ result.add(video);
|
|
|
+ break;
|
|
|
}
|
|
|
}
|
|
|
+ return start;
|
|
|
}
|
|
|
}
|
