|
@@ -15,6 +15,7 @@ import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
|
|
|
import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
|
|
import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
|
|
|
import com.tzld.piaoquan.recommend.server.util.*;
|
|
import com.tzld.piaoquan.recommend.server.util.*;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
+import org.apache.commons.collections4.CollectionUtils;
|
|
|
import org.apache.commons.collections4.MapUtils;
|
|
import org.apache.commons.collections4.MapUtils;
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
@@ -23,6 +24,8 @@ import org.springframework.stereotype.Service;
|
|
|
import java.util.*;
|
|
import java.util.*;
|
|
|
import java.util.concurrent.Future;
|
|
import java.util.concurrent.Future;
|
|
|
import java.util.concurrent.TimeUnit;
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
|
+import java.util.stream.Collectors;
|
|
|
|
|
+import java.util.stream.Stream;
|
|
|
|
|
|
|
|
@Service
|
|
@Service
|
|
|
@Slf4j
|
|
@Slf4j
|
|
@@ -91,8 +94,22 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
// 1. 批量获取特征 省份参数要对齐 headvid 要传递过来!
|
|
// 1. 批量获取特征 省份参数要对齐 headvid 要传递过来!
|
|
|
// k1:视频、k2:表、k3:特征、v:特征值
|
|
// k1:视频、k2:表、k3:特征、v:特征值
|
|
|
Map<String, String> headVideoInfo = param.getHeadInfo();
|
|
Map<String, String> headVideoInfo = param.getHeadInfo();
|
|
|
|
|
+
|
|
|
|
|
+ // 用户的序列特征
|
|
|
|
|
+ Map<String, Map<String, String>> unionIdFeature = featureService.getUnionIdFeature(param.getUnionId());
|
|
|
|
|
+ Map<String, String> userNetworkSeqFeature = unionIdFeature.getOrDefault("alg_user_network_seq_feature", new HashMap<>());
|
|
|
|
|
+ List<String> actVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_v_s");
|
|
|
|
|
+ List<String> netVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "n_v_s");
|
|
|
|
|
+
|
|
|
List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
|
|
List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
|
|
|
- Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", vids);
|
|
|
|
|
|
|
+
|
|
|
|
|
+ List<String> allVids = Stream.of(actVidSeq, netVidSeq, vids)
|
|
|
|
|
+ .flatMap(Collection::stream)
|
|
|
|
|
+ .distinct()
|
|
|
|
|
+ .filter(StringUtils::isNotBlank)
|
|
|
|
|
+ .collect(Collectors.toList());
|
|
|
|
|
+
|
|
|
|
|
+ Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", allVids);
|
|
|
Map<String, Map<String, Map<String, String>>> videoBCData = featureService.getVideoStatistics(vids);
|
|
Map<String, Map<String, Map<String, String>>> videoBCData = featureService.getVideoStatistics(vids);
|
|
|
|
|
|
|
|
FeatureService.Feature feature = featureService.getFeatureV4(param, headVideoInfo, videoBaseInfoMap, vids);
|
|
FeatureService.Feature feature = featureService.getFeatureV4(param, headVideoInfo, videoBaseInfoMap, vids);
|
|
@@ -106,15 +123,20 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
Map<String, Map<String, String>> userBehaviorVideoMap = param.getBehaviorVideos();
|
|
Map<String, Map<String, String>> userBehaviorVideoMap = param.getBehaviorVideos();
|
|
|
Map<String, String> creativeInfo = param.getCreativeInfoFeature();
|
|
Map<String, String> creativeInfo = param.getCreativeInfoFeature();
|
|
|
|
|
|
|
|
|
|
+ Map<String, String> featureMapToString = new HashMap<>();
|
|
|
|
|
+ FeatureV6.parseStringFeatureMap(featureMapToString, param);
|
|
|
|
|
+ FeatureV6.putVideoStringFeatures("h", headVideoInfo, featureMapToString);
|
|
|
|
|
+
|
|
|
// 3. 特征处理
|
|
// 3. 特征处理
|
|
|
List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
|
|
List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
|
|
|
Map<String, Float> userFeatureMap = getUserFeature(currentMs, param, creativeInfo, headVideoInfo, userProfile, featureOriginUser);
|
|
Map<String, Float> userFeatureMap = getUserFeature(currentMs, param, creativeInfo, headVideoInfo, userProfile, featureOriginUser);
|
|
|
batchGetVideoFeature(currentMs, userProfile, creativeInfo, headVideoInfo, videoBaseInfoMap,
|
|
batchGetVideoFeature(currentMs, userProfile, creativeInfo, headVideoInfo, videoBaseInfoMap,
|
|
|
- newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, rankItems);
|
|
|
|
|
|
|
+ newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, featureMapToString, userFeatureMap, rankItems);
|
|
|
|
|
+
|
|
|
|
|
|
|
|
// 4. 排序模型计算
|
|
// 4. 排序模型计算
|
|
|
Map<String, Float> sceneFeatureMap = new HashMap<>(0);
|
|
Map<String, Float> sceneFeatureMap = new HashMap<>(0);
|
|
|
- List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250729.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
|
|
|
|
|
|
|
+ List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_str_ros_dnn_20260424.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
|
|
|
|
|
|
|
|
// 5. 排序公式特征
|
|
// 5. 排序公式特征
|
|
|
double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.059);
|
|
double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.059);
|
|
@@ -127,6 +149,9 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
double vorAdd = mergeWeight.getOrDefault("vor_add", 0.1d);
|
|
double vorAdd = mergeWeight.getOrDefault("vor_add", 0.1d);
|
|
|
double vorW = mergeWeight.getOrDefault("vor_w", 1.0d);
|
|
double vorW = mergeWeight.getOrDefault("vor_w", 1.0d);
|
|
|
|
|
|
|
|
|
|
+ double leaveW = mergeWeight.getOrDefault("leave_w", 1d);
|
|
|
|
|
+ double leaveExp = mergeWeight.getOrDefault("leave_exp", 1d);
|
|
|
|
|
+
|
|
|
double c1Rovn1hW = mergeWeight.getOrDefault("c1_rovn_1h_w", 0d);
|
|
double c1Rovn1hW = mergeWeight.getOrDefault("c1_rovn_1h_w", 0d);
|
|
|
double c1Rovn24hW = mergeWeight.getOrDefault("c1_rovn_24h_w", 0d);
|
|
double c1Rovn24hW = mergeWeight.getOrDefault("c1_rovn_24h_w", 0d);
|
|
|
|
|
|
|
@@ -144,7 +169,7 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
|
|
|
|
|
Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
|
|
Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
|
|
|
|
|
|
|
|
- // 获取权重
|
|
|
|
|
|
|
+
|
|
|
Map<String, String> contextInfo = getContextInfo(param);
|
|
Map<String, String> contextInfo = getContextInfo(param);
|
|
|
|
|
|
|
|
List<Video> result = new ArrayList<>();
|
|
List<Video> result = new ArrayList<>();
|
|
@@ -158,8 +183,9 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
|
|
double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
|
|
|
item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
|
|
item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
|
|
|
|
|
|
|
|
- double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);
|
|
|
|
|
- double newNorXGBScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norXGBScore);
|
|
|
|
|
|
|
+ double norDNNScore = item.getScoresMap().getOrDefault("NorDNNScore", 0d);
|
|
|
|
|
+ double newNorDNNScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norDNNScore);
|
|
|
|
|
+ item.getScoresMap().put("newNorDNNScore", newNorDNNScore);
|
|
|
item.getScoresMap().put("rosAdd", rosAdd);
|
|
item.getScoresMap().put("rosAdd", rosAdd);
|
|
|
item.getScoresMap().put("rosW", rosW);
|
|
item.getScoresMap().put("rosW", rosW);
|
|
|
|
|
|
|
@@ -168,6 +194,12 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
item.getScoresMap().put("vorAdd", vorAdd);
|
|
item.getScoresMap().put("vorAdd", vorAdd);
|
|
|
item.getScoresMap().put("vorW", vorW);
|
|
item.getScoresMap().put("vorW", vorW);
|
|
|
|
|
|
|
|
|
|
+ double pLeave = item.getScoresMap().getOrDefault("pLeave", 0d);
|
|
|
|
|
+ double newPLeave = Math.pow((1 - leaveW * pLeave), leaveExp);
|
|
|
|
|
+ item.getScoresMap().put("leaveW", leaveW);
|
|
|
|
|
+ item.getScoresMap().put("leaveExp", leaveExp);
|
|
|
|
|
+ item.getScoresMap().put("newPLeave", newPLeave);
|
|
|
|
|
+
|
|
|
Map<String, String> bcData = videoBCData.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>()).getOrDefault("alg_vid_feature_b_c_data", new HashMap<>());
|
|
Map<String, String> bcData = videoBCData.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>()).getOrDefault("alg_vid_feature_b_c_data", new HashMap<>());
|
|
|
Map<String, String> cdNData = videoBCData.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>()).getOrDefault("alg_vid_feature_cn_dn_data", new HashMap<>());
|
|
Map<String, String> cdNData = videoBCData.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>()).getOrDefault("alg_vid_feature_cn_dn_data", new HashMap<>());
|
|
|
|
|
|
|
@@ -217,7 +249,7 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
item.getScoresMap().put("dnRovn24hW", dnRovn24hW);
|
|
item.getScoresMap().put("dnRovn24hW", dnRovn24hW);
|
|
|
item.getScoresMap().put("dnRovn24h", dnRovn24h);
|
|
item.getScoresMap().put("dnRovn24h", dnRovn24h);
|
|
|
|
|
|
|
|
- score = fmRov * (rosAdd + rosW * newNorXGBScore) * (vorAdd + vorW * vor) + c1RovnScore + b0StrScore + b0RorScore + cnRovnScore + dnRovnScore;
|
|
|
|
|
|
|
+ score = fmRov * (rosAdd + rosW * newNorDNNScore) * (vorAdd + vorW * vor) * newPLeave + c1RovnScore + b0StrScore + b0RorScore + cnRovnScore + dnRovnScore;
|
|
|
|
|
|
|
|
Video video = item.getVideo();
|
|
Video video = item.getVideo();
|
|
|
video.setScore(score);
|
|
video.setScore(score);
|
|
@@ -251,7 +283,7 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
if (MapUtils.isNotEmpty(contextInfo)) {
|
|
if (MapUtils.isNotEmpty(contextInfo)) {
|
|
|
video.getMetaFeatureMap().put("context", contextInfo);
|
|
video.getMetaFeatureMap().put("context", contextInfo);
|
|
|
}
|
|
}
|
|
|
- if (Objects.nonNull(video.getRankVideoInfoMap()) && video.getRankVideoInfoMap().containsKey(video.getVideoId())){
|
|
|
|
|
|
|
+ if (Objects.nonNull(video.getRankVideoInfoMap()) && video.getRankVideoInfoMap().containsKey(video.getVideoId())) {
|
|
|
video.getRankVideoInfoMap().get(video.getVideoId()).setScore(score);
|
|
video.getRankVideoInfoMap().get(video.getVideoId()).setScore(score);
|
|
|
video.getRankVideoInfoMap().get(video.getVideoId()).setScoresMap(video.getScoresMap());
|
|
video.getRankVideoInfoMap().get(video.getVideoId()).setScoresMap(video.getScoresMap());
|
|
|
}
|
|
}
|
|
@@ -338,15 +370,28 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
Map<String, Map<String, String>> userOriginInfo,
|
|
Map<String, Map<String, String>> userOriginInfo,
|
|
|
Map<String, Map<String, String>> historyVideoMap,
|
|
Map<String, Map<String, String>> historyVideoMap,
|
|
|
Map<String, Map<String, Map<String, String>>> videoOriginInfo,
|
|
Map<String, Map<String, Map<String, String>>> videoOriginInfo,
|
|
|
|
|
+ Map<String, String> featureMapToString,
|
|
|
|
|
+ Map<String, Float> userFeatureMap,
|
|
|
List<RankItem> rankItems) {
|
|
List<RankItem> rankItems) {
|
|
|
- if (null != rankItems && !rankItems.isEmpty()) {
|
|
|
|
|
|
|
+ if (CollectionUtils.isNotEmpty(rankItems)) {
|
|
|
List<Future<Integer>> futures = new ArrayList<>();
|
|
List<Future<Integer>> futures = new ArrayList<>();
|
|
|
for (RankItem item : rankItems) {
|
|
for (RankItem item : rankItems) {
|
|
|
String vid = item.getVideoId() + "";
|
|
String vid = item.getVideoId() + "";
|
|
|
Map<String, String> rankInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
Map<String, String> rankInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
Future<Integer> future = ThreadPoolFactory.defaultPool().submit(() -> {
|
|
Future<Integer> future = ThreadPoolFactory.defaultPool().submit(() -> {
|
|
|
- item.featureMap = getVideoFeature(currentMs, vid, userProfile, creativeInfo, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
|
|
|
|
|
- item.norFeatureMap = item.featureMap;
|
|
|
|
|
|
|
+ Map<String, Float> featureMap = new HashMap<>(userFeatureMap);
|
|
|
|
|
+ Map<String, Float> videoFeature = getVideoFeature(currentMs, vid, userProfile, creativeInfo, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
|
|
|
|
|
+ featureMap.putAll(videoFeature);
|
|
|
|
|
+ item.featureMap = featureMap;
|
|
|
|
|
+
|
|
|
|
|
+ Map<String, String> userNetworkSeqFeature = userOriginInfo.getOrDefault("alg_user_network_seq_feature", new HashMap<>());
|
|
|
|
|
+
|
|
|
|
|
+ Map<String, String> featureMapString = new HashMap<>(featureMapToString);
|
|
|
|
|
+ FeatureV6.putVideoStringFeatures("r", rankInfo, featureMapString);
|
|
|
|
|
+ featureMapString.put("r@vid", "r_vid_" + vid);
|
|
|
|
|
+ FeatureV6.putProfileVideoCrossStringFeature(currentMs, userProfile, historyVideoMap, featureMapString);
|
|
|
|
|
+ FeatureV6.putUserNetworkSeqFeature(featureMapString, userNetworkSeqFeature, videoBaseInfoMap);
|
|
|
|
|
+ item.featureMapString = featureMapString;
|
|
|
return 1;
|
|
return 1;
|
|
|
});
|
|
});
|
|
|
futures.add(future);
|
|
futures.add(future);
|
|
@@ -418,48 +463,4 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
|
|
|
}
|
|
}
|
|
|
return newScore;
|
|
return newScore;
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
- private Map<String, Double> findSimCateScore(String headCate2, int length) {
|
|
|
|
|
- if (StringUtils.isBlank(headCate2)) {
|
|
|
|
|
- return new HashMap<>();
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- String redisKey = String.format("alg_recsys_good_cate_pair_list:%s", headCate2);
|
|
|
|
|
- String cate2Value = redisTemplate.opsForValue().get(redisKey);
|
|
|
|
|
- if (StringUtils.isEmpty(cate2Value)) {
|
|
|
|
|
- return new HashMap<>();
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- return this.parsePair(cate2Value, length);
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- private Map<String, Double> parsePair(String value, int length) {
|
|
|
|
|
- if (StringUtils.isBlank(value)) {
|
|
|
|
|
- return new HashMap<>();
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- String[] split = value.split("\t");
|
|
|
|
|
- if (split.length != 2) {
|
|
|
|
|
- return new HashMap<>();
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- String[] valueList = split[0].trim().split(",");
|
|
|
|
|
- String[] scoreList = split[1].trim().split(",");
|
|
|
|
|
- if (valueList.length != scoreList.length) {
|
|
|
|
|
- return new HashMap<>();
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- int minLength = Math.min(length, valueList.length);
|
|
|
|
|
- Map<String, Double> resultMap = new HashMap<>();
|
|
|
|
|
- for (int i = 0; i < minLength; i++) {
|
|
|
|
|
- resultMap.put(valueList[i].trim(), Double.parseDouble(scoreList[i].trim()));
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- return resultMap;
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- private String findVideoMergeCate2(Map<String, Map<String, Map<String, String>>> featureOriginVideo, String vid) {
|
|
|
|
|
- Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
|
|
- return videoInfo.get("merge_second_level_cate");
|
|
|
|
|
- }
|
|
|
|
|
}
|
|
}
|