|
|
@@ -14,9 +14,7 @@ import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractVideoMer
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.tansform.FeatureV6;
|
|
|
import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
|
|
|
import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
|
|
|
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
|
|
|
-import com.tzld.piaoquan.recommend.server.util.FeatureBucketUtils;
|
|
|
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
|
|
|
+import com.tzld.piaoquan.recommend.server.util.*;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.apache.commons.collections4.CollectionUtils;
|
|
|
import org.apache.commons.collections4.MapUtils;
|
|
|
@@ -36,28 +34,11 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
@ApolloJsonValue("${rank.score.merge.weightv562:}")
|
|
|
private Map<String, Double> mergeWeight;
|
|
|
|
|
|
- @Value("#{'${rootsessionid.tails.562:}'.split(',')}")
|
|
|
- private Set<String> rootSessionIdTails;
|
|
|
-
|
|
|
@Autowired
|
|
|
private FeatureService featureService;
|
|
|
|
|
|
- @Autowired
|
|
|
- private RankStrategy4RegionMergeModelV567 modelV567;
|
|
|
-
|
|
|
@Override
|
|
|
public List<Video> mergeAndRankRovRecall(RankParam param) {
|
|
|
-
|
|
|
- String rootSessionId = param.getRootSessionId();
|
|
|
-
|
|
|
- if (StringUtils.isBlank(rootSessionId) || CollectionUtils.isEmpty(rootSessionIdTails)) {
|
|
|
- return modelV567.mergeAndRankRovRecall(param);
|
|
|
- }
|
|
|
-
|
|
|
- String tail = rootSessionId.substring(rootSessionId.length() - 1);
|
|
|
- if (!rootSessionIdTails.contains(tail)) {
|
|
|
- return modelV567.mergeAndRankRovRecall(param);
|
|
|
- }
|
|
|
Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
|
|
|
|
|
|
//-------------------融-------------------
|
|
|
@@ -66,145 +47,113 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
//-------------------辑-------------------
|
|
|
|
|
|
long currentMs = System.currentTimeMillis();
|
|
|
- List<Video> oldRovs = new ArrayList<>();
|
|
|
- oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
|
|
|
- oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
|
|
|
- oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
|
|
|
- oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
|
|
|
- oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
|
|
|
- removeDuplicate(oldRovs);
|
|
|
- int sizeReturn = param.getSize();
|
|
|
- List<Video> v0 = oldRovs.size() <= sizeReturn
|
|
|
- ? oldRovs
|
|
|
- : oldRovs.subList(0, sizeReturn);
|
|
|
Set<Long> setVideo = new HashSet<>();
|
|
|
- this.duplicate(setVideo, v0);
|
|
|
- setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
- List<Video> rovRecallRank = new ArrayList<>(v0);
|
|
|
+ setVideo.add(param.getHeadVid());
|
|
|
+ List<Video> rovRecallRank = new ArrayList<>();
|
|
|
+ // -------------------5路特殊旧召回------------------
|
|
|
+ RecallUtils.extractOldSpecialRecall(mergeWeight.getOrDefault("oldSpecialN", (double) param.getSize()).intValue(), param, setVideo, rovRecallRank);
|
|
|
//-------------------return相似召回------------------
|
|
|
- List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
|
|
|
- v6 = v6.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
- v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
|
|
|
- rovRecallRank.addAll(v6);
|
|
|
- setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("v6", 5.0).intValue(), param, ReturnVideoRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
//-------------------新地域召回------------------
|
|
|
- List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
|
|
|
- v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
- v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
|
|
|
- rovRecallRank.addAll(v1);
|
|
|
- setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("v1", 5.0).intValue(), param, RegionRealtimeRecallStrategyV1.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
//-------------------scene cf rovn------------------
|
|
|
- List<Video> sceneCFRovn = extractAndSort(param, SceneCFRovnRecallStrategy.PUSH_FORM);
|
|
|
- sceneCFRovn = sceneCFRovn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
- sceneCFRovn = sceneCFRovn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), sceneCFRovn.size()));
|
|
|
- rovRecallRank.addAll(sceneCFRovn);
|
|
|
- setVideo.addAll(sceneCFRovn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), param, SceneCFRovnRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
//-------------------scene cf rosn------------------
|
|
|
- List<Video> sceneCFRosn = extractAndSort(param, SceneCFRosnRecallStrategy.PUSH_FORM);
|
|
|
- sceneCFRosn = sceneCFRosn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
- sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
|
|
|
- rovRecallRank.addAll(sceneCFRosn);
|
|
|
- setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
- // -------------------cate1------------------
|
|
|
- int cate1RecallN = mergeWeight.getOrDefault("cate1RecallN", 5.0).intValue();
|
|
|
- addRecall(param, cate1RecallN, UserCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
- // -------------------cate2------------------
|
|
|
- int cate2RecallN = mergeWeight.getOrDefault("cate2RecallN", 5.0).intValue();
|
|
|
- addRecall(param, cate2RecallN, UserCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), param, SceneCFRosnRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+ // -------------------user cate1------------------
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("cate1RecallN", 5.0).intValue(), param, UserCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+ // -------------------user cate2------------------
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("cate2RecallN", 5.0).intValue(), param, UserCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
// -------------------head province cate1------------------
|
|
|
- int headCate1RecallN = mergeWeight.getOrDefault("headCate1RecallN", 3.0).intValue();
|
|
|
- addRecall(param, headCate1RecallN, HeadProvinceCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate1RecallN", 3.0).intValue(), param, HeadProvinceCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
// -------------------head province cate2------------------
|
|
|
- int headCate2RecallN = mergeWeight.getOrDefault("headCate2RecallN", 3.0).intValue();
|
|
|
- addRecall(param, headCate2RecallN, HeadProvinceCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate2RecallN", 3.0).intValue(), param, HeadProvinceCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
//-------------------head cate2 of rovn------------------
|
|
|
- List<Video> headCate2Rov = extractAndSort(param, HeadCate2RovRecallStrategy.PUSH_FROM);
|
|
|
- // 视频去重
|
|
|
- removeDuplicate(headCate2Rov);
|
|
|
- headCate2Rov = headCate2Rov.stream().filter(o -> !setVideo.contains(o.getVideoId())).collect(Collectors.toList());
|
|
|
- headCate2Rov = headCate2Rov.subList(0, Math.min(mergeWeight.getOrDefault("headCate2Rov", 5.0).intValue(), headCate2Rov.size()));
|
|
|
- rovRecallRank.addAll(headCate2Rov);
|
|
|
- setVideo.addAll(headCate2Rov.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate2Rov", 5.0).intValue(), param, HeadCate2RovRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+ //-------------------city rovn------------------
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("cityRov", 5.0).intValue(), param, CityRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+ //-------------------priori province rovn------------------
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRov", 3.0).intValue(), param, PrioriProvinceRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+ //-------------------priori province str------------------
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceStr", 1.0).intValue(), param, PrioriProvinceStrRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+ //-------------------priori province ros------------------
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRos", 1.0).intValue(), param, PrioriProvinceRosRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+ //-------------------return1 cate2 ros------------------
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("return1Cate2Ros", 5.0).intValue(), param, Return1Cate2RosRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+ //-------------------return1 cate2 str------------------
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("return1Cate2Str", 5.0).intValue(), param, Return1Cate2StrRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
|
|
|
//-------------------排-------------------
|
|
|
//-------------------序-------------------
|
|
|
//-------------------逻-------------------
|
|
|
//-------------------辑-------------------
|
|
|
+ Map<String, String> rtFeatureDumpsMap = dumpsRtFeature(param.getUserRTShareList());
|
|
|
|
|
|
// 1. 批量获取特征 省份参数要对齐 headvid 要传递过来！
|
|
|
// k1:视频、k2:表、k3:特征、v:特征值
|
|
|
+ Map<String, String> headVideoInfo = param.getHeadInfo();
|
|
|
List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
|
|
|
- String headVid = String.valueOf(param.getHeadVid());
|
|
|
- Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo(headVid, vids);
|
|
|
- FeatureService.Feature feature = featureService.getFeatureV3(param, videoBaseInfoMap, vids);
|
|
|
+ Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", vids);
|
|
|
+ FeatureService.Feature feature = featureService.getFeatureV4(param, headVideoInfo, videoBaseInfoMap, vids);
|
|
|
Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
|
|
|
Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
|
|
|
- Map<String, String> headVideoInfo = videoBaseInfoMap.getOrDefault(headVid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
|
|
|
// 2. 用户信息预处理
|
|
|
Map<String, Map<String, String[]>> newC7Map = FeatureV6.parseUCFScore(featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>()));
|
|
|
Map<String, Map<String, String[]>> newC8Map = FeatureV6.parseUCFScore(featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>()));
|
|
|
UserShareReturnProfile userProfile = parseUserProfile(featureOriginUser);
|
|
|
- Map<String, Map<String, String>> userBehaviorVideoMap = getUserBehaviorVideoMap(userProfile);
|
|
|
+ Map<String, Map<String, String>> userBehaviorVideoMap = param.getBehaviorVideos();
|
|
|
+ Map<String, String> creativeInfo = param.getCreativeInfoFeature();
|
|
|
|
|
|
// 3. 特征处理
|
|
|
List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
|
|
|
- Map<String, Float> userFeatureMap = getUserFeature(currentMs, param, headVideoInfo, userProfile, featureOriginUser);
|
|
|
- batchGetVideoFeature(currentMs, userProfile, headVideoInfo, videoBaseInfoMap,
|
|
|
+ Map<String, Float> userFeatureMap = getUserFeature(currentMs, param, creativeInfo, headVideoInfo, userProfile, featureOriginUser);
|
|
|
+ batchGetVideoFeature(currentMs, userProfile, creativeInfo, headVideoInfo, videoBaseInfoMap,
|
|
|
newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, rankItems);
|
|
|
|
|
|
// 4. 排序模型计算
|
|
|
Map<String, Float> sceneFeatureMap = new HashMap<>(0);
|
|
|
- List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250317.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
|
|
|
+ List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_xgb_ros_20250311.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
|
|
|
|
|
|
// 5. 排序公式特征
|
|
|
- double alpha = mergeWeight.getOrDefault("alpha", 0.1);
|
|
|
- double beta = mergeWeight.getOrDefault("beta", 0.0);
|
|
|
- double gamma = mergeWeight.getOrDefault("gamma", 0.5);
|
|
|
- double vorMod = mergeWeight.getOrDefault("vorMod", 4.0);
|
|
|
+ double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.059);
|
|
|
double xgbNorPowerWeight = mergeWeight.getOrDefault("xgbNorPowerWeight", 1.22);
|
|
|
double xgbNorPowerExp = mergeWeight.getOrDefault("xgbNorPowerExp", 1.15);
|
|
|
-
|
|
|
Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
|
|
|
|
|
|
- Map<String, Map<String, String>> rosSpreadDivMap = this.getVideoRedisFeature(vids, "vid_for_spread:");
|
|
|
+ // 获取权重
|
|
|
+ Map<String, Double> cate2Coefficient = new HashMap<>();
|
|
|
+ double cate2CoefficientFunc = mergeWeight.getOrDefault("cate2CoefficientFunc", 0d);
|
|
|
+ if (cate2CoefficientFunc == 1d) {
|
|
|
+ String headVidStr = String.valueOf(param.getHeadVid());
|
|
|
+ String mergeCate2 = this.findVideoMergeCate2(videoBaseInfoMap, headVidStr);
|
|
|
+ Double length = mergeWeight.getOrDefault("cate2CoefficientLength", 10000d);
|
|
|
+ Map<String, Double> simCateScore = this.findSimCateScore(mergeCate2, length.intValue());
|
|
|
+ cate2Coefficient.putAll(simCateScore);
|
|
|
+ }
|
|
|
+ Double cate2CoefficientDenominator = mergeWeight.getOrDefault("cate2CoefficientDenominator", 1d);
|
|
|
+ Map<String, String> contextInfo = getContextInfo(param);
|
|
|
|
|
|
List<Video> result = new ArrayList<>();
|
|
|
for (RankItem item : items) {
|
|
|
+ double score;
|
|
|
double fmRovOrigin = item.getScoreRov();
|
|
|
item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
|
|
|
- double fmRov = restoreScore(fmRovOrigin);
|
|
|
+ double fmRov = restoreScore(fmRovOrigin, xgbRovNegRate);
|
|
|
item.getScoresMap().put("fmRov", fmRov);
|
|
|
- double strTransfor = fmRov;
|
|
|
- item.getScoresMap().put("strTransfor", strTransfor);
|
|
|
- double originRos = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>(0)).getOrDefault("rov", "0"));
|
|
|
- item.getScoresMap().put("hasReturnRovScore", originRos);
|
|
|
- double rosTransfor = originRos;
|
|
|
- item.getScoresMap().put("rosTransfor", rosTransfor);
|
|
|
- double vor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
|
|
|
- item.getScoresMap().put("vor", vor);
|
|
|
- double vorTransfor = this.handleVor(vor, vorMod, item, mergeWeight);
|
|
|
- item.getScoresMap().put("vorTransfor", vorTransfor);
|
|
|
- double spreadRate = Double.parseDouble(rosSpreadDivMap.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>(0)).getOrDefault("head_video_recommend_fission_rate", "0.01"));
|
|
|
- item.getScoresMap().put("spreadRate", spreadRate);
|
|
|
-
|
|
|
+ double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
|
|
|
+ item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
|
|
|
double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);
|
|
|
double newNorXGBScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norXGBScore);
|
|
|
- item.getScoresMap().put("norXGBScore", norXGBScore);
|
|
|
- item.getScoresMap().put("newNorXGBScore", newNorXGBScore);
|
|
|
-
|
|
|
- // 最终融合公式计算
|
|
|
- double scoreRov = strTransfor * (alpha + rosTransfor);
|
|
|
- double scoreVov = scoreRov * (beta + newNorXGBScore);
|
|
|
- double score = scoreVov + gamma * spreadRate;
|
|
|
+ double vor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
|
|
|
+ item.getScoresMap().put("vor", vor);
|
|
|
|
|
|
- item.getScoresMap().put("scoreRov", scoreRov);
|
|
|
- item.getScoresMap().put("scoreVov", scoreVov);
|
|
|
- item.getScoresMap().put("score", score);
|
|
|
+ String vidMergeCate2 = this.findVideoMergeCate2(videoBaseInfoMap, String.valueOf(item.getVideoId()));
|
|
|
+ Double scoreCoefficient = cate2Coefficient.getOrDefault(vidMergeCate2, 0d);
|
|
|
+ item.getScoresMap().put("scoreCoefficient", scoreCoefficient);
|
|
|
+ item.getScoresMap().put("cate2CoefficientDenominator", cate2CoefficientDenominator);
|
|
|
|
|
|
- item.getScoresMap().put("alpha", alpha);
|
|
|
- item.getScoresMap().put("beta", beta);
|
|
|
- item.getScoresMap().put("gamma", gamma);
|
|
|
+ score = fmRov * (0.1 + newNorXGBScore) * (0.1 + vor) * (1 + scoreCoefficient / cate2CoefficientDenominator);
|
|
|
|
|
|
Video video = item.getVideo();
|
|
|
video.setScore(score);
|
|
|
@@ -212,7 +161,7 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
video.setScoresMap(item.getScoresMap());
|
|
|
// video.setAllFeatureMap(item.getAllFeatureMap());
|
|
|
|
|
|
- String mergeCate2 = ExtractVideoMergeCate.parseMergeCate2(String.valueOf(item.getVideoId()), featureOriginVideo);
|
|
|
+ String mergeCate2 = ExtractVideoMergeCate.parseMergeCate2(String.valueOf(item.getVideoId()), videoBaseInfoMap);
|
|
|
if (StringUtils.isNotBlank(mergeCate2)) {
|
|
|
video.getMergeCateList().add(mergeCate2);
|
|
|
}
|
|
|
@@ -220,9 +169,24 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
if (MapUtils.isNotEmpty(feature.getVideoFeature()) && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
|
|
|
video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
|
|
|
}
|
|
|
+ if (MapUtils.isNotEmpty(videoBaseInfoMap) && MapUtils.isNotEmpty(videoBaseInfoMap.get(item.getVideoId() + ""))) {
|
|
|
+ video.getMetaFeatureMap().putAll(videoBaseInfoMap.get(item.getVideoId() + ""));
|
|
|
+ }
|
|
|
+ if (MapUtils.isNotEmpty(headVideoInfo)) {
|
|
|
+ video.getMetaFeatureMap().put("head_video", headVideoInfo);
|
|
|
+ }
|
|
|
if (MapUtils.isNotEmpty(feature.getUserFeature())) {
|
|
|
video.getMetaFeatureMap().putAll(feature.getUserFeature());
|
|
|
}
|
|
|
+ if (null != rtFeatureDumpsMap && !rtFeatureDumpsMap.isEmpty()) {
|
|
|
+ video.getMetaFeatureMap().put("rt", rtFeatureDumpsMap);
|
|
|
+ }
|
|
|
+ if (MapUtils.isNotEmpty(param.getCreativeInfoFeature())) {
|
|
|
+ video.getMetaFeatureMap().put("creativeInfo", param.getCreativeInfoFeature());
|
|
|
+ }
|
|
|
+ if (MapUtils.isNotEmpty(contextInfo)) {
|
|
|
+ video.getMetaFeatureMap().put("context", contextInfo);
|
|
|
+ }
|
|
|
result.add(video);
|
|
|
}
|
|
|
ExtractVideoMergeCate.addOtherParam(result, videoBaseInfoMap);
|
|
|
@@ -247,42 +211,13 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
- private Map<String, Map<String, String>> getUserBehaviorVideoMap(UserShareReturnProfile userProfile) {
|
|
|
- Set<String> vidSet = new HashSet<>();
|
|
|
- if (null != userProfile) {
|
|
|
- for (List<UserSRBO> list : Arrays.asList(userProfile.getM_s_s(), userProfile.getM_r_s(), userProfile.getL_s_s(), userProfile.getL_r_s())) {
|
|
|
- if (null != list) {
|
|
|
- for (UserSRBO u : list) {
|
|
|
- if (null != u) {
|
|
|
- vidSet.add(u.getId() + "");
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- Map<String, Map<String, String>> historyVideoMap = new HashMap<>();
|
|
|
- if (!vidSet.isEmpty()) {
|
|
|
- Map<String, Map<String, Map<String, String>>> videoMap = featureService.getVideoBaseInfo("", new ArrayList<>(vidSet));
|
|
|
- if (null != videoMap && !videoMap.isEmpty()) {
|
|
|
- for (Map.Entry<String, Map<String, Map<String, String>>> entry : videoMap.entrySet()) {
|
|
|
- String vid = entry.getKey();
|
|
|
- Map<String, Map<String, String>> map = entry.getValue();
|
|
|
- if (null != map && map.containsKey("alg_vid_feature_basic_info")) {
|
|
|
- historyVideoMap.put(vid, map.get("alg_vid_feature_basic_info"));
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- return historyVideoMap;
|
|
|
- }
|
|
|
-
|
|
|
- private Map<String, Float> getUserFeature(long currentMs, RankParam param, Map<String, String> headInfo, UserShareReturnProfile userProfile, Map<String, Map<String, String>> userOriginInfo) {
|
|
|
+ private Map<String, Float> getUserFeature(long currentMs, RankParam param, Map<String, String> creativeInfo, Map<String, String> headInfo, UserShareReturnProfile userProfile, Map<String, Map<String, String>> userOriginInfo) {
|
|
|
Map<String, Double> featMap = new HashMap<>();
|
|
|
// context feature
|
|
|
String appType = String.valueOf(param.getAppType());
|
|
|
String hotSceneType = String.valueOf(param.getHotSceneType());
|
|
|
FeatureV6.getContextFeature(currentMs, appType, hotSceneType, featMap);
|
|
|
+ FeatureV6.getCreativeBaseFeature("e1", creativeInfo, featMap);
|
|
|
|
|
|
// head video feature
|
|
|
FeatureV6.getVideoBaseFeature("h", currentMs, headInfo, featMap);
|
|
|
@@ -297,6 +232,7 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
|
|
|
private Map<String, Float> getVideoFeature(long currentMs, String vid,
|
|
|
UserShareReturnProfile userProfile,
|
|
|
+ Map<String, String> creativeInfo,
|
|
|
Map<String, String> headInfo, Map<String, String> rankInfo,
|
|
|
Map<String, Map<String, String[]>> c7Map,
|
|
|
Map<String, Map<String, String[]>> c8Map,
|
|
|
@@ -316,6 +252,7 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
|
|
|
// head&rank cross feature
|
|
|
FeatureV6.getHeadRankVideoCrossFeature(headInfo, rankInfo, featMap);
|
|
|
+ FeatureV6.getCreativeCrossFeature("e1", creativeInfo, rankInfo, featMap);
|
|
|
|
|
|
// user profile & rank cross
|
|
|
FeatureV6.getProfileVideoCrossFeature(currentMs, userProfile, rankInfo, historyVideoMap, featMap);
|
|
|
@@ -325,6 +262,7 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
|
|
|
private void batchGetVideoFeature(long currentMs,
|
|
|
UserShareReturnProfile userProfile,
|
|
|
+ Map<String, String> creativeInfo,
|
|
|
Map<String, String> headInfo,
|
|
|
Map<String, Map<String, Map<String, String>>> videoBaseInfoMap,
|
|
|
Map<String, Map<String, String[]>> c7Map,
|
|
|
@@ -339,7 +277,7 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
String vid = item.getVideoId() + "";
|
|
|
Map<String, String> rankInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
Future<Integer> future = ThreadPoolFactory.defaultPool().submit(() -> {
|
|
|
- item.featureMap = getVideoFeature(currentMs, vid, userProfile, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
|
|
|
+ item.featureMap = getVideoFeature(currentMs, vid, userProfile, creativeInfo, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
|
|
|
item.norFeatureMap = item.featureMap;
|
|
|
return 1;
|
|
|
});
|
|
|
@@ -393,6 +331,13 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
baseInfo.put("system", system);
|
|
|
}
|
|
|
}
|
|
|
+ String userChannel = param.getChannelName();
|
|
|
+ if (null != userChannel && !userChannel.isEmpty()) {
|
|
|
+ baseInfo.put("user_channel", userChannel);
|
|
|
+ }
|
|
|
+ if (FeatureUtils.firstLevel(param.getUserShareDepth())) {
|
|
|
+ baseInfo.put("user_level", "1st");
|
|
|
+ }
|
|
|
return baseInfo;
|
|
|
}
|
|
|
|
|
|
@@ -406,14 +351,47 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
|
|
|
return newScore;
|
|
|
}
|
|
|
|
|
|
- private void addRecall(RankParam param, int recallNum, String recallName, Set<Long> setVideo, List<Video> rovRecallRank) {
|
|
|
- if (recallNum > 0) {
|
|
|
- List<Video> list = extractAndSort(param, recallName);
|
|
|
- list = list.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
- list = list.subList(0, Math.min(recallNum, list.size()));
|
|
|
- rovRecallRank.addAll(list);
|
|
|
- setVideo.addAll(list.stream().map(Video::getVideoId).collect(Collectors.toSet()));
|
|
|
+ private Map<String, Double> findSimCateScore(String headCate2, int length) {
|
|
|
+ if (StringUtils.isBlank(headCate2)) {
|
|
|
+ return new HashMap<>();
|
|
|
+ }
|
|
|
+
|
|
|
+ String redisKey = String.format("alg_recsys_good_cate_pair_list:%s", headCate2);
|
|
|
+ String cate2Value = redisTemplate.opsForValue().get(redisKey);
|
|
|
+ if (StringUtils.isEmpty(cate2Value)) {
|
|
|
+ return new HashMap<>();
|
|
|
+ }
|
|
|
+
|
|
|
+ return this.parsePair(cate2Value, length);
|
|
|
+ }
|
|
|
+
|
|
|
+ private Map<String, Double> parsePair(String value, int length) {
|
|
|
+ if (StringUtils.isBlank(value)) {
|
|
|
+ return new HashMap<>();
|
|
|
+ }
|
|
|
+
|
|
|
+ String[] split = value.split("\t");
|
|
|
+ if (split.length != 2) {
|
|
|
+ return new HashMap<>();
|
|
|
}
|
|
|
+
|
|
|
+ String[] valueList = split[0].trim().split(",");
|
|
|
+ String[] scoreList = split[1].trim().split(",");
|
|
|
+ if (valueList.length != scoreList.length) {
|
|
|
+ return new HashMap<>();
|
|
|
+ }
|
|
|
+
|
|
|
+ int minLength = Math.min(length, valueList.length);
|
|
|
+ Map<String, Double> resultMap = new HashMap<>();
|
|
|
+ for (int i = 0; i < minLength; i++) {
|
|
|
+ resultMap.put(valueList[i].trim(), Double.parseDouble(scoreList[i].trim()));
|
|
|
+ }
|
|
|
+
|
|
|
+ return resultMap;
|
|
|
}
|
|
|
|
|
|
+ private String findVideoMergeCate2(Map<String, Map<String, Map<String, String>>> featureOriginVideo, String vid) {
|
|
|
+ Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
+ return videoInfo.get("merge_second_level_cate");
|
|
|
+ }
|
|
|
}
|
