|
|
@@ -7,30 +7,47 @@ import com.tzld.piaoquan.recommend.server.common.base.RankItem;
|
|
|
import com.tzld.piaoquan.recommend.server.model.MachineInfo;
|
|
|
import com.tzld.piaoquan.recommend.server.model.Video;
|
|
|
import com.tzld.piaoquan.recommend.server.service.FeatureService;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.funnel.ColdStartAction;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.funnel.FunnelContext;
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.bo.UserShareReturnProfile;
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractVideoMergeCate;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
|
|
|
import com.tzld.piaoquan.recommend.server.service.rank.tansform.FeatureV6;
|
|
|
import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
|
|
|
import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
|
|
|
import com.tzld.piaoquan.recommend.server.util.*;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.collections4.CollectionUtils;
|
|
|
import org.apache.commons.collections4.MapUtils;
|
|
|
+import org.apache.commons.lang3.RandomUtils;
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.beans.factory.annotation.Value;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
import java.util.*;
|
|
|
import java.util.concurrent.Future;
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+import java.util.stream.Stream;
|
|
|
|
|
|
/**
|
|
|
- * V536 实验:基于 V569,对以下 4 路召回来源命中的 item 在排序公式上提权
|
|
|
- * - recall_strategy_user_cate1
|
|
|
- * - recall_strategy_user_cate2
|
|
|
- * - return_1_cate2_ros
|
|
|
- * - return_1_cate2_str
|
|
|
- * 提权倍数读 mergeWeight[item_recall_weight],默认 1.0。
|
|
|
+ * V536 实验(2026-05-27 复写):基于 V565 召回 + DNN 打分,fusion 后续做减法
|
|
|
+ *
|
|
|
+ * 当前状态(基线 commit):mergeAndSort 完整等同 Basic.mergeAndSort(含 boost/filter/insert/festive/density 后处理)
|
|
|
+ * 后续 commit 会做减法:删除 rov boost / 强插 / 标签 filter / 品类降权 / 节日降权 / 密度控制,
|
|
|
+ * 只保留"流量池按比例强插 + rov 兜底"机制。
|
|
|
+ *
|
|
|
+ * Apollo key 保留 ${rank.score.merge.weightv536},召回/打分逻辑与 V565 同源但参数独立可调,仅 fusion 策略不同。
|
|
|
+ * 注意:召回侧需要在 RecallService 里加 isHit536Exp 块,镜像 V565 的 strategies 增删(all_rov 系列 + 9 路老召回剔除)。
|
|
|
+ *
|
|
|
+ * 历史:原 V536(V569 + 4 路召回提权)在 2026-05-27 被复写为本逻辑。
|
|
|
*/
|
|
|
@Service
|
|
|
@Slf4j
|
|
|
@@ -41,13 +58,6 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
@Autowired
|
|
|
private FeatureService featureService;
|
|
|
|
|
|
- private static final Set<String> BOOST_PUSH_FORMS = new HashSet<>(Arrays.asList(
|
|
|
- UserCate1RecallStrategy.PUSH_FORM,
|
|
|
- UserCate2RecallStrategy.PUSH_FORM,
|
|
|
- Return1Cate2RosRecallStrategy.PUSH_FORM,
|
|
|
- Return1Cate2StrRecallStrategy.PUSH_FORM
|
|
|
- ));
|
|
|
-
|
|
|
@Override
|
|
|
public List<Video> mergeAndRankRovRecall(RankParam param) {
|
|
|
Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
|
|
|
@@ -61,12 +71,10 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
Set<Long> setVideo = new HashSet<>();
|
|
|
setVideo.add(param.getHeadVid());
|
|
|
List<Video> rovRecallRank = new ArrayList<>();
|
|
|
- // -------------------5路特殊旧召回------------------
|
|
|
- RecallUtils.extractOldSpecialRecall(mergeWeight.getOrDefault("oldSpecialN", (double) param.getSize()).intValue(), param, setVideo, rovRecallRank);
|
|
|
//-------------------return相似召回------------------
|
|
|
RecallUtils.extractRecall(mergeWeight.getOrDefault("v6", 5.0).intValue(), param, ReturnVideoRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
- //-------------------新地域召回------------------
|
|
|
- RecallUtils.extractRecall(mergeWeight.getOrDefault("v1", 5.0).intValue(), param, RegionRealtimeRecallStrategyV1.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+ //-------------------新地域召回 (V565: all_rov)------------------
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("v1", 5.0).intValue(), param, RegionRealtimeRecallStrategyV1AllRov.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
//-------------------scene cf rovn------------------
|
|
|
RecallUtils.extractRecall(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), param, SceneCFRovnRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
//-------------------scene cf rosn------------------
|
|
|
@@ -81,20 +89,12 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate2RecallN", 3.0).intValue(), param, HeadProvinceCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
//-------------------head cate2 of rovn------------------
|
|
|
RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate2Rov", 5.0).intValue(), param, HeadCate2RovRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
- //-------------------city rovn------------------
|
|
|
- RecallUtils.extractRecall(mergeWeight.getOrDefault("cityRov", 5.0).intValue(), param, CityRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
- //-------------------priori province rovn------------------
|
|
|
- RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRov", 3.0).intValue(), param, PrioriProvinceRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
- //-------------------priori province str------------------
|
|
|
- RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceStr", 1.0).intValue(), param, PrioriProvinceStrRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
- //-------------------priori province ros------------------
|
|
|
- RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRos", 1.0).intValue(), param, PrioriProvinceRosRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+ //-------------------city rovn (V565: all_rov)------------------
|
|
|
+ RecallUtils.extractRecall(mergeWeight.getOrDefault("cityRov", 5.0).intValue(), param, CityRovnAllRovRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
//-------------------return1 cate2 ros------------------
|
|
|
RecallUtils.extractRecall(mergeWeight.getOrDefault("return1Cate2Ros", 5.0).intValue(), param, Return1Cate2RosRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
//-------------------return1 cate2 str------------------
|
|
|
RecallUtils.extractRecall(mergeWeight.getOrDefault("return1Cate2Str", 5.0).intValue(), param, Return1Cate2StrRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
- //--------------deconstruction keywords ros-------------
|
|
|
- RecallUtils.extractRecall(mergeWeight.getOrDefault("deconstructionKeywordsRos", 5.0).intValue(), param, UserDeconstructionKeywordsRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
|
|
|
RecallUtils.extractRecall(mergeWeight.getOrDefault("yearShareCate1", 5.0).intValue(), param, YearShareCate1RecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
RecallUtils.extractRecall(mergeWeight.getOrDefault("yearShareCate2", 5.0).intValue(), param, YearShareCate2RecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
@@ -112,8 +112,22 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
// 1. 批量获取特征 省份参数要对齐 headvid 要传递过来!
|
|
|
// k1:视频、k2:表、k3:特征、v:特征值
|
|
|
Map<String, String> headVideoInfo = param.getHeadInfo();
|
|
|
+
|
|
|
+ // 用户的序列特征
|
|
|
+ Map<String, Map<String, String>> unionIdFeature = featureService.getUnionIdFeature(param.getUnionId());
|
|
|
+ Map<String, String> userNetworkSeqFeature = unionIdFeature.getOrDefault("alg_user_network_seq_feature", new HashMap<>());
|
|
|
+ List<String> actVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_v_s");
|
|
|
+ List<String> netVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "n_v_s");
|
|
|
+
|
|
|
List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
|
|
|
- Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", vids);
|
|
|
+
|
|
|
+ List<String> allVids = Stream.of(actVidSeq, netVidSeq, vids)
|
|
|
+ .flatMap(Collection::stream)
|
|
|
+ .distinct()
|
|
|
+ .filter(StringUtils::isNotBlank)
|
|
|
+ .collect(Collectors.toList());
|
|
|
+
|
|
|
+ Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", allVids);
|
|
|
Map<String, Map<String, Map<String, String>>> videoBCData = featureService.getVideoStatistics(vids);
|
|
|
|
|
|
FeatureService.Feature feature = featureService.getFeatureV4(param, headVideoInfo, videoBaseInfoMap, vids);
|
|
|
@@ -127,15 +141,20 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
Map<String, Map<String, String>> userBehaviorVideoMap = param.getBehaviorVideos();
|
|
|
Map<String, String> creativeInfo = param.getCreativeInfoFeature();
|
|
|
|
|
|
+ Map<String, String> featureMapToString = new HashMap<>();
|
|
|
+ FeatureV6.parseStringFeatureMap(featureMapToString, param);
|
|
|
+ FeatureV6.putVideoStringFeatures("h", headVideoInfo, featureMapToString);
|
|
|
+
|
|
|
// 3. 特征处理
|
|
|
List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
|
|
|
Map<String, Float> userFeatureMap = getUserFeature(currentMs, param, creativeInfo, headVideoInfo, userProfile, featureOriginUser);
|
|
|
batchGetVideoFeature(currentMs, userProfile, creativeInfo, headVideoInfo, videoBaseInfoMap,
|
|
|
- newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, rankItems);
|
|
|
+ newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, featureMapToString, userFeatureMap, rankItems);
|
|
|
+
|
|
|
|
|
|
// 4. 排序模型计算
|
|
|
Map<String, Float> sceneFeatureMap = new HashMap<>(0);
|
|
|
- List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_str_and_ros_20260319.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
|
|
|
+ List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_dnn_20260407.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
|
|
|
|
|
|
// 5. 排序公式特征
|
|
|
double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.059);
|
|
|
@@ -157,9 +176,15 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
double b0Ror1hW = mergeWeight.getOrDefault("b0_ror_1h_w", 0d);
|
|
|
double b0Ror24hW = mergeWeight.getOrDefault("b0_ror_24h_w", 0d);
|
|
|
|
|
|
+ double cnRovn1hW = mergeWeight.getOrDefault("cn_rovn_1h_w", 0d);
|
|
|
+ double cnRovn24hW = mergeWeight.getOrDefault("cn_rovn_24h_w", 0d);
|
|
|
+
|
|
|
+ double dnRovn1hW = mergeWeight.getOrDefault("dn_rovn_1h_w", 0d);
|
|
|
+ double dnRovn24hW = mergeWeight.getOrDefault("dn_rovn_24h_w", 0d);
|
|
|
+
|
|
|
Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
|
|
|
|
|
|
- // 获取权重
|
|
|
+
|
|
|
Map<String, String> contextInfo = getContextInfo(param);
|
|
|
|
|
|
List<Video> result = new ArrayList<>();
|
|
|
@@ -173,8 +198,9 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
|
|
|
item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
|
|
|
|
|
|
- double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);
|
|
|
- double newNorXGBScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norXGBScore);
|
|
|
+ double norDNNScore = item.getScoresMap().getOrDefault("NorDNNScore", 0d);
|
|
|
+ double newNorDNNScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norDNNScore);
|
|
|
+ item.getScoresMap().put("newNorDNNScore", newNorDNNScore);
|
|
|
item.getScoresMap().put("rosAdd", rosAdd);
|
|
|
item.getScoresMap().put("rosW", rosW);
|
|
|
|
|
|
@@ -184,6 +210,7 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
item.getScoresMap().put("vorW", vorW);
|
|
|
|
|
|
Map<String, String> bcData = videoBCData.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>()).getOrDefault("alg_vid_feature_b_c_data", new HashMap<>());
|
|
|
+ Map<String, String> cdNData = videoBCData.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>()).getOrDefault("alg_vid_feature_cn_dn_data", new HashMap<>());
|
|
|
|
|
|
double c1Rovn1h = Double.parseDouble(bcData.getOrDefault("c1_rovn_1h", "0"));
|
|
|
double c1Rovn24h = Double.parseDouble(bcData.getOrDefault("c1_rovn_24h", "0"));
|
|
|
@@ -203,6 +230,7 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
item.getScoresMap().put("b0Str24hW", b0Str24hW);
|
|
|
item.getScoresMap().put("b0Str24h", b0Str24h);
|
|
|
|
|
|
+
|
|
|
double b0Ror1h = Double.parseDouble(bcData.getOrDefault("b_ror1_1h", "0"));
|
|
|
double b0Ror24h = Double.parseDouble(bcData.getOrDefault("b_ror1_24h", "0"));
|
|
|
double b0RorScore = b0Ror1hW * b0Ror1h + b0Ror24hW * b0Ror24h;
|
|
|
@@ -212,20 +240,27 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
item.getScoresMap().put("b0Ror24hW", b0Ror24hW);
|
|
|
item.getScoresMap().put("b0Ror24h", b0Ror24h);
|
|
|
|
|
|
- score = fmRov * (rosAdd + rosW * newNorXGBScore) * (vorAdd + vorW * vor) + c1RovnScore + b0StrScore + b0RorScore;
|
|
|
+ double cnRovn1h = Double.parseDouble(cdNData.getOrDefault("cn_rovn_1h", "0"));
|
|
|
+ double cnRovn24h = Double.parseDouble(cdNData.getOrDefault("cn_rovn_24h", "0"));
|
|
|
+ double cnRovnScore = cnRovn1hW * cnRovn1h + cnRovn24hW * cnRovn24h;
|
|
|
+ item.getScoresMap().put("cnRovnScore", cnRovnScore);
|
|
|
+ item.getScoresMap().put("cnRovn1hW", cnRovn1hW);
|
|
|
+ item.getScoresMap().put("cnRovn1h", cnRovn1h);
|
|
|
+ item.getScoresMap().put("cnRovn24hW", cnRovn24hW);
|
|
|
+ item.getScoresMap().put("cnRovn24h", cnRovn24h);
|
|
|
+
|
|
|
+ double dnRovn1h = Double.parseDouble(cdNData.getOrDefault("dn_rovn_1h", "0"));
|
|
|
+ double dnRovn24h = Double.parseDouble(cdNData.getOrDefault("dn_rovn_24h", "0"));
|
|
|
+ double dnRovnScore = dnRovn1hW * dnRovn1h + dnRovn24hW * dnRovn24h;
|
|
|
+ item.getScoresMap().put("dnRovnScore", dnRovnScore);
|
|
|
+ item.getScoresMap().put("dnRovn1hW", dnRovn1hW);
|
|
|
+ item.getScoresMap().put("dnRovn1h", dnRovn1h);
|
|
|
+ item.getScoresMap().put("dnRovn24hW", dnRovn24hW);
|
|
|
+ item.getScoresMap().put("dnRovn24h", dnRovn24h);
|
|
|
+
|
|
|
+ score = fmRov * (rosAdd + rosW * newNorDNNScore) * (vorAdd + vorW * vor) + c1RovnScore + b0StrScore + b0RorScore + cnRovnScore + dnRovnScore;
|
|
|
|
|
|
- // V536: 命中目标 4 路召回来源的 item 提权
|
|
|
Video video = item.getVideo();
|
|
|
- double itemRecallWeight = mergeWeight.getOrDefault("item_recall_weight", 1.0);
|
|
|
- double itemWeight = 1.0;
|
|
|
- if (video.getPushFromIndex() != null
|
|
|
- && !Collections.disjoint(video.getPushFromIndex().keySet(), BOOST_PUSH_FORMS)) {
|
|
|
- itemWeight = itemRecallWeight;
|
|
|
- }
|
|
|
- item.getScoresMap().put("itemRecallWeight", itemRecallWeight);
|
|
|
- item.getScoresMap().put("itemWeight", itemWeight);
|
|
|
- score = score * itemWeight;
|
|
|
-
|
|
|
video.setScore(score);
|
|
|
video.setSortScore(score);
|
|
|
video.setScoresMap(item.getScoresMap());
|
|
|
@@ -257,7 +292,7 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
if (MapUtils.isNotEmpty(contextInfo)) {
|
|
|
video.getMetaFeatureMap().put("context", contextInfo);
|
|
|
}
|
|
|
- if (Objects.nonNull(video.getRankVideoInfoMap()) && video.getRankVideoInfoMap().containsKey(video.getVideoId())){
|
|
|
+ if (Objects.nonNull(video.getRankVideoInfoMap()) && video.getRankVideoInfoMap().containsKey(video.getVideoId())) {
|
|
|
video.getRankVideoInfoMap().get(video.getVideoId()).setScore(score);
|
|
|
video.getRankVideoInfoMap().get(video.getVideoId()).setScoresMap(video.getScoresMap());
|
|
|
}
|
|
|
@@ -268,6 +303,141 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
return result;
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * V536 fusion 基线 — 整段照搬 Basic.mergeAndSort(boost/filter/insert/festive/density 全保留)
|
|
|
+ * 后续 commit 会在这里做减法:删除标签 filter / rov boost / 强插 / 品类降权 / 节日降权 / 密度控制,
|
|
|
+ * 只保留"流量池按比例强插 + rov 兜底"机制。
|
|
|
+ */
|
|
|
+ @Override
|
|
|
+ public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos, List<Video> douHotFlowPoolVideos) {
|
|
|
+
|
|
|
+ // 1 兜底策略,rov池子不足时,用冷启池填补。直接返回。
|
|
|
+ if (CollectionUtils.isEmpty(rovVideos)) {
|
|
|
+ if (param.getSize() < flowVideos.size()) {
|
|
|
+ return new RankResult(flowVideos.subList(0, param.getSize()));
|
|
|
+ } else {
|
|
|
+ return new RankResult(flowVideos);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 2 根据实验号解析阿波罗参数。
|
|
|
+ Set<String> abExpCodes = param.getAbExpCodes();
|
|
|
+ Map<String, Map<String, String>> rulesMap = Collections.emptyMap();
|
|
|
+
|
|
|
+ Map<String, List<Map<String, String>>> rankReduceRulesMap = Collections.emptyMap();
|
|
|
+
|
|
|
+ if (CollectionUtils.isNotEmpty(abExpCodes)) {
|
|
|
+ for (Map.Entry<String, Map<String, Map<String, String>>> entry : this.filterRules.entrySet()) {
|
|
|
+ if (abExpCodes.contains(entry.getKey())) {
|
|
|
+ rulesMap = entry.getValue();
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ for (Map.Entry<String, Map<String, List<Map<String, String>>>> entry : this.rankReduceConfig.entrySet()) {
|
|
|
+ if (abExpCodes.contains(entry.getKey())) {
|
|
|
+ rankReduceRulesMap = entry.getValue();
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ // 3 标签读取
|
|
|
+ if (rulesMap != null && !rulesMap.isEmpty()) {
|
|
|
+ RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
|
|
|
+ extractorItemTags.processor(rovVideos, flowVideos, douHotFlowPoolVideos);
|
|
|
+ }
|
|
|
+ // 6 合并结果时间卡控
|
|
|
+ if (rulesMap != null && !rulesMap.isEmpty()) {
|
|
|
+ RankProcessorTagFilter.processor(rovVideos, flowVideos, douHotFlowPoolVideos, rulesMap);
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ // if (MapUtils.isNotEmpty(rankReduceRulesMap)) {
|
|
|
+ // ExtractVideoMergeCate.addMergeCate(rovVideos, flowVideos);
|
|
|
+ // }
|
|
|
+
|
|
|
+ // 4 rov池提权功能
|
|
|
+ RankProcessorBoost.boostByTag(rovVideos, rulesMap);
|
|
|
+
|
|
|
+ // 5 rov池强插功能
|
|
|
+ RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
|
|
|
+
|
|
|
+ // 6.品类降权
|
|
|
+ RankProcessorBoost.boostByMergeCate(rovVideos, rankReduceRulesMap);
|
|
|
+
|
|
|
+ // 节日视频降权
|
|
|
+ RankProcessorBoost.boostByFestive(param, rovVideos, rankReduceByFestiveConfig);
|
|
|
+
|
|
|
+ // 7 流量池按比例强插
|
|
|
+ FunnelContext funnelCtx = param.getFunnelContext();
|
|
|
+ List<Video> result = new ArrayList<>();
|
|
|
+ for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
|
|
|
+ result.add(rovVideos.get(i));
|
|
|
+ }
|
|
|
+ double flowPoolP = getFlowPoolP(param);
|
|
|
+ int flowPoolIndex = 0;
|
|
|
+ int rovPoolIndex = param.getTopK();
|
|
|
+ for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
|
|
|
+ double rand = RandomUtils.nextDouble(0, 1);
|
|
|
+ if (rand < flowPoolP) {
|
|
|
+ if (flowPoolIndex < flowVideos.size()) {
|
|
|
+ Video v = flowVideos.get(flowPoolIndex++);
|
|
|
+ result.add(v);
|
|
|
+ markColdStartInserted(funnelCtx, v);
|
|
|
+ } else {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ } else if (this.isInsertDouHotFlowPoolVideo()) {
|
|
|
+ if (flowPoolIndex < douHotFlowPoolVideos.size()) {
|
|
|
+ Video v = douHotFlowPoolVideos.get(flowPoolIndex++);
|
|
|
+ result.add(v);
|
|
|
+ markColdStartInserted(funnelCtx, v);
|
|
|
+ } else {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ if (rovPoolIndex < rovVideos.size()) {
|
|
|
+ result.add(rovVideos.get(rovPoolIndex++));
|
|
|
+ } else {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (rovPoolIndex >= rovVideos.size()) {
|
|
|
+ for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
|
|
|
+ Video v = flowVideos.get(i);
|
|
|
+ result.add(v);
|
|
|
+ markColdStartInserted(funnelCtx, v);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (flowPoolIndex >= flowVideos.size()) {
|
|
|
+ for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
|
|
|
+ result.add(rovVideos.get(i));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 8 合并结果密度控制
|
|
|
+ Map<String, Integer> densityRules = new HashMap<>();
|
|
|
+ if (rulesMap != null && !rulesMap.isEmpty()) {
|
|
|
+ for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
|
|
|
+ String key = entry.getKey();
|
|
|
+ Map<String, String> value = entry.getValue();
|
|
|
+ if (value.containsKey("density")) {
|
|
|
+ densityRules.put(key, Integer.valueOf(value.get("density")));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
|
|
|
+ List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
+ List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
|
|
|
+ List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
|
|
|
+ rovRecallRankNew, flowPoolRankNew, densityRules);
|
|
|
+
|
|
|
+ return new RankResult(resultWithDensity);
|
|
|
+ }
|
|
|
+
|
|
|
private UserShareReturnProfile parseUserProfile(Map<String, Map<String, String>> userOriginInfo) {
|
|
|
if (null != userOriginInfo) {
|
|
|
Map<String, String> c9 = userOriginInfo.get("alg_recsys_feature_user_share_return_stat");
|
|
|
@@ -344,15 +514,28 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
Map<String, Map<String, String>> userOriginInfo,
|
|
|
Map<String, Map<String, String>> historyVideoMap,
|
|
|
Map<String, Map<String, Map<String, String>>> videoOriginInfo,
|
|
|
+ Map<String, String> featureMapToString,
|
|
|
+ Map<String, Float> userFeatureMap,
|
|
|
List<RankItem> rankItems) {
|
|
|
- if (null != rankItems && !rankItems.isEmpty()) {
|
|
|
+ if (CollectionUtils.isNotEmpty(rankItems)) {
|
|
|
List<Future<Integer>> futures = new ArrayList<>();
|
|
|
for (RankItem item : rankItems) {
|
|
|
String vid = item.getVideoId() + "";
|
|
|
Map<String, String> rankInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
Future<Integer> future = ThreadPoolFactory.defaultPool().submit(() -> {
|
|
|
- item.featureMap = getVideoFeature(currentMs, vid, userProfile, creativeInfo, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
|
|
|
- item.norFeatureMap = item.featureMap;
|
|
|
+ Map<String, Float> featureMap = new HashMap<>(userFeatureMap);
|
|
|
+ Map<String, Float> videoFeature = getVideoFeature(currentMs, vid, userProfile, creativeInfo, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
|
|
|
+ featureMap.putAll(videoFeature);
|
|
|
+ item.featureMap = featureMap;
|
|
|
+
|
|
|
+ Map<String, String> userNetworkSeqFeature = userOriginInfo.getOrDefault("alg_user_network_seq_feature", new HashMap<>());
|
|
|
+
|
|
|
+ Map<String, String> featureMapString = new HashMap<>(featureMapToString);
|
|
|
+ FeatureV6.putVideoStringFeatures("r", rankInfo, featureMapString);
|
|
|
+ featureMapString.put("r@vid", "r_vid_" + vid);
|
|
|
+ FeatureV6.putProfileVideoCrossStringFeature(currentMs, userProfile, historyVideoMap, featureMapString);
|
|
|
+ FeatureV6.putUserNetworkSeqFeature(featureMapString, userNetworkSeqFeature, videoBaseInfoMap);
|
|
|
+ item.featureMapString = featureMapString;
|
|
|
return 1;
|
|
|
});
|
|
|
futures.add(future);
|
|
|
@@ -424,48 +607,4 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
|
|
|
}
|
|
|
return newScore;
|
|
|
}
|
|
|
-
|
|
|
- private Map<String, Double> findSimCateScore(String headCate2, int length) {
|
|
|
- if (StringUtils.isBlank(headCate2)) {
|
|
|
- return new HashMap<>();
|
|
|
- }
|
|
|
-
|
|
|
- String redisKey = String.format("alg_recsys_good_cate_pair_list:%s", headCate2);
|
|
|
- String cate2Value = redisTemplate.opsForValue().get(redisKey);
|
|
|
- if (StringUtils.isEmpty(cate2Value)) {
|
|
|
- return new HashMap<>();
|
|
|
- }
|
|
|
-
|
|
|
- return this.parsePair(cate2Value, length);
|
|
|
- }
|
|
|
-
|
|
|
- private Map<String, Double> parsePair(String value, int length) {
|
|
|
- if (StringUtils.isBlank(value)) {
|
|
|
- return new HashMap<>();
|
|
|
- }
|
|
|
-
|
|
|
- String[] split = value.split("\t");
|
|
|
- if (split.length != 2) {
|
|
|
- return new HashMap<>();
|
|
|
- }
|
|
|
-
|
|
|
- String[] valueList = split[0].trim().split(",");
|
|
|
- String[] scoreList = split[1].trim().split(",");
|
|
|
- if (valueList.length != scoreList.length) {
|
|
|
- return new HashMap<>();
|
|
|
- }
|
|
|
-
|
|
|
- int minLength = Math.min(length, valueList.length);
|
|
|
- Map<String, Double> resultMap = new HashMap<>();
|
|
|
- for (int i = 0; i < minLength; i++) {
|
|
|
- resultMap.put(valueList[i].trim(), Double.parseDouble(scoreList[i].trim()));
|
|
|
- }
|
|
|
-
|
|
|
- return resultMap;
|
|
|
- }
|
|
|
-
|
|
|
- private String findVideoMergeCate2(Map<String, Map<String, Map<String, String>>> featureOriginVideo, String vid) {
|
|
|
- Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
- return videoInfo.get("merge_second_level_cate");
|
|
|
- }
|
|
|
}
|