|
@@ -2,17 +2,13 @@ package com.tzld.piaoquan.recommend.server.service.rank;
|
|
|
|
|
|
|
|
|
|
import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
-import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
|
|
|
|
import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
|
|
import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
|
|
-import com.tzld.piaoquan.recommend.server.model.MachineInfo;
|
|
|
|
import com.tzld.piaoquan.recommend.server.model.Video;
|
|
import com.tzld.piaoquan.recommend.server.model.Video;
|
|
import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
|
|
import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
|
|
import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
|
|
import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
|
|
import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
|
|
import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
|
|
-import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.apache.commons.collections4.CollectionUtils;
|
|
import org.apache.commons.collections4.CollectionUtils;
|
|
-import org.apache.commons.lang3.RandomUtils;
|
|
|
|
import org.apache.commons.lang3.math.NumberUtils;
|
|
import org.apache.commons.lang3.math.NumberUtils;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Qualifier;
|
|
import org.springframework.beans.factory.annotation.Qualifier;
|
|
@@ -20,7 +16,6 @@ import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.data.redis.core.RedisTemplate;
|
|
import org.springframework.data.redis.core.RedisTemplate;
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
-import java.text.SimpleDateFormat;
|
|
|
|
import java.util.*;
|
|
import java.util.*;
|
|
import java.util.stream.Collectors;
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
@@ -29,7 +24,7 @@ import java.util.stream.Collectors;
|
|
*/
|
|
*/
|
|
@Service
|
|
@Service
|
|
@Slf4j
|
|
@Slf4j
|
|
-public class RankService {
|
|
|
|
|
|
+public abstract class RankService {
|
|
@Autowired
|
|
@Autowired
|
|
@Qualifier("redisTemplate")
|
|
@Qualifier("redisTemplate")
|
|
public RedisTemplate<String, String> redisTemplate;
|
|
public RedisTemplate<String, String> redisTemplate;
|
|
@@ -75,7 +70,7 @@ public class RankService {
|
|
}
|
|
}
|
|
return new RankResult(results);
|
|
return new RankResult(results);
|
|
}
|
|
}
|
|
- if (this.matchSpecialApp(param.getAppType())){
|
|
|
|
|
|
+ if (this.matchSpecialApp(param.getAppType())) {
|
|
// 特殊app固定列表,只有一路召回,且不走排序。
|
|
// 特殊app固定列表,只有一路召回,且不走排序。
|
|
List<Video> results = new ArrayList<>(extractAndSort(param, RegionRealtimeRecallStrategyV5Hand.PUSH_FORM));
|
|
List<Video> results = new ArrayList<>(extractAndSort(param, RegionRealtimeRecallStrategyV5Hand.PUSH_FORM));
|
|
removeDuplicate(results);
|
|
removeDuplicate(results);
|
|
@@ -95,8 +90,6 @@ public class RankService {
|
|
List<Video> rovRecallRank = mergeAndRankRovRecall(param);
|
|
List<Video> rovRecallRank = mergeAndRankRovRecall(param);
|
|
List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
|
|
List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
|
|
|
|
|
|
- rankFilter(param, rovRecallRank, flowPoolRank);
|
|
|
|
-
|
|
|
|
removeDuplicate(param, rovRecallRank, flowPoolRank);
|
|
removeDuplicate(param, rovRecallRank, flowPoolRank);
|
|
|
|
|
|
|
|
|
|
@@ -156,105 +149,8 @@ public class RankService {
|
|
|
|
|
|
}
|
|
}
|
|
|
|
|
|
- public void rankFilter(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- public List<Video> mergeAndRankRovRecall(RankParam param) {
|
|
|
|
- // TODO ab test
|
|
|
|
- // TODO 抽象成Strategy
|
|
|
|
-
|
|
|
|
- return mergeAndRankRovRecallOld(param);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- public List<Video> mergeAndRankRovRecallOld(RankParam param) {
|
|
|
|
- List<Video> rovRecallRank = new ArrayList<>();
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
|
|
|
|
-
|
|
|
|
- // @zhangbo 增加不同召回子策略的返回数量-阿波罗:region.recall.return.size
|
|
|
|
- String abCode = param.getAbCode();
|
|
|
|
- int sizeReturn = param.getSize();
|
|
|
|
- if (abCode != null && this.regionRecallReturnSize != null &&
|
|
|
|
- this.regionRecallReturnSize.containsKey(abCode) &&
|
|
|
|
- this.regionRecallReturnSize.get(abCode) != null) {
|
|
|
|
- sizeReturn = this.regionRecallReturnSize.get(abCode).getOrDefault("result_size", sizeReturn);
|
|
|
|
- }
|
|
|
|
|
|
+ public abstract List<Video> mergeAndRankRovRecall(RankParam param);
|
|
|
|
|
|
- if (param.getAbCode().equals("60054")
|
|
|
|
- || param.getAbCode().equals("60068")
|
|
|
|
- || param.getAbCode().equals("60081")
|
|
|
|
- || param.getAbCode().equals("60084")
|
|
|
|
- || param.getAbCode().equals("60087")
|
|
|
|
- || param.getAbCode().equals("60088")
|
|
|
|
- || param.getAbCode().equals("60089")
|
|
|
|
- || param.getAbCode().equals("60090")
|
|
|
|
- || param.getAbCode().equals("60091")
|
|
|
|
- || param.getAbCode().equals("60092")
|
|
|
|
- || param.getAbCode().equals("60093")
|
|
|
|
- || param.getAbCode().equals("60094")
|
|
|
|
- || param.getAbCode().equals("60095")
|
|
|
|
- || param.getAbCode().equals("60096")
|
|
|
|
- || param.getAbCode().equals("60097")
|
|
|
|
- || param.getAbCode().equals("60098")
|
|
|
|
- || param.getAbCode().equals("60111")
|
|
|
|
- || param.getAbCode().equals("60103")
|
|
|
|
- || param.getAbCode().equals("60104")
|
|
|
|
- || param.getAbCode().equals("60110")
|
|
|
|
- || param.getAbCode().equals("60150")
|
|
|
|
- ) {
|
|
|
|
- // 地域召回要做截取,再做融合排序
|
|
|
|
- removeDuplicate(rovRecallRank);
|
|
|
|
- rovRecallRank = rovRecallRank.size() <= sizeReturn
|
|
|
|
- ? rovRecallRank
|
|
|
|
- : rovRecallRank.subList(0, sizeReturn);
|
|
|
|
-
|
|
|
|
-
|
|
|
|
- // 补充不分地域小时数据
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, RegionHWithoutDupRecallStrategy.PUSH_FORM));
|
|
|
|
- // merge sim recall 和 return recall
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
|
|
|
|
- if (param.getAbCode().equals("60150")) {
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, ShareDeepRecallStrategy.PUSH_FORM));
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, ShareWidthRecallStrategy.PUSH_FORM));
|
|
|
|
- }
|
|
|
|
- removeDuplicate(rovRecallRank);
|
|
|
|
-
|
|
|
|
- // 融合排序
|
|
|
|
- List<String> videoIdKeys = rovRecallRank.stream()
|
|
|
|
- .map(t -> param.getRankKeyPrefix() + t.getVideoId())
|
|
|
|
- .collect(Collectors.toList());
|
|
|
|
- List<String> videoScores = redisTemplate.opsForValue().multiGet(videoIdKeys);
|
|
|
|
- if (CollectionUtils.isNotEmpty(videoScores)
|
|
|
|
- && videoScores.size() == rovRecallRank.size()) {
|
|
|
|
- for (int i = 0; i < videoScores.size(); i++) {
|
|
|
|
- rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
|
|
|
|
- }
|
|
|
|
- Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
|
|
|
|
- }
|
|
|
|
- } else {
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
|
|
|
|
- rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
|
|
|
|
- removeDuplicate(rovRecallRank);
|
|
|
|
-
|
|
|
|
- // 融合排序
|
|
|
|
- List<String> videoIdKeys = rovRecallRank.stream()
|
|
|
|
- .map(t -> param.getRankKeyPrefix() + t.getVideoId())
|
|
|
|
- .collect(Collectors.toList());
|
|
|
|
- List<String> videoScores = redisTemplate.opsForValue().multiGet(videoIdKeys);
|
|
|
|
- if (CollectionUtils.isNotEmpty(videoScores)
|
|
|
|
- && videoScores.size() == rovRecallRank.size()) {
|
|
|
|
- for (int i = 0; i < videoScores.size(); i++) {
|
|
|
|
- rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
|
|
|
|
- }
|
|
|
|
- rovRecallRank.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- return rovRecallRank;
|
|
|
|
- }
|
|
|
|
|
|
|
|
public void removeDuplicate(List<Video> rovRecallRank) {
|
|
public void removeDuplicate(List<Video> rovRecallRank) {
|
|
if (CollectionUtils.isNotEmpty(rovRecallRank)) {
|
|
if (CollectionUtils.isNotEmpty(rovRecallRank)) {
|
|
@@ -271,38 +167,6 @@ public class RankService {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
- protected ScoreParam convert(RankParam param) {
|
|
|
|
- ScoreParam scoreParam = new ScoreParam();
|
|
|
|
-
|
|
|
|
- scoreParam.setMid(param.getMid());
|
|
|
|
-
|
|
|
|
- // TODO hardcode 为了兼容写入逻辑
|
|
|
|
- RequestContext context = new RequestContext();
|
|
|
|
- context.setApptype(param.getAppType() + "");
|
|
|
|
-
|
|
|
|
- // TODO 地域转换
|
|
|
|
- context.setRegion(param.getProvince());
|
|
|
|
- context.setCity(param.getCity());
|
|
|
|
-
|
|
|
|
- Calendar calendar = Calendar.getInstance();
|
|
|
|
- context.setWeek((calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
|
|
|
|
- context.setDay(new SimpleDateFormat("yyyyMMdd").format(calendar.getTime()));
|
|
|
|
- context.setHour(new SimpleDateFormat("HH").format(calendar.getTime()));
|
|
|
|
-
|
|
|
|
- MachineInfo machineInfo = param.getMachineInfo();
|
|
|
|
- if (machineInfo != null) {
|
|
|
|
- context.setMachineinfo_brand(machineInfo.getBrand());
|
|
|
|
- context.setMachineinfo_model(machineInfo.getModel());
|
|
|
|
- context.setMachineinfo_platform(machineInfo.getPlatform());
|
|
|
|
- context.setMachineinfo_sdkversion(machineInfo.getSdkVersion());
|
|
|
|
- context.setMachineinfo_system(machineInfo.getSystem());
|
|
|
|
- context.setMachineinfo_wechatversion(machineInfo.getWechatVersion());
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- scoreParam.setRequestContext(context);
|
|
|
|
- return scoreParam;
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
|
|
public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
|
|
List<Video> quickFlowPoolVideos = extractAndSort(param, FlowPoolConstants.QUICK_PUSH_FORM);
|
|
List<Video> quickFlowPoolVideos = extractAndSort(param, FlowPoolConstants.QUICK_PUSH_FORM);
|
|
if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
|
|
if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
|
|
@@ -373,60 +237,15 @@ public class RankService {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
- public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
|
|
|
|
- if (CollectionUtils.isEmpty(rovRecallRank)) {
|
|
|
|
- if (param.getSize() < flowPoolRank.size()) {
|
|
|
|
- return new RankResult(flowPoolRank.subList(0, param.getSize()));
|
|
|
|
- } else {
|
|
|
|
- return new RankResult(flowPoolRank);
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- List<Video> result = new ArrayList<>();
|
|
|
|
- for (int i = 0; i < param.getTopK() && i < rovRecallRank.size(); i++) {
|
|
|
|
- result.add(rovRecallRank.get(i));
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- double flowPoolP = getFlowPoolP(param);
|
|
|
|
- int flowPoolIndex = 0;
|
|
|
|
- int rovPoolIndex = param.getTopK();
|
|
|
|
-
|
|
|
|
- for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
|
|
|
|
- double rand = RandomUtils.nextDouble(0, 1);
|
|
|
|
- if (rand < flowPoolP) {
|
|
|
|
- if (flowPoolIndex < flowPoolRank.size()) {
|
|
|
|
- result.add(flowPoolRank.get(flowPoolIndex++));
|
|
|
|
- } else {
|
|
|
|
- break;
|
|
|
|
- }
|
|
|
|
- } else {
|
|
|
|
- if (rovPoolIndex < rovRecallRank.size()) {
|
|
|
|
- result.add(rovRecallRank.get(rovPoolIndex++));
|
|
|
|
- } else {
|
|
|
|
- break;
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- if (rovPoolIndex >= rovRecallRank.size()) {
|
|
|
|
- for (int i = flowPoolIndex; i < flowPoolRank.size() && result.size() < param.getSize(); i++) {
|
|
|
|
- result.add(flowPoolRank.get(i));
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- if (flowPoolIndex >= flowPoolRank.size()) {
|
|
|
|
- for (int i = rovPoolIndex; i < rovRecallRank.size() && result.size() < param.getSize(); i++) {
|
|
|
|
- result.add(rovRecallRank.get(i));
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- return new RankResult(result);
|
|
|
|
- }
|
|
|
|
|
|
+ public abstract RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank);
|
|
|
|
|
|
- private boolean matchSpecialApp(int appId){
|
|
|
|
|
|
+ private boolean matchSpecialApp(int appId) {
|
|
Set<Integer> notSpecialApp = new HashSet<>(Arrays.asList(0, 4, 5));
|
|
Set<Integer> notSpecialApp = new HashSet<>(Arrays.asList(0, 4, 5));
|
|
- if (notSpecialApp.contains(appId)){
|
|
|
|
|
|
+ if (notSpecialApp.contains(appId)) {
|
|
// vlog 票圈视频 内容精选 不允许走特殊列表,即使配置了也无效。
|
|
// vlog 票圈视频 内容精选 不允许走特殊列表,即使配置了也无效。
|
|
return false;
|
|
return false;
|
|
}
|
|
}
|
|
- if (specialAppVid != null && specialAppVid.getOrDefault("app", new ArrayList<>()).contains((long) appId)){
|
|
|
|
|
|
+ if (specialAppVid != null && specialAppVid.getOrDefault("app", new ArrayList<>()).contains((long) appId)) {
|
|
log.info("This request hits a special logic in matchSpecialApp with appId={}", appId);
|
|
log.info("This request hits a special logic in matchSpecialApp with appId={}", appId);
|
|
return true;
|
|
return true;
|
|
}
|
|
}
|