Merge branch 'master' into feature/zhangbo_rank

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/Constant.java

@@ -19,4 +19,9 @@ public class Constant {
      * 供给流量池实验 648 random
      */
     public static final String SUPPLY_AB_CODE = "60600";
+    /**
+     * 供给流量池实验 648 random
+     */
+    public static final String SUPPLY_AB_CODE_ID= "648";
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/Video.java

@@ -38,5 +38,6 @@ public class Video {
     public double scoreRegion = 0.0D;
     public Map<String, Double> scoresMap = new HashMap<>();
     public Map<String, List<String>> pushFromIndex = new HashMap<>();
+    public Map<String, Integer> pushFromRank = new HashMap<>();
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -203,6 +203,9 @@ public class RecommendService {
                 map.put("score", String.valueOf(v.getScore()));
                 map.put("scoresMap", JSONUtils.toJson(v.getScoresMap()));
 
+                map.put("pushFromRank", JSONUtils.toJson(v.getPushFromRank()));
+                map.put("abExpCode", JSONUtils.toJson(param.getAbExpCodes()));
+
                 return map;
 
             }).collect(Collectors.toList());
@@ -343,7 +346,9 @@ public class RecommendService {
         // 流量池分发实验组划分
         int flowPoolIdChoice = flowPoolIds.get(RandomUtils.nextInt(0, flowPoolIds.size()));
         Map<String, List<Integer>> flowPoolConfig = flowPoolConfigService.getFlowPoolConfig();
-        if (Objects.equals(Constant.SUPPLY_AB_CODE, param.getAbCode())) {
+        Set<String> abExpCodes = param.getAbExpCodes();
+        if (Objects.nonNull(abExpCodes) && abExpCodes.contains(Constant.SUPPLY_AB_CODE_ID)) {
+//        if (Objects.equals(Constant.SUPPLY_AB_CODE, param.getAbCode())) {
             List<Integer> supplyFlowPoolIdList = flowPoolConfig.get(FlowPoolConstants.SUPPLY_FLOW_SET_LEVEL);
             if (Objects.nonNull(supplyFlowPoolIdList) && !supplyFlowPoolIdList.isEmpty()) {
                 flowPoolIdChoice = supplyFlowPoolIdList.get(0);
@@ -359,7 +364,6 @@ public class RecommendService {
                 }
             }
         }
-
         // @desc 新的流量池分发实验组划分，每个尾号不同策略分组。 @time 20240318 @author 张博
         int lastDigit = RandomUtils.nextInt(0, 10);
         param.setLastDigit(lastDigit);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithLevelFilterService.java

@@ -44,12 +44,13 @@ public class FlowPoolWithLevelFilterService extends AbstractFilterService {
 
         Map<Long, Integer> distributeCountMap;
         //供给流量池实验
-        if (Objects.equals(Constant.SUPPLY_AB_CODE, param.getAbCode())) {
+        Set<String> abExpCodes = param.getAbExpCodes();
+        if (Objects.nonNull(abExpCodes) && abExpCodes.contains(Constant.SUPPLY_AB_CODE_ID)) {
+//        if (Objects.equals(Constant.SUPPLY_AB_CODE, param.getAbCode())) {
             distributeCountMap = flowPoolService.getSupplyDistributeCountWithLevel(flowPoolMap);
         } else {
             distributeCountMap = flowPoolService.getDistributeCountWithLevel(flowPoolMap);
         }
-
 //        Map<Long, Integer> distributeCountMap = flowPoolService.getDistributeCountWithLevel(flowPoolMap);
 
         List<Long> remainVideoIds = new ArrayList<>();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/AppletVideoStatusStrategy.java

@@ -89,7 +89,8 @@ public class AppletVideoStatusStrategy implements FilterStrategy {
                 // 数据库有数据
                 for (VideoAppTypeStatus videoAppTypeStatus : videoAppTypeStatusList) {
                     activeStatusMap.put(videoAppTypeStatus.getVideoId(), videoAppTypeStatus.getVideoStatus());
-                    updateRedisStatus.put(String.format(videoAppTypeStatusKeyFormat, appType, videoAppTypeStatus.getVideoId()), "1");
+                    updateRedisStatus.put(String.format(videoAppTypeStatusKeyFormat, appType,
+                            videoAppTypeStatus.getVideoId()), videoAppTypeStatus.getVideoStatus().toString());
                     cacheMissVideoIdSet.remove(videoAppTypeStatus.getVideoId());
                 }
             }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/BlacklistContainer.java

@@ -201,8 +201,8 @@ public class BlacklistContainer {
 
         return videoIds.stream().filter(videoId -> {
             if (videoTagAnyMatch(videoId, tagIdSet)) {
-                LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}, mid={}, usedScene={}, appType={}",
-                        uid, userType, videoId, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
+                // LOG.info("用户 {} 在因命中 {} 移除对应的视频ID {}: 请求参数为: hotSceneType={}, cityCode={}, clientIP={}, mid={}, usedScene={}, appType={}",
+                //         uid, userType, videoId, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
                 return false;
             }
             return true;
@@ -212,15 +212,15 @@ public class BlacklistContainer {
     public String matchUserBlacklistTypeEnum(String uid, Long hotSceneType, String cityCode, String clientIP, String mid,
                                              String usedScene, Integer appType) {
         try {
-            LOG.info("计算用户黑名单类型，判断参数: uid={}, hotSceneType={}, cityCode={}, clientIP={}, mid={}, usedScene={}, appType={}",
-                    uid, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
+            // LOG.info("计算用户黑名单类型，判断参数: uid={}, hotSceneType={}, cityCode={}, clientIP={}, mid={}, usedScene={}, appType={}",
+            //         uid, hotSceneType, cityCode, clientIP, mid, usedScene, appType);
             if (StringUtils.isNotBlank(uid)) {
                 String key = this.calcUserRedisKey(uid);
                 Map<String, String> uidBlacklistMap = blacklistCache.get(key);
                 if (uidBlacklistMap.containsKey(uid)) {
                     String userType = uidBlacklistMap.get(uid);
                     this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "UID", mid, usedScene, appType);
-                    LOG.info("用户 {} 在UID黑名单中命中 {}", uid, userType);
+                    // LOG.info("用户 {} 在UID黑名单中命中 {}", uid, userType);
                     return userType;
                 }
             }
@@ -230,7 +230,7 @@ public class BlacklistContainer {
                 if (ipBlacklistMap.containsKey(clientIP)) {
                     String userType = ipBlacklistMap.get(clientIP);
                     this.filterLogUpload(uid, cityCode, hotSceneType, clientIP, userType, "IP", mid, usedScene, appType);
-                    LOG.info("用户 {} 在IP黑名单中命中 {}, 参数为: clientIP为: {}", uid, userType, clientIP);
+                    // LOG.info("用户 {} 在IP黑名单中命中 {}, 参数为: clientIP为: {}", uid, userType, clientIP);
                     return userType;
                 }
             }
@@ -263,7 +263,7 @@ public class BlacklistContainer {
         if (StringUtils.isNotBlank(cityCode) && Objects.nonNull(appType)) {
             GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(cityCode + CONNECTOR_STR + appType);
             if (Objects.nonNull(userConfig) && userConfig.isExcludeHotSceneType(hotSceneType)) {
-                LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: appType = {}, cityCode={}, hotSceneType={}", uid, userConfig.fullUserType, appType, cityCode, hotSceneType);
+                // LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: appType = {}, cityCode={}, hotSceneType={}", uid, userConfig.fullUserType, appType, cityCode, hotSceneType);
                 return userConfig.getFullUserType();
             }
         }
@@ -272,7 +272,7 @@ public class BlacklistContainer {
         if (Objects.nonNull(appType)) {
             GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(appType.toString());
             if (Objects.nonNull(userConfig) && userConfig.isExcludeHotSceneType(hotSceneType)) {
-                LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: appType={}, hotSceneType={}", uid, userConfig.fullUserType, appType, hotSceneType);
+                // LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: appType={}, hotSceneType={}", uid, userConfig.fullUserType, appType, hotSceneType);
                 return userConfig.getFullUserType();
             }
         }
@@ -281,7 +281,7 @@ public class BlacklistContainer {
         if (StringUtils.isNotBlank(cityCode)) {
             GeneralizationUserConfig userConfig = generalizationUserConditionConfig.get(cityCode);
             if (Objects.nonNull(userConfig) && userConfig.isExcludeHotSceneType(hotSceneType)) {
-                LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: cityCode={}, hotSceneType={}", uid, userConfig.fullUserType, cityCode, hotSceneType);
+                // LOG.info("用户 {} 在泛化用户规则中命中: {}, 参数为: cityCode={}, hotSceneType={}", uid, userConfig.fullUserType, cityCode, hotSceneType);
                 return userConfig.getFullUserType();
             }
         }
@@ -329,7 +329,7 @@ public class BlacklistContainer {
             String key = userType + CONNECTOR_STR + appType;
             TagFilterConfig tagFilterConfig = tagFilterConfigMap.get(key);
             if (Objects.nonNull(tagFilterConfig)) {
-                LOG.info("命中过滤标签配置: {} == {}", key, tagFilterConfig.getRecommendExcludeTag());
+                // LOG.info("命中过滤标签配置: {} == {}", key, tagFilterConfig.getRecommendExcludeTag());
                 return tagFilterConfig.getRecommendExcludeTag();
             }
         }
@@ -337,7 +337,7 @@ public class BlacklistContainer {
         if (StringUtils.isNotBlank(userType)) {
             TagFilterConfig tagFilterConfig = tagFilterConfigMap.get(userType);
             if (Objects.nonNull(tagFilterConfig)) {
-                LOG.info("命中过滤标签配置: {} == {}", userType, tagFilterConfig.getRecommendExcludeTag());
+                // LOG.info("命中过滤标签配置: {} == {}", userType, tagFilterConfig.getRecommendExcludeTag());
                 return tagFilterConfig.getRecommendExcludeTag();
             }
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/flowpool/FlowPoolConstants.java

@@ -10,7 +10,6 @@ public class FlowPoolConstants {
 
     public static final String PUSH_FORM = "flow_pool";
     public static final String QUICK_PUSH_FORM = "quick_flow_pool";
-    public static final String SUPPLY_PUSH_FORM = "supply_flow_pool";
 
     public static final String KEY_WITH_LEVEL_FORMAT = "flow:pool:level:item:%s:%s";
     public static final String KEY_WITH_LEVEL_SUPPLY_FORMAT = "flow:pool:level:item:supply:%s:%s";

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -117,6 +117,7 @@ public class RankService {
         if (CollectionUtils.isEmpty(param.getRecallResult().getData())) {
             return;
         }
+        // pushFromIndex
         Map<Long, Set<String>> videoIdPushFromMap = new HashMap<>();
         for (RecallResult.RecallData data : param.getRecallResult().getData()) {
             if (CollectionUtils.isEmpty(data.getVideos())) {
@@ -127,6 +128,11 @@ public class RankService {
                 pushFromSet.add(data.getPushFrom());
             }
         }
+
+        // pushFromRank
+        Map<String, List<Video>> pushFromVideosMap = param.getRecallResult().getData().stream()
+                .collect(Collectors.toMap(RecallResult.RecallData::getPushFrom, result -> result.getVideos() == null
+                        ? new ArrayList<>() : result.getVideos(), (v1, v2) -> v2));
         for (RecallResult.RecallData data : param.getRecallResult().getData()) {
             if (CollectionUtils.isEmpty(data.getVideos())) {
                 continue;
@@ -134,12 +140,29 @@ public class RankService {
             for (Video video : data.getVideos()) {
                 Set<String> pushFromSet = videoIdPushFromMap.get(video.getVideoId());
                 if (pushFromSet != null && pushFromSet.size() > 0) {
-                    Map<String, List<String>> map = new HashMap<>();
-                    pushFromSet.forEach(p -> map.computeIfAbsent(p, k -> new ArrayList<>()));
-                    video.setPushFromIndex(map);
+                    Map<String, List<String>> pushFromIndex = new HashMap<>();
+                    Map<String, Integer> pushFromRank = new HashMap<>();
+                    pushFromSet.forEach(p -> {
+
+                        pushFromIndex.computeIfAbsent(p, k -> new ArrayList<>());
+
+                        List<Video> videos = pushFromVideosMap.get(p);
+                        if (CollectionUtils.isNotEmpty(videos)) {
+                            for (int i = 0; i < videos.size(); i++) {
+                                if (Objects.equals(videos.get(i).getVideoId(), video.getVideoId())) {
+                                    pushFromRank.put(p, i + 1);
+                                    break;
+                                }
+                            }
+                        }
+
+                    });
+                    video.setPushFromIndex(pushFromIndex);
+                    video.setPushFromRank(pushFromRank);
                 }
             }
         }
+
     }
 
     public void rankFilter(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
@@ -324,14 +347,7 @@ public class RankService {
         if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
             return quickFlowPoolVideos;
         } else {
-//            return extractAndSort(param, FlowPoolConstants.PUSH_FORM);
-            //供给流量池 新增pushForm；由于供给流量池占比小，先判断 正常流量池
-            List<Video> flowPoolVideos = extractAndSort(param, FlowPoolConstants.PUSH_FORM);
-            if (CollectionUtils.isNotEmpty(flowPoolVideos)) {
-                return flowPoolVideos;
-            } else {
-                return extractAndSort(param, FlowPoolConstants.SUPPLY_PUSH_FORM);
-            }
+            return extractAndSort(param, FlowPoolConstants.PUSH_FORM);
         }
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4ShareDeepAndWidth.java

@@ -163,6 +163,8 @@ public class RankStrategy4ShareDeepAndWidth extends RankStrategy4RegionMergeMode
 
             if (sumShareCount > 30) {
                 score += a * daySharedepthMaxAvg + b * daySharewidthMaxAvg;
+                item.getScoresMap().put("dayShareDepthMaxAvg", daySharedepthMaxAvg);
+                item.getScoresMap().put("dayShareWidthMaxAvg", daySharewidthMaxAvg);
             }
 
             Video video = item.getVideo();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -172,7 +172,9 @@ public class RecallService implements ApplicationContextAware {
         }
         // 命中用户黑名单不走流量池
         if (!hitUserBlacklist || !isInBlacklist) {
-            if (Objects.equals(Constant.SUPPLY_AB_CODE, abCode)) {
+            Set<String> abExpCodes = param.getAbExpCodes();
+            if (Objects.nonNull(abExpCodes) && abExpCodes.contains(Constant.SUPPLY_AB_CODE_ID)) {
+//            if (Objects.equals(Constant.SUPPLY_AB_CODE, abCode)) {
                 // 供给流量池策略 648 实验 random
                 strategies.add(strategyMap.get(FlowPoolWithLevelSupplyRecallStrategy.class.getSimpleName()));
                 //2:通过“流量池标记”控制“流量池召回子策略” 其中有9组会走EXPERIMENTAL_FLOW_SET_LEVEL 有1组会走EXPERIMENTAL_FLOW_SET_LEVEL_SCORE

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithLevelSupplyRecallStrategy.java

@@ -93,6 +93,6 @@ public class FlowPoolWithLevelSupplyRecallStrategy extends AbstractFlowPoolWithL
 
     @Override
     public String pushFrom() {
-        return FlowPoolConstants.SUPPLY_PUSH_FORM;
+        return FlowPoolConstants.PUSH_FORM;
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/ShareDeepRecallStrategy.java

@@ -56,13 +56,15 @@ public class ShareDeepRecallStrategy implements RecallStrategy {
         FilterResult filterResult = filterService.filter(filterParam);
         List<Video> videosResult = new ArrayList<>();
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
-            filterResult.getVideoIds().stream().limit(Integer.parseInt(shareDeepRecallStrategyRecallNumberAfterFilter)).forEach(vid -> {
-                Video video = new Video();
-                video.setVideoId(vid);
-                video.setAbCode(param.getAbCode());
-                video.setPushFrom(pushFrom());
-                videosResult.add(video);
-            });
+            filterResult.getVideoIds().stream()
+                    .sorted(Comparator.comparingDouble(vid -> videoMap.getOrDefault(vid, 0.0)).reversed())
+                    .limit(Integer.parseInt(shareDeepRecallStrategyRecallNumberAfterFilter)).forEach(vid -> {
+                        Video video = new Video();
+                        video.setVideoId(vid);
+                        video.setAbCode(param.getAbCode());
+                        video.setPushFrom(pushFrom());
+                        videosResult.add(video);
+                    });
         }
         return videosResult;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/ShareWidthRecallStrategy.java

@@ -56,13 +56,15 @@ public class ShareWidthRecallStrategy implements RecallStrategy {
         FilterResult filterResult = filterService.filter(filterParam);
         List<Video> videosResult = new ArrayList<>();
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
-            filterResult.getVideoIds().stream().limit(Integer.parseInt(shareDeepRecallStrategyRecallNumberAfterFilter)).forEach(vid -> {
-                Video video = new Video();
-                video.setVideoId(vid);
-                video.setAbCode(param.getAbCode());
-                video.setPushFrom(pushFrom());
-                videosResult.add(video);
-            });
+            filterResult.getVideoIds().stream()
+                    .sorted(Comparator.comparingDouble(vid -> videoMap.getOrDefault(vid, 0.0)).reversed())
+                    .limit(Integer.parseInt(shareDeepRecallStrategyRecallNumberAfterFilter)).forEach(vid -> {
+                        Video video = new Video();
+                        video.setVideoId(vid);
+                        video.setAbCode(param.getAbCode());
+                        video.setPushFrom(pushFrom());
+                        videosResult.add(video);
+                    });
         }
         return videosResult;
     }