Merge branch 'feature_refactor' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/RecommendParam.java

@@ -19,16 +19,10 @@ import java.util.Set;
 public class RecommendParam {
     private int topK;
     private double flowPoolP;
-    private String abCode;
     private String ruleKey;
     private String dataKey;
 
-    // hardcode   兼容旧服务的518实验
-    private String hDataKey;
-    private String hRuleKey;
-
     private int flowPoolId;
-    private int lastDigit;
     private String flowPoolAbtestGroup;
     private String rankKeyPrefix;
     private int appType;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/Video.java

@@ -16,7 +16,6 @@ public class Video {
     private double rovScore;
     private double sortScore;
     private String pushFrom;
-    private String abCode;
 
     // 流量池相关 start
     private String flowPool;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -53,12 +53,6 @@ public class RecommendService {
     @Qualifier("redisTemplate")
     private RedisTemplate<String, String> redisTemplate;
 
-
-    private final Map<Integer, String> ab_initial_config_map = new HashMap<>();
-
-    @ApolloJsonValue("${ab_exp_code:{}}")
-    private Map<String, Map<String, String>> abExpCodeMap;
-
     @ApolloJsonValue("${app_ab_code:{}}")
     private Map<String, Map<String, Map<String, String>>> appAbCodeMap;
 
@@ -156,7 +150,6 @@ public class RecommendService {
             updateCache(request, param, videos);
             long updateCacheTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
             timerLogMapTL.get().put("updateCacheTime", updateCacheTime);
-            timerLogMapTL.get().put("abCodeFake", param.getAbCode());
             timerLogService.log(timerLogMapTL.get());
             logStatisticsInfo(request, param, videos);
             // 更新position
@@ -174,7 +167,6 @@ public class RecommendService {
                 vps.add(VideoProto.newBuilder()
                         .setPosition(i + 1)
                         .setPushFrom(Strings.nullToEmpty(videos.get(i).getPushFrom()))
-                        .setAbCode(Strings.nullToEmpty(videos.get(i).getAbCode()))
                         .setVideoId(videos.get(i).getVideoId())
                         .setRovScore(videos.get(i).getRovScore())
                         .setSortScore(videos.get(i).getSortScore())
@@ -299,7 +291,6 @@ public class RecommendService {
                 .setResult(Result.newBuilder().setCode(1).setMessage("success"))
                 .addAllVideo(CommonCollectionUtils.toList(videos, v -> VideoProto.newBuilder()
                         .setPushFrom(Strings.nullToEmpty(v.getPushFrom()))
-                        .setAbCode(Strings.nullToEmpty(v.getAbCode()))
                         .setVideoId(v.getVideoId())
                         .setRovScore(v.getRovScore())
                         .build()))
@@ -321,11 +312,8 @@ public class RecommendService {
         param.setCity(request.getCity());
 
 
-        String abInitialConfig = ab_initial_config_map.getOrDefault(request.getAppType(), "095-1");
-        Map<String, String> abExpCode = abExpCodeMap.getOrDefault(abInitialConfig, Collections.emptyMap());
-        param.setAbCode(abExpCode.get("ab_code"));
-        param.setRuleKey(abExpCode.get("rule_key"));
-        param.setDataKey(abExpCode.get("data_key"));
+        param.setRuleKey("rule66");
+        param.setDataKey("data66");
         param.setRankKeyPrefix("rank:score1:");
 
         // 实验配置
@@ -346,37 +334,6 @@ public class RecommendService {
             } else if (abExpCodes.contains("339")) {
                 param.setFlowPoolP(0);
             }
-
-            for (Map.Entry<String, Map<String, String>> entry : abExpCodeMap.entrySet()) {
-                if (abExpCodes.contains(entry.getKey())) {
-                    param.setAbCode(entry.getValue().get("ab_code"));
-                    param.setRuleKey(entry.getValue().get("rule_key"));
-                    param.setDataKey(entry.getValue().get("data_key"));
-
-                    param.setHDataKey(entry.getValue().get("h_data_key"));
-                    param.setHRuleKey(entry.getValue().get("h_rule_key"));
-
-                    param.setRankKeyPrefix(StringUtils.isNotBlank(entry.getValue().get("rank_key_prefix"))
-                            ? entry.getValue().get("rank_key_prefix")
-                            : "rank:score1:");
-                    break;
-                }
-            }
-        }
-
-        // app实验组
-        if (request.getEventIdCount() > 0) {
-            if (recommendType == 0) {
-                String eventId = request.getEventIdMap().get("10003");
-                Map<String, Map<String, String>> appAbCode = appAbCodeMap.get("10003");
-                if (StringUtils.isNotBlank(eventId)
-                        && appAbCode != null
-                        && appAbCode.containsKey(eventId)) {
-                    param.setAbCode(appAbCode.get(eventId).get("ab_code"));
-                    param.setRuleKey(appAbCode.get(eventId).get("rule_key"));
-                    param.setDataKey(appAbCode.get(eventId).get("data_key"));
-                }
-            }
         }
 
         // 流量池分发实验组划分
@@ -390,9 +347,6 @@ public class RecommendService {
                 param.setFlowPoolAbtestGroup(entry.getKey());
             }
         }
-        // @desc 新的流量池分发实验组划分，每个尾号不同策略分组。 @time 20240318 @author 张博
-        int lastDigit = RandomUtils.nextInt(0, 10);
-        param.setLastDigit(lastDigit);
 
         // 风险过滤
         if (riskVideoFilterSwitch) {
@@ -450,14 +404,6 @@ public class RecommendService {
         if (param.getSize() < rankResult.getVideos().size()) {
             videos = rankResult.getVideos().subList(0, param.getSize());
         }
-
-
-        // mark video in flow pool
-        // TODO 业务上有什么用处
-        // 需要被标记的视频满足什么条件
-        // 逻辑已经废弃
-        // markFlowPoolVideo(param, videos);
-
         return videos;
     }
 
@@ -466,15 +412,11 @@ public class RecommendService {
         recallParam.setAppType(param.getAppType());
         // hard code 算法实验配置化之前，复用abcode做AB验证
         // note 避免非实验产品被覆盖
-        recallParam.setAbCode(param.getAbCode());
         recallParam.setRuleKey(param.getRuleKey());
         recallParam.setDataKey(param.getDataKey());
-        recallParam.setHRuleKey(param.getHRuleKey());
-        recallParam.setHDataKey(param.getHDataKey());
 
         recallParam.setVideoId(param.getVideoId());
         recallParam.setFlowPoolAbtestGroup(param.getFlowPoolAbtestGroup());
-        recallParam.setLastDigit(param.getLastDigit());
 
         String provinceCode = StringUtils.isNotBlank(param.getProvinceCode())
                 ? param.getProvinceCode()
@@ -494,7 +436,6 @@ public class RecommendService {
         recallParam.setCategoryId(param.getCategoryId());
 
         recallParam.setCityCode(param.getCityCode());
-        // recallParam.setAbCode(param.getAbCode());
 
         recallParam.setHotSceneType(param.getHotSceneType());
         recallParam.setClientIp(param.getClientIp());
@@ -508,9 +449,6 @@ public class RecommendService {
     public RankParam convertToRankParam(RecommendParam param, RecallResult recallResult) {
         RankParam rankParam = new RankParam();
         rankParam.setRecallResult(recallResult);
-        // hard code 算法实验配置化之前，复用abcode做AB验证
-        // note 避免非实验产品被覆盖
-        rankParam.setAbCode(param.getAbCode());
         rankParam.setRankKeyPrefix(param.getRankKeyPrefix());
 
         rankParam.setSize(param.getSize());

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterParam.java

@@ -22,8 +22,6 @@ public class FilterParam {
     private String cityCode;
     private Set<String> abExpCodes;
 
-    private String abCode;
-
     private Long hotSceneType;
 
     private String clientIp;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankParam.java

@@ -17,7 +17,6 @@ public class RankParam {
     private int topK;
     private String rankKeyPrefix;
     private double flowPoolP;
-    private String abCode;
     private int appType;
     private boolean specialRecommend;
     private String mid;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -1,49 +1,65 @@
 package com.tzld.piaoquan.recommend.server.service.rank;
 
-import com.aliyun.odps.utils.StringUtils;
+import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
 import com.tzld.piaoquan.recommend.server.service.rank.strategy.*;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.*;
+
 @Service
 @Slf4j
 public class RankRouter {
+
+    private LinkedHashMap<String, RankService> strategyMap;
+
+    private static final Map<String, Class<? extends RankService>> STRATEGY_CLASSES = new HashMap<>();
+
+    static {
+        STRATEGY_CLASSES.put("536", RankStrategy4RegionMergeModelV536.class);
+        STRATEGY_CLASSES.put("562", RankStrategy4RegionMergeModelV562.class);
+        STRATEGY_CLASSES.put("563", RankStrategy4RegionMergeModelV563.class);
+        STRATEGY_CLASSES.put("564", RankStrategy4RegionMergeModelV564.class);
+        STRATEGY_CLASSES.put("567", RankStrategy4RegionMergeModelV567.class);
+        STRATEGY_CLASSES.put("569", RankStrategy4RegionMergeModelV569.class);
+    }
+
     @Autowired
-    private RankStrategy4RegionMergeModelV536 rankStrategy4RegionMergeModelV536;
-    @Autowired
-    private RankStrategy4RegionMergeModelV562 rankStrategy4RegionMergeModelV562;
-    @Autowired
-    private RankStrategy4RegionMergeModelV563 rankStrategy4RegionMergeModelV563;
-    @Autowired
-    private RankStrategy4RegionMergeModelV564 rankStrategy4RegionMergeModelV564;
-    @Autowired
-    private RankStrategy4RegionMergeModelV567 rankStrategy4RegionMergeModelV567;
-    @Autowired
-    private RankStrategy4RegionMergeModelV569 rankStrategy4RegionMergeModelV569;
+    public RankRouter(ServiceBeanFactory serviceBeanFactory) {
+        this.strategyMap = new LinkedHashMap<>();
+        for (Map.Entry<String, Class<? extends RankService>> entry : STRATEGY_CLASSES.entrySet()) {
+            try {
+                this.strategyMap.put(entry.getKey(), serviceBeanFactory.getBean(entry.getValue()));
+            } catch (Exception e) {
+                log.error("Failed to initialize strategy: {}", entry.getKey(), e);
+            }
+        }
+    }
 
     public RankResult rank(RankParam param) {
-        String abCode = param.getAbCode();
-        if (StringUtils.isBlank(abCode)) {
-            return rankStrategy4RegionMergeModelV536.rank(param);
+        Set<String> abExpCodes = param.getAbExpCodes();
+        if (CollectionUtils.isNotEmpty(abExpCodes)) {
+            for (Map.Entry<String, RankService> entry : strategyMap.entrySet()) {
+                if (abExpCodes.contains(entry.getKey())) {
+                    //log.info("rank strategies {} {}", entry.getKey(), entry.getValue().getClass().getSimpleName());
+                    return entry.getValue().rank(param);
+                }
+            }
         }
-        // 536 562 563 564 567 569
-        switch (abCode) {
-
-            case "60112": // 562
-                return rankStrategy4RegionMergeModelV562.rank(param);
-            case "60113": // 563
-                return rankStrategy4RegionMergeModelV563.rank(param); // 60113 CF召回 doing
-            case "60114": // 564
-                return rankStrategy4RegionMergeModelV564.rank(param); // 60114 排序r-cnt实验 doing
-            case "60117": // 567
-                return rankStrategy4RegionMergeModelV567.rank(param); // 60117 cf叠加r-cnt模型 doing
-            case "60119": // 569
-                return rankStrategy4RegionMergeModelV569.rank(param); // rov fm模型 60119 实验关闭
-            case "60121": // 536
-            default: // base 536
-                return rankStrategy4RegionMergeModelV536.rank(param); // vor统计量 doing
+
+
+        Optional<RankService> defaultStrategy = strategyMap.values().stream().findFirst();
+        if (defaultStrategy.isPresent()) {
+            return defaultStrategy.get().rank(param);
+        } else {
+            log.error("No available rank strategy found.");
+            return null;
         }
     }
+
 }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelBasic.java

@@ -63,8 +63,15 @@ public abstract class RankStrategy4RegionMergeModelBasic extends RankService {
         }
 
         //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+        Set<String> abExpCodes = param.getAbExpCodes();
+        Map<String, Map<String, String>> rulesMap = Collections.emptyMap();
+        for (Map.Entry<String, Map<String, Map<String, String>>> entry : this.filterRules.entrySet()) {
+            if (abExpCodes.contains(entry.getKey())) {
+                rulesMap = entry.getValue();
+                break;
+            }
+        }
+
 
         //3 标签读取
         if (rulesMap != null && !rulesMap.isEmpty()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/FilterParamFactory.java

@@ -23,7 +23,6 @@ public class FilterParamFactory {
         filterParam.setRegionCode(param.getRegionCode());
         filterParam.setAbExpCodes(param.getAbExpCodes());
         filterParam.setCityCode(param.getCityCode());
-        filterParam.setAbCode(param.getAbCode());
         filterParam.setHotSceneType(param.getHotSceneType());
         filterParam.setClientIp(param.getClientIp());
         return filterParam;
@@ -41,7 +40,6 @@ public class FilterParamFactory {
         filterParam.setRegionCode(param.getRegionCode());
         filterParam.setAbExpCodes(param.getAbExpCodes());
         filterParam.setCityCode(param.getCityCode());
-        filterParam.setAbCode(param.getAbCode());
         filterParam.setHotSceneType(param.getHotSceneType());
         filterParam.setClientIp(param.getClientIp());
         return filterParam;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallParam.java

@@ -19,14 +19,8 @@ public class RecallParam {
     private String ruleKey;
     private String province;
 
-    // hardcode   兼容旧服务的518实验
-    private String hDataKey;
-    private String hRuleKey;
-
-    private String abCode;
     private int size;
     private String flowPoolAbtestGroup;
-    private int lastDigit;
     private Long videoId;
     private String uid;
     private boolean specialRecommend;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -6,7 +6,10 @@ import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeansException;
 import org.springframework.context.ApplicationContext;
@@ -44,7 +47,8 @@ public class RecallService implements ApplicationContextAware {
 
     public RecallResult recall(RecallParam param) {
         List<RecallStrategy> strategies = getRecallStrategy(param);
-        //log.info("strategies {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies,o -> o.getClass().getSimpleName())));
+//        log.info("recall strategies {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies,
+//                o -> o.getClass().getSimpleName())));
         CountDownLatch cdl = new CountDownLatch(strategies.size());
         List<Future<RecallResult.RecallData>> recallResultFutures = new ArrayList<>();
         for (final RecallStrategy strategy : strategies) {
@@ -52,6 +56,7 @@ public class RecallService implements ApplicationContextAware {
                 List<Video> result = Collections.emptyList();
                 try {
                     result = strategy.recall(param);
+                    //log.info("recall result {} {}", strategy.getClass().getSimpleName(), JSONUtils.toJson(result));
                 } catch (Throwable e) {
                     log.error("recall error {}", strategy.getClass().getSimpleName(), e);
                 }
@@ -87,7 +92,6 @@ public class RecallService implements ApplicationContextAware {
             return strategies;
         }
 
-        String abCode = param.getAbCode();
         //1：通过“产品”控制“召回子策略”. 票圈美好祝福与内部tab只走祝福召回。APP只走固定列表。特殊配置的app只有固定召回列表。
         if (this.matchSpecialApp(param.getAppType())) {
             strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV5Hand.class.getSimpleName()));
@@ -101,32 +105,20 @@ public class RecallService implements ApplicationContextAware {
             strategies.add(strategyMap.get(AppFallbackRecallStrategy.class.getSimpleName()));
             return strategies;
         }
-        switch (abCode) {
-//            60112  562
-//            60113  563
-//            60114  564
-//            60117  567
-//            60119  569
-//            60121  536
-            case "60121": // 536
-            case "60112": // 562
-            case "60114": // 564
-            case "60119": // 569
-                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
-                strategies.addAll(getRegionRecallStrategy(param));
-                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
-                break;
-            case "60113": // 563
-            case "60117": // 567
-                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
-                strategies.addAll(getRegionRecallStrategy(param));
-                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+
+        strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
+        strategies.addAll(getRegionRecallStrategy(param));
+        strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+
+        Set<String> abExpCodes = param.getAbExpCodes();
+        if (CollectionUtils.isNotEmpty(abExpCodes)) {
+            if (abExpCodes.contains("563")
+                    || abExpCodes.contains("567")) {
                 strategies.add(strategyMap.get(SceneCFRovnRecallStrategy.class.getSimpleName()));
                 strategies.add(strategyMap.get(SceneCFRosnRecallStrategy.class.getSimpleName()));
-                break;
-            default:
-                strategies.addAll(getRegionRecallStrategy(param));
+            }
         }
+
         // 命中用户黑名单不走流量池
         if (!param.isRiskUser()) {
             if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL)) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractFlowPoolWithLevelRecallStrategy.java

@@ -53,7 +53,6 @@ public abstract class AbstractFlowPoolWithLevelRecallStrategy implements RecallS
             filterResult.getVideoIds().stream().forEach(vid -> {
                 Video recallData = new Video();
                 recallData.setVideoId(vid);
-                recallData.setAbCode(param.getAbCode());
                 recallData.setRovScore(RandomUtils.nextDouble(0, 100));
                 recallData.setPushFrom(pushFrom());
                 recallData.setFlowPool(videoFlowPoolMap.get(vid));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractFlowPoolWithLevelScoreRecallStrategy.java

@@ -54,7 +54,6 @@ public abstract class AbstractFlowPoolWithLevelScoreRecallStrategy implements Re
             filterResult.getVideoIds().stream().forEach(vid -> {
                 Video video = new Video();
                 video.setVideoId(vid);
-                video.setAbCode(param.getAbCode());
                 video.setRovScore(videoScoreMap.get(vid));
                 video.setPushFrom(pushFrom());
                 video.setFlowPool(videoFlowPoolMap.get(vid));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractRegionRecallStrategy.java

@@ -34,8 +34,6 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
     @Autowired
     @Qualifier("redisTemplate")
     protected RedisTemplate<String, String> redisTemplate;
-    @ApolloJsonValue("${region.recall.return.size:{}}")
-    protected Map<String, Map<String, Integer>> regionRecallReturnSize;
 
     @Autowired
     protected RegionFilterService filterService;
@@ -141,7 +139,6 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
                 filterResult.getVideoIds().stream().forEach(vid -> {
                     Video video = new Video();
                     video.setVideoId(vid);
-                    video.setAbCode(param.getAbCode());
                     video.setRovScore(videoMap.get(vid));
                     video.setPushFrom(pushFrom());
                     video.setLastVideoKey(lastVideoKey);
@@ -162,16 +159,7 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
             redisTemplate.opsForValue().set(lastVideoKey, lastVideoId, 2, TimeUnit.HOURS);
         }
 
-        // @zhangbo 增加不同召回子策略的返回数量-阿波罗：region.recall.return.size
-        String abCode = param.getAbCode();
-        String pushFrom = this.pushFrom();
         int sizeReturn = param.getSize();
-        if (abCode != null && this.regionRecallReturnSize != null &&
-                this.regionRecallReturnSize.containsKey(abCode) &&
-                this.regionRecallReturnSize.get(abCode) != null) {
-            sizeReturn = this.regionRecallReturnSize.get(abCode).getOrDefault(pushFrom, sizeReturn);
-        }
-
         return results.subList(0, results.size() < sizeReturn ? results.size() : sizeReturn);
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractVideoRecallStrategy.java

@@ -62,7 +62,6 @@ public abstract class AbstractVideoRecallStrategy implements RecallStrategy {
             filterResult.getVideoIds().stream().forEach(vid -> {
                 Video recallData = new Video();
                 recallData.setVideoId(vid);
-                recallData.setAbCode(param.getAbCode());
                 recallData.setRovScore(videoScoreMap.get(vid));
                 recallData.setPushFrom(pushFrom());
                 recallData.setFlowPool("");

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AppFallbackRecallStrategy.java

@@ -62,7 +62,6 @@ public class AppFallbackRecallStrategy implements RecallStrategy {
                     .forEach(vid -> {
                         Video video = new Video();
                         video.setVideoId(vid);
-                        video.setAbCode(param.getAbCode());
                         video.setPushFrom(pushFrom());
                         videosResult.add(video);
                     });

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/BlessRecallStrategy.java

@@ -65,7 +65,6 @@ public class BlessRecallStrategy implements RecallStrategy {
             filterResult.getVideoIds().stream().forEach(vid -> {
                 Video video = new Video();
                 video.setVideoId(vid);
-                video.setAbCode(param.getAbCode());
                 video.setRovScore(videoMap.get(vid));
                 video.setPushFrom(pushFrom());
                 videosResult.add(video);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithLevelRecallStrategyTomson.java

@@ -166,7 +166,6 @@ public class FlowPoolWithLevelRecallStrategyTomson extends AbstractFlowPoolWithL
             filterResult.getVideoIds().forEach(vid -> {
                 Video recallData = new Video();
                 recallData.setVideoId(vid);
-                recallData.setAbCode(param.getAbCode());
                 recallData.setRovScore(finalResultmap.getOrDefault(vid, 0.0));
                 recallData.setPushFrom(pushFrom());
                 recallData.setFlowPool(videoFlowPoolMap_.get(vid));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithLevelScoreRecallStrategy.java

@@ -164,7 +164,6 @@ public class FlowPoolWithLevelScoreRecallStrategy extends AbstractFlowPoolWithLe
             filterResult.getVideoIds().forEach(vid -> {
                 Video recallData = new Video();
                 recallData.setVideoId(vid);
-                recallData.setAbCode(param.getAbCode());
                 recallData.setRovScore(finalResultmap.getOrDefault(vid, 0.0));
                 recallData.setPushFrom(pushFrom());
                 recallData.setFlowPool(videoFlowPoolMap_.get(vid));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV1.java

@@ -59,7 +59,6 @@ public class RegionRealtimeRecallStrategyV1 implements RecallStrategy {
             filterResult.getVideoIds().forEach(vid -> {
                 Video video = new Video();
                 video.setVideoId(vid);
-                video.setAbCode(param.getAbCode());
                 video.setRovScore(videoMap.get(vid));
                 video.setPushFrom(pushFrom());
                 videosResult.add(video);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV5Hand.java

@@ -42,7 +42,6 @@ public class RegionRealtimeRecallStrategyV5Hand implements RecallStrategy {
         vids.forEach(vid -> {
             Video video = new Video();
             video.setVideoId(vid);
-            video.setAbCode(param.getAbCode());
             video.setRovScore(RandomUtils.nextFloat(0, 1));
             video.setPushFrom(pushFrom());
             videosResult.add(video);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/SceneCFRosnRecallStrategy.java

@@ -55,7 +55,6 @@ public class SceneCFRosnRecallStrategy implements RecallStrategy {
                 filterResult.getVideoIds().forEach(vid -> {
                     Video video = new Video();
                     video.setVideoId(vid);
-                    video.setAbCode(param.getAbCode());
                     video.setRovScore(vid2Score.getOrDefault(vid, 0.0D));
                     video.setPushFrom(pushFrom());
                     videosResult.add(video);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/SceneCFRovnRecallStrategy.java

@@ -55,7 +55,6 @@ public class SceneCFRovnRecallStrategy implements RecallStrategy {
                 filterResult.getVideoIds().forEach(vid -> {
                     Video video = new Video();
                     video.setVideoId(vid);
-                    video.setAbCode(param.getAbCode());
                     video.setRovScore(vid2Score.getOrDefault(vid, 0.0D));
                     video.setPushFrom(pushFrom());
                     videosResult.add(video);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/SpecialRecallStrategy.java

@@ -70,7 +70,6 @@ public class SpecialRecallStrategy implements RecallStrategy {
                 Video video = new Video();
                 video.setVideoId(NumberUtils.toLong(t.getValue(), 0L));
                 video.setRovScore(t.getScore());
-                video.setAbCode("99999");
                 video.setPushFrom(PUSH_FROM);
                 video.setLastVideoKey(lastSpecialRecallKey);
                 results.add(video);

recommend-server-service/src/test/java/com/tzld/piaoquan/recommend/test/merger/MergeUtilsTest.java

@@ -1,304 +0,0 @@
-package com.tzld.piaoquan.recommend.test.merger;
-
-import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
-import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.EnableAspectJAutoProxy;
-import org.springframework.http.converter.protobuf.ProtobufHttpMessageConverter;
-
-
-
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.MergeRule;
-import com.tzld.piaoquan.recommend.server.framework.merger.MergeUtils;
-import com.tzld.piaoquan.recommend.server.framework.merger.SimilarityUtils;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.apache.commons.lang3.tuple.Pair;
-//import org.junit.Assert;
-//import org.junit.Test;
-import com.google.common.base.Function;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-public class MergeUtilsTest {
-
-//    @Test
-    public void mergeWithProtectionTest() {
-        User user = new User();
-
-        Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsList = new HashMap<String, Pair<MergeRule, List<RankItem>>>();
-        MergeRule rule1 = new MergeRule();
-        rule1.queueName = "rule1";
-        rule1.minMergeNum = 2;
-        // 构建一个队列的ranklist
-        List<RankItem> items1 = new LinkedList<RankItem>();
-        RankItem item11 = new RankItem();
-        item11.setScore(0.1);
-        items1.add(item11);
-        rankerItemsList.put(rule1.queueName, Pair.of(rule1, items1));
-
-        List<RankItem> resultItems = new LinkedList<RankItem>();
-        MergeUtils.simpleMergeWithProtection(resultItems, rankerItemsList, 2, 0);
-//        Assert.assertTrue(resultItems.size() == 1);
-
-
-        RankItem item12 = new RankItem();
-        item12.setScore(0.2);
-        // list增加第二个
-        items1.add(item12);
-        resultItems.clear();
-        MergeUtils.simpleMergeWithProtection(resultItems, rankerItemsList, 2, 1);
-
-        // 判断是否受到最小合并控制， 插入两个，召回两个
-//        Assert.assertTrue(resultItems.size() == 2);
-        resultItems.clear();
-
-        // 插入两个，召回三个
-        MergeUtils.simpleMergeWithProtection(resultItems, rankerItemsList, 3, 0);
-//        Assert.assertTrue(resultItems.size() == 2);
-
-        // 最小合并条目保护,因为min mergenum 是2
-        resultItems.clear();
-        MergeUtils.simpleMergeWithProtection(resultItems, rankerItemsList, 1, 0);
-//        Assert.assertTrue(resultItems.size() == 2);
-
-        // 第二个list
-        List<RankItem> items2 = new LinkedList<RankItem>();
-        RankItem item21 = new RankItem();
-        // 一个item
-        item21.setScore(0.12);
-        items2.add(item21);
-
-        // 第二个rules
-        MergeRule rule2 = new MergeRule();
-        rule2.queueName = "rule2";
-        rule2.minMergeNum = 0;
-        // 加入第二个rule
-        rankerItemsList.put(rule2.queueName, Pair.of(rule2, items2));
-        resultItems.clear();
-        // 共同合并，因为rule2 min MergeNum =0，合并完只有两个
-        MergeUtils.simpleMergeWithProtection(resultItems, rankerItemsList, 2, 0);
-//        Assert.assertTrue(resultItems.size() == 2);
-
-        // 共同合并，rule2最小合并1个，因为rule1是1个，所以合并完是3个
-        rule2.minMergeNum = 1;
-        resultItems.clear();
-        MergeUtils.simpleMergeWithProtection(resultItems, rankerItemsList, 2, 0);
-//        Assert.assertTrue(resultItems.size() == 3);
-
-        // 第三个list， 3个item
-        List<RankItem> items3 = new LinkedList<RankItem>();
-        RankItem item31 = new RankItem();
-        item31.setScore(0.113);
-        items3.add(item31);
-        RankItem item32 = new RankItem();
-        item32.setScore(0.114);
-        items3.add(item31);
-        RankItem item33 = new RankItem();
-        item33.setScore(0.123);
-        items3.add(item31);
-
-        // 第三个rule，最小merge 1
-        MergeRule rule3 = new MergeRule();
-        rule3.queueName = "rule3";
-        rule3.minMergeNum = 1;
-        rankerItemsList.put(rule3.queueName, Pair.of(rule3, items3));
-        resultItems.clear();
-        // rule1 2个，rule2 1个，rule3 1个，所以合并完是4个
-        MergeUtils.simpleMergeWithProtection(resultItems, rankerItemsList, 2, 0);
-//        Assert.assertTrue(resultItems.size() == 4);
-    }
-
-    private List<RankItem> getItems() {
-        List<RankItem> items = new LinkedList<RankItem>();
-        RankItem item11 = new RankItem();
-        item11.setId("11");
-        item11.setScore(100.0);
-        Map<String, Double> categories11 = new HashMap<String, Double>();
-        categories11.put("美女", 100.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("22");
-        item11.setScore(99.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("动物", 99.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("33");
-        item11.setScore(98.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("美女", 98.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setScore(97.0);
-        item11.setId("44");
-        categories11 = new HashMap<String, Double>();
-        categories11.put("美女", 97.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("55");
-        item11.setScore(96.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("动物", 96.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("66");
-        item11.setScore(95.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("美女", 95.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("77");
-        item11.setScore(94.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("动物", 94.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("88");
-        item11.setScore(93.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("美女", 93.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("99");
-        item11.setScore(92.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("动物", 92.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("100");
-        item11.setScore(91.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("动物", 91.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("101");
-        item11.setScore(90.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("美女", 90.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("102");
-        item11.setScore(89.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("动物", 89.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("103");
-        item11.setScore(88.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("美女", 88.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        item11 = new RankItem();
-        item11.setId("104");
-        item11.setScore(87.0);
-        categories11 = new HashMap<String, Double>();
-        categories11.put("美女", 87.0);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        return items;
-    }
-
-//    @Test
-    public void testDiversityRerank() {
-        List<RankItem> items = new LinkedList<RankItem>();
-
-        Function<Pair<RankItem, RankItem>, Boolean> isSameCategoryFunc = SimilarityUtils.getIsSameCategoryFunc();
-
-        RankItem item11 = new RankItem();
-        Map<String, Double> categories11 = new HashMap<String, Double>();
-        categories11.put("1", 0.1);
-        item11.setRankItemCategories(categories11);
-        items.add(item11);
-
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 1, 1, 1);
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 2, 1, 1);
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 2, 2, 1);
-
-        RankItem item12 = new RankItem();
-        Map<String, Double> categories12 = new HashMap<String, Double>();
-        categories12.put("1", 0.1);
-        item12.setRankItemCategories(categories12);
-        item12.setScore(0.2);
-        items.add(item12);
-
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 2, 2, 1);
-//        Assert.assertTrue(SimilarityUtils.isSameCategory(items.get(0), items.get(1)));
-
-        RankItem item13 = new RankItem();
-        Map<String, Double> categories13 = new HashMap<String, Double>();
-        categories13.put("2", 0.1);
-        item13.setRankItemCategories(categories13);
-        items.add(item13);
-
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 2, 2, 2);
-//        Assert.assertTrue(SimilarityUtils.isSameCategory(items.get(0), items.get(1)));
-
-        RankItem item14 = new RankItem();
-        Map<String, Double> categories14 = new HashMap<String, Double>();
-        categories14.put("1", 0.1);
-        item14.setRankItemCategories(categories14);
-        items.add(item14);
-
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 3, 2, 2);
-//        Assert.assertTrue(SimilarityUtils.isSameCategory(items.get(0), items.get(1)));
-
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 3, 2, 1);
-//        Assert.assertTrue(SimilarityUtils.isSameCategory(items.get(0), items.get(1)) == false);
-
-        RankItem item15 = new RankItem();
-        Map<String, Double> categories15 = new HashMap<String, Double>();
-        categories15.put("2", 0.1);
-        item15.setRankItemCategories(categories15);
-        items.add(item15);
-
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 4, 2, 2);
-//        Assert.assertTrue(SimilarityUtils.isSameCategory(items.get(0), items.get(3)));
-
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 4, 3, 2);
-//        Assert.assertTrue(SimilarityUtils.isSameCategory(items.get(0), items.get(3)));
-
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 4, 4, 2);
-//        Assert.assertTrue(SimilarityUtils.isSameCategory(items.get(0), items.get(3)) == false);
-
-        items = getItems();
-        MergeUtils.diversityRerank(items, isSameCategoryFunc, 8, 8, 2);
-//        Assert.assertTrue(SimilarityUtils.isSameCategory(items.get(0), items.get(3)) == false);
-//        Assert.assertTrue(SimilarityUtils.isSameCategory(items.get(3), items.get(5)) == false);
-    }
-}