feat:添加预曝光过滤实验

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/enums/RootSessionIdLayer.java

@@ -0,0 +1,22 @@
+package com.tzld.piaoquan.recommend.server.common.enums;
+
+import lombok.Getter;
+
+@Getter
+public enum RootSessionIdLayer {
+
+    algo("算法", 1),
+    product("产品", 3),
+
+    ;
+    private final String layer;
+
+    private final int tailIndex;
+
+    RootSessionIdLayer(String layer, int tailIndex) {
+        this.layer = layer;
+        this.tailIndex = tailIndex;
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/RecommendParam.java

@@ -54,6 +54,7 @@ public class RecommendParam {
 
     private String channelName;
 
+    private String sessionId;
     private String rootSessionId;
     private List<UserSRBO> userRTShareList;
     private int recommendType;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/RootSessionIdExpConfig.java

@@ -12,9 +12,5 @@ public class RootSessionIdExpConfig {
 
     private Set<Integer> appType = new HashSet<>();
 
-    // TODO 只做排序或者召回实验，后续废弃
-    private String expCode;
-
-    // 兼容一个配置，同时开启排序和召回两个实验的情况
     private Set<String> expCodes = new HashSet<>();
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/ExperimentService.java

@@ -1,24 +1,28 @@
 package com.tzld.piaoquan.recommend.server.service;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.recommend.server.common.enums.RootSessionIdLayer;
 import com.tzld.piaoquan.recommend.server.model.RootSessionIdExpConfig;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 @Slf4j
 @Service
 public class ExperimentService {
 
-    @ApolloJsonValue("${rank.root.session.id.exp.config:[]}")
-    private List<RootSessionIdExpConfig> rootSessionIdExpConfigs;
+    @ApolloJsonValue("${root.session.id.exp.layer.config:{}}")
+    private Map<String, List<RootSessionIdExpConfig>> rootSessionIdExpConfigMap;
 
     /**
-     * 是否命中某个实验号
+     * 是否命中某个算法层实验号
      *
      * @param appType       appType
      * @param rootSessionId rootSessionId
@@ -26,34 +30,60 @@ public class ExperimentService {
      * @param expCode       要判断的实验号
      * @return true: 命中要判断的实验号; false: 未命中要判断的实验号
      */
-    public boolean judgeHitExp(int appType, String rootSessionId, Set<String> abExpCodes, String expCode) {
-        // 要判断的实验号为空，直接返回false
-        if (StringUtils.isBlank(expCode)) {
-            return false;
-        }
-        // 先判断AB实验号集合中，是否包含要判断的实验号
-        if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains(expCode)) {
-            return true;
-        }
+    public boolean judgeHitAlgoExp(int appType, String rootSessionId, Set<String> abExpCodes, String expCode) {
+        return judgeHitExp(RootSessionIdLayer.algo, appType, rootSessionId, abExpCodes, expCode);
+    }
 
-        // rootSessionId尾号实验配置为空或者rootSessionId为空，表示没有尾号实验。直接返回false
-        if (CollectionUtils.isEmpty(rootSessionIdExpConfigs) || StringUtils.isBlank(rootSessionId)) {
-            return false;
-        }
+    public boolean judgeHitExp(RootSessionIdLayer layer, int appType, String rootSessionId, Set<String> abExpCodes, String expCode) {
+        try {
+            // 要判断的实验号为空，直接返回false
+            if (StringUtils.isBlank(expCode) || Objects.isNull(layer)) {
+                return false;
+            }
+            // 先判断AB实验号集合中，是否包含要判断的实验号
+            if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains(expCode)) {
+                return true;
+            }
+            if (MapUtils.isEmpty(rootSessionIdExpConfigMap) || StringUtils.isBlank(rootSessionId)) {
+                return false;
+            }
+            List<RootSessionIdExpConfig> rootSessionIdExpConfigs = rootSessionIdExpConfigMap.get(layer.getLayer());
+            if (CollectionUtils.isEmpty(rootSessionIdExpConfigs)) {
+                return false;
+            }
 
-        // 根据appType和rootSessionId尾号，判断是否命中实验
-        String tail = rootSessionId.substring(rootSessionId.length() - 1);
-        for (RootSessionIdExpConfig config : rootSessionIdExpConfigs) {
-            if (config.getAppType().contains(appType) && config.getTail().contains(tail)) {
-                if (StringUtils.equals(config.getExpCode(), expCode)) {
-                    return true;
+            // 根据appType和rootSessionId尾号，判断是否命中实验
+            String cleanId = rootSessionId.replace("-", "");
+            if (cleanId.length() < layer.getTailIndex()) {
+                return false;
+            }
+            String tail = String.valueOf(cleanId.charAt(cleanId.length() - layer.getTailIndex()));
+            for (RootSessionIdExpConfig config : rootSessionIdExpConfigs) {
+
+                if (CollectionUtils.isEmpty(config.getExpCodes())) {
+                    continue;
+                }
+                if (CollectionUtils.isEmpty(config.getAppType())) {
+                    continue;
                 }
-                if (CollectionUtils.isNotEmpty(config.getExpCodes()) && config.getExpCodes().contains(expCode)) {
+                if (CollectionUtils.isEmpty(config.getTail())) {
+                    continue;
+                }
+
+
+                if (config.getAppType().contains(appType)
+                        && config.getTail().contains(tail)
+                        && config.getExpCodes().contains(expCode)) {
                     return true;
                 }
             }
-        }
 
-        return false;
+            return false;
+        } catch (Exception e) {
+            log.error("RootSessionId tail experiment judge hit exp error layer: {}, appType: {}, rootSessionId: {}, abExpCodes: {}, expCode: {}",
+                    layer, appType, rootSessionId, abExpCodes, expCode, e
+            );
+            return false;
+        }
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/PreViewedService.java

@@ -34,6 +34,8 @@ public class PreViewedService {
 
     public static final String KEY_PRE_VIEWED_FORMAT = "previewed:videos:%s:%s";
 
+    private static final String SESSION_PRE_VIEWED_FILTER_KEY = "session:previewed:video:%s:%s:%s";
+
 //    private LoadingCache<String, Set<Long>> preViewedCache = CacheBuilder.newBuilder()
 //            .maximumSize(100000) // TODO 评估容量
 //            .refreshAfterWrite(10, TimeUnit.SECONDS)
@@ -81,7 +83,7 @@ public class PreViewedService {
         String key = String.format(KEY_PRE_VIEWED_FORMAT, appType, mid);
         try {
 
-            //return preViewedCache.get(key);
+            // return preViewedCache.get(key);
             // 预曝光要求实时性，不适合本地缓存
             return getVideoIdsFromCache(key);
         } catch (Exception e) {
@@ -90,6 +92,48 @@ public class PreViewedService {
         return Collections.emptySet();
     }
 
+    public void updateSessionCache(int appType, String mid, String sessionId, List<Video> videos, Set<String> abExpCodes) {
+        if (StringUtils.isBlank(mid)
+                || StringUtils.isBlank(sessionId)
+                || CollectionUtils.isEmpty(videos)) {
+            return;
+        }
+        // 1 更新预曝光数据 过期时间 30min
+        String[] videoIds = new String[videos.size()];
+        for (int i = 0; i < videoIds.length; i++) {
+            videoIds[i] = String.valueOf(videos.get(i).getVideoId());
+        }
+        String key = String.format(SESSION_PRE_VIEWED_FILTER_KEY, appType, mid, sessionId);
+
+        int expire = 30;
+        if (MapUtils.isNotEmpty(preViewExpConfig)) {
+            for (Map.Entry<String, Integer> entry : preViewExpConfig.entrySet()) {
+                if (CommonCollectionUtils.contains(abExpCodes, entry.getKey())) {
+                    expire = entry.getValue() != null
+                            ? entry.getValue()
+                            : 30;
+                    break;
+                }
+            }
+        }
+
+        redisTemplate.opsForSet().add(key, videoIds);
+        redisTemplate.expire(key, expire, TimeUnit.MINUTES);
+    }
+
+    public Set<Long> getSessionVideoIds(int appType, String mid, String sessionId) {
+        if (StringUtils.isBlank(mid) || StringUtils.isBlank(sessionId)) {
+            return Collections.emptySet();
+        }
+        String key = String.format(SESSION_PRE_VIEWED_FILTER_KEY, appType, mid, sessionId);
+        try {
+            return getVideoIdsFromCache(key);
+        } catch (Exception e) {
+            log.error("get session pre viewed cache error! key={}", key, e);
+        }
+        return Collections.emptySet();
+    }
+
     private Set<Long> getVideoIdsFromCache(String key) {
         Set<Long> result = new HashSet<>();
         Cursor<String> cursor = redisTemplate.opsForSet().scan(key, ScanOptions.NONE);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -9,6 +9,7 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.tzld.piaoquan.recommend.server.common.RedisKeyConstants;
+import com.tzld.piaoquan.recommend.server.common.enums.RootSessionIdLayer;
 import com.tzld.piaoquan.recommend.server.feign.GrowthManagerFeign;
 import com.tzld.piaoquan.recommend.server.feign.model.CommonResponse;
 import com.tzld.piaoquan.recommend.server.feign.model.VideoCharacteristicInfo;
@@ -21,13 +22,13 @@ import com.tzld.piaoquan.recommend.server.repository.WxUserInfo;
 import com.tzld.piaoquan.recommend.server.repository.WxUserInfoRepository;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolService;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankRouter;
 import com.tzld.piaoquan.recommend.server.service.funnel.FunnelAggregator;
 import com.tzld.piaoquan.recommend.server.service.funnel.FunnelContext;
 import com.tzld.piaoquan.recommend.server.service.funnel.FunnelLogConfig;
 import com.tzld.piaoquan.recommend.server.service.funnel.FunnelLogService;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.rank.RankRouter;
 import com.tzld.piaoquan.recommend.server.service.rank.bo.UserSRBO;
 import com.tzld.piaoquan.recommend.server.service.rank.bo.UserShareReturnProfile;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
@@ -79,6 +80,8 @@ public class RecommendService {
     @Autowired
     private PreViewedService preViewedService;
     @Autowired
+    private ExperimentService experimentService;
+    @Autowired
     private FlowPoolService flowPoolService;
     @Autowired
     private StatisticsLogService statisticsLogService;
@@ -339,7 +342,8 @@ public class RecommendService {
         recallParam.setSize(request.getSize());
         recallParam.setProvince(request.getProvince());
         recallParam.setCityCode(request.getCityCode());
-
+        recallParam.setRootSessionId(request.getRootSessionId());
+        recallParam.setSessionId(request.getSessionId());
         RecallResult recallResult = recallService.recall(recallParam);
 
         RankParam rankParam = new RankParam();
@@ -464,6 +468,7 @@ public class RecommendService {
         param.setHotSceneType(request.getHotSceneType());
         param.setClientIp(request.getClientIp());
         param.setVersionCode(request.getVersionCode());
+        param.setSessionId(request.getSessionId());
         param.setRootSourceId(request.getRootSourceId());
         param.setUserShareDepth(request.getUserShareDepth());
         param.setPageNum(request.getPageNum());
@@ -699,6 +704,7 @@ public class RecommendService {
 
         recallParam.setHotSceneType(param.getHotSceneType());
         recallParam.setClientIp(param.getClientIp());
+        recallParam.setSessionId(param.getSessionId());
         recallParam.setRootSourceId(param.getRootSourceId());
         recallParam.setUserShareDepth(param.getUserShareDepth());
         recallParam.setChannelName(param.getChannelName());
@@ -754,6 +760,12 @@ public class RecommendService {
         }
 
         preViewedService.updateCache(request.getAppType(), request.getMid(), videos, param.getAbExpCodes());
+
+        boolean hit860Exp = experimentService.judgeHitExp(RootSessionIdLayer.product, param.getAppType(), param.getRootSessionId(), param.getAbExpCodes(), "860");
+        if (hit860Exp) {
+            preViewedService.updateSessionCache(request.getAppType(), request.getMid(), request.getSessionId(), videos, param.getAbExpCodes());
+        }
+
         updateLastVideoCache(videos);
 
         updateFlowPoolCache(request, param, videos);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterParam.java

@@ -37,4 +37,7 @@ public class FilterParam {
 
     private boolean testingRiskRegion;
 
+    private String rootSessionId;
+
+    private String sessionId;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterService.java

@@ -3,22 +3,19 @@ package com.tzld.piaoquan.recommend.server.service.filter;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.collect.Lists;
 import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
+import com.tzld.piaoquan.recommend.server.common.enums.RootSessionIdLayer;
+import com.tzld.piaoquan.recommend.server.service.ExperimentService;
 import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
 import com.tzld.piaoquan.recommend.server.service.filter.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.funnel.FunnelContext;
 import com.tzld.piaoquan.recommend.server.service.funnel.RecallVideoEntry;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
@@ -31,7 +28,11 @@ import java.util.concurrent.TimeUnit;
 @Component
 public class FilterService {
 
+    @Autowired
+    private ExperimentService experimentService;
+
     private final ExecutorService pool = ThreadPoolFactory.filterPool();
+
     @ApolloJsonValue("${filter_exp_config:{}}")
     private Map<String, Integer> filterExpConfig;
 
@@ -174,6 +175,12 @@ public class FilterService {
             }
         }
 
+        boolean hit860Exp = experimentService.judgeHitExp(RootSessionIdLayer.product, param.getAppType(), param.getRootSessionId(), param.getAbExpCodes(), "860");
+        if (hit860Exp) {
+            strategies.remove(ServiceBeanFactory.getBean(PreViewedStrategy.class));
+            strategies.add(ServiceBeanFactory.getBean(SessionPreViewedStrategy.class));
+        }
+
 //        if (CollectionUtils.isNotEmpty(param.getAbExpCodes()) && param.getAbExpCodes().contains("697")) {
 //            strategies.add(ServiceBeanFactory.getBean(VideoSourceTypeStrategy.class));
 //        }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/SessionPreViewedStrategy.java

@@ -0,0 +1,40 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.tzld.piaoquan.recommend.server.service.PreViewedService;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Component
+public class SessionPreViewedStrategy implements FilterStrategy {
+    @Autowired
+    private PreViewedService preViewedService;
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+
+        if (param == null) {
+            return Collections.emptyList();
+        }
+        if (StringUtils.isBlank(param.getMid())
+                || StringUtils.isBlank(param.getSessionId())
+                || CollectionUtils.isEmpty(param.getVideoIds())) {
+            return param.getVideoIds();
+        }
+        Set<Long> preViewedVideoIds = preViewedService.getSessionVideoIds(param.getAppType(), param.getMid(), param.getSessionId());
+        return param.getVideoIds().stream()
+                .filter(l -> !preViewedVideoIds.contains(l))
+                .collect(Collectors.toList());
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -61,7 +61,7 @@ public class RankRouter {
 
         // 裂变层实验，使用RootSessionId尾号进行实验
         for (Map.Entry<String, RankService> entry : strategyMap.entrySet()) {
-            boolean hitExp = experimentService.judgeHitExp(param.getAppType(), param.getRootSessionId(), null, entry.getKey());
+            boolean hitExp = experimentService.judgeHitAlgoExp(param.getAppType(), param.getRootSessionId(), null, entry.getKey());
             if (hitExp) {
                 // log.info("apType: {}, rootSessionId: {} 命中实验: {} strategies: {}", param.getAppType(), param.getRootSessionId(), entry.getKey(), entry.getValue().getClass().getSimpleName());
                 return entry.getValue().rank(param);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/FilterParamFactory.java

@@ -35,6 +35,8 @@ public class FilterParamFactory {
         filterParam.setCityCode(param.getCityCode());
         filterParam.setHotSceneType(param.getHotSceneType());
         filterParam.setClientIp(param.getClientIp());
+        filterParam.setRootSessionId(param.getRootSessionId());
+        filterParam.setSessionId(param.getSessionId());
 
         filterParam.setPushFrom(pushFrom);
         filterParam.setScoresMap(scoresMap);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallParam.java

@@ -46,6 +46,7 @@ public class RecallParam {
     private UserShareReturnProfile userProfile;
     private Map<String, String> headInfo;
 
+    private String sessionId;
     private String rootSessionId;
     private List<UserSRBO> userRTShareList;
     private int recommendType;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -170,12 +170,12 @@ public class RecallService implements ApplicationContextAware {
             // }
         }
 
-        boolean isHit842Exp = experimentService.judgeHitExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "842");
+        boolean isHit842Exp = experimentService.judgeHitAlgoExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "842");
         if (isHit842Exp) {
             strategies.add(strategyMap.get(UserDeconstructionKeywordsRecallStrategy.class.getSimpleName()));
         }
 
-        boolean isHit562Exp = experimentService.judgeHitExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "562");
+        boolean isHit562Exp = experimentService.judgeHitAlgoExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "562");
         if (isHit562Exp) {
             strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1AllRov.class.getSimpleName()));
             strategies.add(strategyMap.get(CityRovnAllRovRecallStrategy.class.getSimpleName()));
@@ -187,7 +187,7 @@ public class RecallService implements ApplicationContextAware {
             strategies.removeIf(s -> s != null && v562RemoveSet.contains(s.getClass().getSimpleName()));
         }
 
-        boolean isHit565Exp = experimentService.judgeHitExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "565");
+        boolean isHit565Exp = experimentService.judgeHitAlgoExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "565");
         if (isHit565Exp) {
             strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1AllRov.class.getSimpleName()));
             strategies.add(strategyMap.get(CityRovnAllRovRecallStrategy.class.getSimpleName()));
@@ -208,7 +208,7 @@ public class RecallService implements ApplicationContextAware {
         }
 
         // V536: 召回组合同步 V565 (all_rov 系列 + 剔除 9 路老召回), 排序后只保留流量池 + rov 兜底
-        boolean isHit536Exp = experimentService.judgeHitExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "536");
+        boolean isHit536Exp = experimentService.judgeHitAlgoExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "536");
         if (isHit536Exp) {
             strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1AllRov.class.getSimpleName()));
             strategies.add(strategyMap.get(CityRovnAllRovRecallStrategy.class.getSimpleName()));
@@ -227,7 +227,7 @@ public class RecallService implements ApplicationContextAware {
             strategies.removeIf(s -> s != null && v536RemoveSet.contains(s.getClass().getSimpleName()));
         }
 
-        boolean isHit564Exp = experimentService.judgeHitExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "564");
+        boolean isHit564Exp = experimentService.judgeHitAlgoExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "564");
         if (isHit564Exp) {
             strategies.add(strategyMap.get(ProvinceRovnRecallStrategy.class.getSimpleName()));
             // V564: rank 侧不再 extract 以下 9 路召回，这里直接剔除避免无效 Redis 调用