Merge branch 'feature_filter' of algorithm/recommend-server into master

recommend-server-service/pom.xml

@@ -75,8 +75,6 @@
         </dependency>
 
 
-
-
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>
@@ -190,6 +188,23 @@
             <groupId>com.xuxueli</groupId>
             <artifactId>xxl-job-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.data</groupId>
+            <artifactId>spring-data-mongodb</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-jpa</artifactId>
+            <version>2.4.3</version>
+        </dependency>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.mongodb</groupId>
+            <artifactId>mongodb-driver-sync</artifactId>
+        </dependency>
     </dependencies>
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/ThreadPoolFactory.java

@@ -25,6 +25,13 @@ public final class ThreadPoolFactory {
             new LinkedBlockingQueue<>(1000),
             new ThreadFactoryBuilder().setNameFormat("RecallService-%d").build(),
             new ThreadPoolExecutor.AbortPolicy());
+    private final static ExecutorService FILTER = new CommonThreadPoolExecutor(
+            256,
+            256,
+            0L, TimeUnit.SECONDS,
+            new LinkedBlockingQueue<>(1000),
+            new ThreadFactoryBuilder().setNameFormat("FilterService-%d").build(),
+            new ThreadPoolExecutor.AbortPolicy());
 
     public static ExecutorService defaultPool() {
         return DEFAULT;
@@ -34,4 +41,8 @@ public final class ThreadPoolFactory {
         return RECALL;
     }
 
+    public static ExecutorService filterPool() {
+        return FILTER;
+    }
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/CustomMongoProperties.java

@@ -0,0 +1,32 @@
+package com.tzld.piaoquan.recommend.server.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+/**
+ * @author dyp
+ */
+@ConfigurationProperties(
+        prefix = "spring.data.mongodb"
+)
+@Data
+public class CustomMongoProperties {
+    private String host;
+    private Integer port;
+    private String uri;
+    private String database;
+    private String username;
+    private char[] password;
+    private String replicaSetName;
+
+    // 连接池配置 单位ms
+    private long maxConnectionIdleTime;
+    private long maxConnectionLifeTime;
+    private long maxWaitTime;
+    private int minSize;
+    private int maxSize;
+
+    // socket
+    private int readTimeout;
+    private int connectTimeout;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/LongVideoRedisTemplateConfig.java

@@ -0,0 +1,63 @@
+package com.tzld.piaoquan.recommend.server.config;
+
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.connection.lettuce.LettucePoolingClientConfiguration;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+
+@Configuration
+public class LongVideoRedisTemplateConfig {
+
+    @Bean("longVideoRedisPool")
+    @ConfigurationProperties(prefix = "spring.long-video-redis.lettuce.pool")
+    public GenericObjectPoolConfig<LettucePoolingClientConfiguration> redisPool() {
+        return new GenericObjectPoolConfig<>();
+    }
+
+    @Bean("longVideoRedisConfig")
+    @ConfigurationProperties(prefix = "spring.long-video-redis")
+    public RedisStandaloneConfiguration tairConfig() {
+        return new RedisStandaloneConfiguration();
+    }
+
+    @Bean("longVideoRedisFactory")
+    @Primary
+    public LettuceConnectionFactory factory(GenericObjectPoolConfig<LettucePoolingClientConfiguration> redisPool,
+                                            RedisStandaloneConfiguration redisConfig) {
+        LettuceClientConfiguration lettuceClientConfiguration =
+                LettucePoolingClientConfiguration.builder().poolConfig(redisPool).build();
+        return new LettuceConnectionFactory(redisConfig, lettuceClientConfiguration);
+    }
+
+    @Bean(name = "longVideoRedisTemplate")
+    public RedisTemplate<String, String> getRedisTemplate(@Qualifier("longVideoRedisFactory") RedisConnectionFactory factory) {
+        return buildRedisTemplateByString(factory);
+    }
+
+    /**
+     * 构建redisTemplate 使用string序列化
+     *
+     * @param factory
+     * @return
+     */
+    public RedisTemplate<String, String> buildRedisTemplateByString(RedisConnectionFactory factory) {
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(factory);
+        // key的序列化类型 保证可读性
+        redisTemplate.setKeySerializer(new StringRedisSerializer());
+        redisTemplate.setValueSerializer(new StringRedisSerializer());
+        redisTemplate.setHashKeySerializer(new StringRedisSerializer());
+        redisTemplate.setHashValueSerializer(new StringRedisSerializer());
+        return redisTemplate;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/MongoTemplateConfig.java

@@ -0,0 +1,82 @@
+package com.tzld.piaoquan.recommend.server.config;
+
+import com.mongodb.MongoClientSettings;
+import com.mongodb.MongoCredential;
+import com.mongodb.ServerAddress;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.mongodb.MongoDatabaseFactory;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory;
+import org.springframework.data.mongodb.core.convert.DbRefResolver;
+import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver;
+import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper;
+import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
+import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
+
+import java.util.concurrent.TimeUnit;
+
+import static java.util.Collections.singletonList;
+
+/**
+ * @author dyp
+ */
+@Configuration
+@EnableConfigurationProperties(CustomMongoProperties.class)
+public class MongoTemplateConfig {
+    @Bean
+    public MongoClient mongoClient(CustomMongoProperties properties) {
+        MongoClientSettings.Builder builder = MongoClientSettings.builder();
+
+        builder.applyToClusterSettings(settings -> {
+            settings.hosts(singletonList(
+                    new ServerAddress(properties.getHost(), properties.getPort())));
+        });
+
+
+        builder.credential(MongoCredential.createCredential(properties.getUsername(), properties.getDatabase(), properties.getPassword()));
+        builder.applyToConnectionPoolSettings(b -> {
+            b.minSize(properties.getMinSize());
+            b.maxSize(properties.getMaxSize());
+            b.maxWaitTime(properties.getMaxWaitTime(), TimeUnit.MILLISECONDS);
+            b.maxConnectionIdleTime(properties.getMaxConnectionIdleTime(), TimeUnit.MILLISECONDS);
+            //   b.maxConnectionLifeTime(properties.getMaxConnectionLifeTime(), TimeUnit.MILLISECONDS);
+//            b.maintenanceFrequency(10, TimeUnit.MILLISECONDS);
+//            b.maintenanceInitialDelay(10, TimeUnit.MILLISECONDS);
+//            b.(properties.getMaintenanceFrequency());
+//            b.maintenanceInitialDelay();
+        });
+//        builder.applyToServerSettings(b -> {
+//            b.heartbeatFrequency()
+//        });
+        builder.applyToSocketSettings(b -> {
+            b.connectTimeout(properties.getConnectTimeout(), TimeUnit.MILLISECONDS);
+            b.readTimeout(properties.getReadTimeout(), TimeUnit.MILLISECONDS);
+        });
+        return MongoClients.create(builder.build());
+    }
+
+    @Bean
+    public MongoDatabaseFactory mongoDatabaseFactory(MongoClient client, CustomMongoProperties properties) {
+        SimpleMongoClientDatabaseFactory mongoDbFactory = new SimpleMongoClientDatabaseFactory(client,
+                properties.getDatabase());
+
+        return mongoDbFactory;
+    }
+
+    @Bean(name = "mongoTemplate")
+    public MongoTemplate mongoTemplate(MongoDatabaseFactory factory) {
+
+        // 去掉 _class 字段
+        DbRefResolver dbRefResolver = new DefaultDbRefResolver(factory);
+        MappingMongoConverter converter = new MappingMongoConverter(dbRefResolver, new MongoMappingContext());
+        converter.setTypeMapper(new DefaultMongoTypeMapper(null));
+
+        MongoTemplate mongoTemplate = new MongoTemplate(factory, converter);
+        return mongoTemplate;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/repository/WxVideoStatus.java

@@ -0,0 +1,47 @@
+package com.tzld.piaoquan.recommend.server.repository;
+
+import lombok.Data;
+
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import java.io.Serializable;
+import java.util.Date;
+
+@Data
+@Entity
+@Table(name = "wx_video_status")
+public class WxVideoStatus implements Serializable {
+    @Id
+    private Long videoId;
+
+    private Long videoUid;
+
+    private Integer auditStatus;
+
+    private Integer appAuditStatus;
+
+    private String auditTranscationId;
+
+    private Integer chargeExamineStatus;
+
+    private Integer videoStatus;
+
+    private Integer examineStatus;
+
+    private Integer recommendStatus;
+
+    private Integer appRecommendStatus;
+
+    private Integer sensitiveStatus;
+
+    private Integer stageRecommendExamineStatus;
+
+    private Date videoGmtCreate;
+
+    private Long videoGmtCreateTimestamp;
+
+    private Integer originalStatus;
+
+    private Integer isReupload;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/repository/WxVideoStatusRepository.java

@@ -0,0 +1,12 @@
+package com.tzld.piaoquan.recommend.server.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+import java.util.Collection;
+import java.util.List;
+
+@Repository
+public interface WxVideoStatusRepository extends JpaRepository<WxVideoStatus, Long> {
+    List<WxVideoStatus> findAllByVideoIdIn(Collection<Long> videoIds);
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/repository/WxVideoTagRel.java

@@ -0,0 +1,35 @@
+package com.tzld.piaoquan.recommend.server.repository;
+
+import lombok.Data;
+
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Table;
+import java.io.Serializable;
+import java.util.Date;
+
+@Data
+@Entity
+@Table(name = "wx_video_tag_rel")
+@IdClass(WxVideoTagRel.WxVideoTagRelId.class)
+public class WxVideoTagRel implements Serializable {
+    @Id
+    private Long videoId;
+    @Id
+    private Long tagId;
+
+    private Date createTime;
+
+    @Data
+    public class WxVideoTagRelId implements Serializable {
+        private Long videoId;
+        private Long tagId;
+
+        public WxVideoTagRelId() {
+
+        }
+
+
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/repository/WxVideoTagRelRepository.java

@@ -0,0 +1,11 @@
+package com.tzld.piaoquan.recommend.server.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+@Repository
+public interface WxVideoTagRelRepository extends JpaRepository<WxVideoTagRel, Long> {
+    List<WxVideoTagRel> findAllByTagId(Long tagId);
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -409,6 +409,8 @@ public class RecommendService {
         recallParam.setRiskFilterFlag(param.getRiskFilterFlag());
         recallParam.setVideosWithRisk(param.getVideosWithRisk());
         recallParam.setAppRegionFiltered(param.getAppRegionFiltered());
+        recallParam.setAbExpCodes(param.getAbExpCodes());
+
 
         return recallParam;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/ServiceBeanFactory.java

@@ -0,0 +1,23 @@
+package com.tzld.piaoquan.recommend.server.service;
+
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author dyp
+ */
+@Component
+public class ServiceBeanFactory implements ApplicationContextAware {
+    private static ApplicationContext applicationContext;
+
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+        this.applicationContext = applicationContext;
+    }
+
+    public static <T> T getBean(Class<T> clazz) {
+        return applicationContext.getBean(clazz);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/WarmUpService.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.recommend.server.service;
 
+import com.tzld.piaoquan.recommend.server.repository.WxVideoStatusRepository;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -18,11 +19,21 @@ public class WarmUpService {
     @Autowired
     @Qualifier("redisTemplate")
     private RedisTemplate<String, String> redisTemplate;
-    
+    @Autowired
+    @Qualifier("featureRedisTemplate")
+    private RedisTemplate<String, String> featureRedisTemplate;
+    @Autowired
+    @Qualifier("longVideoRedisTemplate")
+    private RedisTemplate<String, String> longVideoRedisTemplate;
+    @Autowired
+    private WxVideoStatusRepository wxVideoStatusRepository;
 
     @PostConstruct
     public void warmup() {
         redisTemplate.opsForValue().get("");
+        featureRedisTemplate.opsForValue().get("");
+        longVideoRedisTemplate.opsForValue().get("");
         ScorerUtils.warmUp();
+        wxVideoStatusRepository.count();
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -1,41 +1,51 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
 import com.google.common.base.Stopwatch;
-import com.thoughtworks.xstream.mapper.Mapper;
+import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.service.PreViewedService;
+import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
 import com.tzld.piaoquan.recommend.server.service.ViewedService;
+import com.tzld.piaoquan.recommend.server.service.filter.strategy.*;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 /**
  * @author dyp
  */
 @Slf4j
-public abstract class AbstractFilterService implements FilterService {
-    public Integer forceTruncation;
+public abstract class AbstractFilterService {
     @Autowired
     private PreViewedService preViewedService;
 
     @Autowired
     private ViewedService viewedService;
 
+    private ExecutorService pool = ThreadPoolFactory.filterPool();
 
+    @Value("${newFilterGlobalSwitch:false}")
+    private boolean newFilterGlobalSwitch;
+    @Value("${newFilterAbExpCode:}")
+    private String newFilterAbExpCode;
 
-    protected List<Long> viewFilter(FilterParam param) {
+
+    private List<Long> viewFilterOld(FilterParam param) {
         // log.info("filterParam={}", JSONUtils.toJson(param));
         // 风险过滤
         List<Long> videoIds = filterWithRiskVideo(param.getRiskFilterFlag(),
                 param.getAppType(), param.getRegionCode(), param.getAppRegionFiltered(), param.getVideosWithRisk(),
-                param.getVideoIds());
+                param.getVideoIds(), param.getForceTruncation());
         // log.info("filterByRiskVideos videoIds={}", JSONUtils.toJson(videoIds));
         videoIds = filterByPreViewed(param.getAppType(), param.getMid(), videoIds);
         // log.info("filterByPreViewed videoIds={}", JSONUtils.toJson(videoIds));
@@ -73,22 +83,23 @@ public abstract class AbstractFilterService implements FilterService {
                                            String regionCode,
                                            Map<Integer, List<String>> rules,
                                            List<Long> videosWithRisk,
-                                           List<Long> videoIds){
-        if (!riskFlag){
-            return this.truncation(videoIds);
+                                           List<Long> videoIds,
+                                           int forceTruncation) {
+        if (!riskFlag) {
+            return this.truncation(videoIds, forceTruncation);
         }
         // 1 判断是否过滤,不展示的app+区域列表。
         boolean filterFlag;
-        if (rules.containsKey(appType)){
+        if (rules.containsKey(appType)) {
             filterFlag = false;
-            if (rules.get(appType).contains(regionCode)){
+            if (rules.get(appType).contains(regionCode)) {
                 filterFlag = true;
             }
         } else {
             filterFlag = true;
         }
-        if (!filterFlag){
-            return this.truncation(videoIds);
+        if (!filterFlag) {
+            return this.truncation(videoIds, forceTruncation);
         }
         // 2 开始过滤。
         List<Long> videoIdNew = new ArrayList<>();
@@ -97,13 +108,101 @@ public abstract class AbstractFilterService implements FilterService {
                 videoIdNew.add(videoId);
             }
         }
-        return this.truncation(videoIdNew);
+        return this.truncation(videoIdNew, forceTruncation);
     }
-    private List<Long> truncation(List<Long> videoIds){
-        if (this.forceTruncation == null){
+
+    private List<Long> truncation(List<Long> videoIds, int forceTruncation) {
+        if (forceTruncation == 0) {
             return videoIds;
-        }else{
-            return videoIds.subList(0, Math.min(this.forceTruncation, videoIds.size()));
+        } else {
+            return videoIds.subList(0, Math.min(forceTruncation, videoIds.size()));
+        }
+    }
+
+    protected List<Long> viewFilter(FilterParam param) {
+        boolean hit = newFilterGlobalSwitch
+                || CommonCollectionUtils.contains(param.getAbExpCodes(), newFilterAbExpCode);
+        if (hit) {
+            return viewFilterNew(param);
+        } else {
+            return viewFilterOld(param);
         }
     }
+
+    private List<Long> viewFilterNew(FilterParam param) {
+
+        // hardcode : 风险过滤会做截断，所以先同步调用
+        List<Long> riskVideoIds = filterWithRiskVideo(param.getRiskFilterFlag(),
+                param.getAppType(), param.getRegionCode(), param.getAppRegionFiltered(), param.getVideosWithRisk(),
+                param.getVideoIds(), param.getForceTruncation());
+        param.setVideoIds(riskVideoIds);
+
+        List<FilterStrategy> strategies = getStrategies(param);
+        log.info("filter strategies {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies,
+                o -> o.getClass().getSimpleName())));
+        CountDownLatch cdl = new CountDownLatch(strategies.size());
+        List<Future<List<Long>>> futures = new ArrayList<>();
+        for (final FilterStrategy strategy : strategies) {
+            Future<List<Long>> future = pool.submit(() -> {
+                Stopwatch stopwatch = Stopwatch.createStarted();
+                List<Long> result = strategy.filter(param);
+                cdl.countDown();
+                log.info("{} param {} result {} cost {} ms", strategy.getClass().getSimpleName(),
+                        JSONUtils.toJson(param.getVideoIds()),
+                        JSONUtils.toJson(result),
+                        stopwatch.elapsed().toMillis());
+                return result;
+            });
+            futures.add(future);
+        }
+        try {
+            cdl.await(1000, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            log.error("filter error", e);
+            return null;
+        }
+
+        List<List<Long>> videoIds = new ArrayList<>();
+        for (Future<List<Long>> f : futures) {
+            try {
+                videoIds.add(f.get());
+            } catch (Exception e) {
+                log.error("future get error ", e);
+            }
+        }
+        if (CollectionUtils.isEmpty(videoIds)) {
+            return Collections.emptyList();
+        }
+        List<Long> result = videoIds.get(0);
+        if (CollectionUtils.isEmpty(result)) {
+            return Collections.emptyList();
+        }
+        for (int i = 1; i < videoIds.size(); ++i) {
+            result.retainAll(videoIds.get(i));
+        }
+        log.info("filter result {}", JSONUtils.toJson(result));
+        return result;
+    }
+
+    private List<FilterStrategy> getStrategies(FilterParam param) {
+        List<FilterStrategy> strategies = new ArrayList<>();
+        strategies.add(ServiceBeanFactory.getBean(PreViewedStrategy.class));
+        strategies.add(ServiceBeanFactory.getBean(ViewedStrategy.class));
+        strategies.add(ServiceBeanFactory.getBean(RecommendStatusStrategy.class));
+        switch (param.getAppType()) {
+            case 0:
+            case 4:
+                strategies.add(ServiceBeanFactory.getBean(AllowListStrategy.class));
+                break;
+            case 13:
+                break;
+            default:
+                strategies.add(ServiceBeanFactory.getBean(AllowListStrategy.class));
+                strategies.add(ServiceBeanFactory.getBean(TagStrategy.class));
+                break;
+        }
+
+        return strategies;
+    }
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterParam.java

@@ -4,6 +4,7 @@ import lombok.Data;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * @author dyp
@@ -21,4 +22,6 @@ public class FilterParam {
     private Map<Integer, List<String>> appRegionFiltered;
     private List<Long> videosWithRisk;
     private String regionCode;
+    private int forceTruncation;
+    private Set<String> abExpCodes;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterStrategy.java

@@ -0,0 +1,8 @@
+package com.tzld.piaoquan.recommend.server.service.filter;
+
+
+import java.util.List;
+
+public interface FilterStrategy {
+    List<Long> filter(FilterParam param);
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/RegionFilterService.java

@@ -1,7 +1,5 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
-import lombok.Getter;
-import lombok.Setter;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
@@ -11,11 +9,6 @@ import java.util.List;
  */
 @Service
 public class RegionFilterService extends AbstractFilterService {
-
-    public RegionFilterService(Integer forceTruncation){
-        this.forceTruncation = forceTruncation;
-    }
-    public RegionFilterService(){}
     public FilterResult filter(FilterParam param) {
         List<Long> videoIds = viewFilter(param);
         return new FilterResult(videoIds);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/AllowListStrategy.java

@@ -0,0 +1,110 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.google.common.collect.Lists;
+import com.google.common.hash.Hashing;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+/**
+ * @author dyp
+ */
+@Component
+@Slf4j
+public class AllowListStrategy implements FilterStrategy {
+    @Autowired
+    @Qualifier("longVideoRedisTemplate")
+    private RedisTemplate<String, String> redisTemplate;
+
+    // private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_PREFIX = "movie:videoid:allowlist:";
+    private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX = "movie:videoid:allowSet:";
+    // private static final String VIDEO_ALLOW_LIST_BITMAP_KEY = "movie.store.mp.allowlist.videoid.bitmap";
+    private static final String RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY = "mp:religion:allowlist:videoid:bitmap";
+//    @Value("${movie.videoid.allowlist.compatible:1}")
+//    private Integer mvalCompatible;
+
+    @Override
+    // TODO 未找到优化方法 暂时保留原代码
+    public List<Long> filter(FilterParam param) {
+        if (param == null
+                || CollectionUtils.isEmpty(param.getVideoIds())) {
+            return Collections.emptyList();
+        }
+        //不是新小程序 在白名单则不显示
+        Set<Long> retainVideoIds = new LinkedHashSet<>();
+        if (param.getAppType() != AppTypeEnum.WAN_NENG_VIDEO.getCode()
+                && param.getAppType() != AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
+                && param.getAppType() != AppTypeEnum.ZUI_JING_QI.getCode()
+                && param.getAppType() != AppTypeEnum.H5.getCode()) {
+
+            Iterator<Long> iterator = param.getVideoIds().iterator();
+            while (iterator.hasNext()) {
+                Long videoId = iterator.next();
+                //如果不在新小程序白名单 则保留
+                if (!isMemberOfVideoAllowList(videoId)) {
+                    retainVideoIds.add(videoId);
+                }
+            }
+        } else if (param.getAppType() == AppTypeEnum.WAN_NENG_VIDEO.getCode()
+                || param.getAppType() == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
+                || param.getAppType() == AppTypeEnum.ZUI_JING_QI.getCode()
+                || param.getAppType() == AppTypeEnum.H5.getCode()) {
+            Iterator<Long> iterator = param.getVideoIds().iterator();
+            while (iterator.hasNext()) {
+                Long videoId = iterator.next();
+                //如果不在宗教白名单 则保留
+                if (!isMemberOfReligionVideoAllowList(videoId)) {
+                    retainVideoIds.add(videoId);
+                }
+            }
+        }
+        if (CollectionUtils.isEmpty(retainVideoIds)) {
+            return Collections.emptyList();
+        }
+
+        return Lists.newArrayList(retainVideoIds);
+    }
+
+    private boolean isMemberOfVideoAllowList(Long videoId) {
+        if (Objects.isNull(videoId)) {
+            return false;
+        }
+        try {
+            int newIdx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 100;
+            String newPrefix = VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX + newIdx;
+            Boolean result = redisTemplate.opsForSet().isMember(newPrefix, String.valueOf(videoId));
+//            // 兼容旧 key apollo配置0 所以删除
+//            if (Objects.equals(1, mvalCompatible) && !result) {
+//                int idx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 10;
+//                result = redisTemplate.opsForValue().getBit(VIDEO_ALLOW_LIST_BITMAP_KEY_PREFIX + idx, videoId);
+//            }
+//            if (Objects.equals(1, mvalCompatible) && !result) {
+//                result = redisTemplate.opsForValue().getBit(VIDEO_ALLOW_LIST_BITMAP_KEY, videoId);
+//            }
+            return result;
+        } catch (Exception e) {
+            log.error("isMemberOfVideoAllowList error {}", videoId, e);
+        }
+        return false;
+    }
+
+    public boolean isMemberOfReligionVideoAllowList(Long videoId) {
+        if (Objects.isNull(videoId)) {
+            return false;
+        }
+        try {
+            return redisTemplate.opsForValue().getBit(RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY, videoId);
+        } catch (Exception e) {
+            log.error("isMemberOfReligionVideoAllowList error {}", e, videoId);
+        }
+        return false;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/PreViewedStrategy.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.tzld.piaoquan.recommend.server.service.PreViewedService;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Component
+public class PreViewedStrategy implements FilterStrategy {
+    @Autowired
+    private PreViewedService preViewedService;
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+
+        if (param == null) {
+            return Collections.emptyList();
+        }
+        if (StringUtils.isBlank(param.getMid())
+                || CollectionUtils.isEmpty(param.getVideoIds())) {
+            return param.getVideoIds();
+        }
+        Set<Long> preViewedVideoIds = preViewedService.getVideoIds(param.getAppType(), param.getMid());
+        return param.getVideoIds().stream()
+                .filter(l -> !preViewedVideoIds.contains(l))
+                .collect(Collectors.toList());
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/RecommendStatusStrategy.java

@@ -0,0 +1,98 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoStatus;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoStatusRepository;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.dao.DataAccessException;
+import org.springframework.data.redis.core.RedisOperations;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.core.SessionCallback;
+import org.springframework.data.redis.core.ValueOperations;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Component
+@Slf4j
+public class RecommendStatusStrategy implements FilterStrategy {
+    @Autowired
+    @Qualifier("redisTemplate")
+    private RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private WxVideoStatusRepository wxVideoStatusRepository;
+
+    private String keyFormat = "video:recommend:status:%s";
+
+    private static final int RECOMMEND_STATUS = -6;
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+        if (param == null
+                || CollectionUtils.isEmpty(param.getVideoIds())) {
+            return Collections.emptyList();
+        }
+
+        List<String> keys = param.getVideoIds().stream()
+                .map(id -> String.format(keyFormat, id))
+                .collect(Collectors.toList());
+
+        List<String> recommendStatus = redisTemplate.opsForValue().multiGet(keys);
+        List<Long> cacheMissVideoIds = new ArrayList<>();
+        Map<Long, Integer> recommendStatusMap = new HashMap<>();
+        for (int i = 0; i < param.getVideoIds().size(); i++) {
+            String value = recommendStatus.get(i);
+            if (StringUtils.isBlank(value)) {
+                cacheMissVideoIds.add(param.getVideoIds().get(i));
+            } else {
+                recommendStatusMap.put(param.getVideoIds().get(i), NumberUtils.toInt(value));
+            }
+        }
+
+        if (CollectionUtils.isNotEmpty(cacheMissVideoIds)) {
+            List<WxVideoStatus> status = wxVideoStatusRepository.findAllByVideoIdIn(cacheMissVideoIds);
+            if (CollectionUtils.isNotEmpty(status)) {
+                for (WxVideoStatus v : status) {
+                    recommendStatusMap.put(v.getVideoId(), v.getRecommendStatus());
+
+                    // TODO 异步更新缓存
+                    ThreadPoolFactory.defaultPool().execute(() -> {
+                        redisTemplate.executePipelined(new SessionCallback<String>() {
+                            @Override
+                            public <A, B> String execute(RedisOperations<A, B> redisOperations) throws DataAccessException {
+                                ValueOperations<String, String> operations =
+                                        (ValueOperations<String, String>) redisOperations.opsForValue();
+                                status.forEach(v -> {
+                                    operations.set(String.format(keyFormat, v.getVideoId()),
+                                            String.valueOf(v.getRecommendStatus()), RandomUtils.nextInt(30, 60),
+                                            TimeUnit.SECONDS);
+                                });
+
+                                return null;
+                            }
+                        });
+                    });
+                }
+            }
+        }
+
+        return param.getVideoIds().stream()
+                .filter(id -> recommendStatusMap.containsKey(id) && recommendStatusMap.get(id) == RECOMMEND_STATUS)
+                .collect(Collectors.toList());
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/RiskVideoStrategy.java

@@ -0,0 +1,66 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author dyp
+ */
+@Component
+public class RiskVideoStrategy implements FilterStrategy {
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+
+        if (CollectionUtils.isEmpty(param.getVideoIds())) {
+            return Collections.emptyList();
+        }
+        boolean riskFlag = param.getRiskFilterFlag();
+        int appType = param.getAppType();
+        String regionCode = param.getRegionCode();
+        Map<Integer, List<String>> rules = param.getAppRegionFiltered();
+        List<Long> videosWithRisk = param.getVideosWithRisk();
+        List<Long> videoIds = param.getVideoIds();
+
+
+        if (!riskFlag) {
+            return this.truncation(param.getVideoIds(), param.getForceTruncation());
+        }
+        // 1 判断是否过滤,不展示的app+区域列表。
+        boolean filterFlag = false;
+        if (rules.containsKey(appType)) {
+            filterFlag = false;
+            if (rules.get(appType).contains(regionCode)) {
+                filterFlag = true;
+            }
+        } else {
+            filterFlag = true;
+        }
+        if (!filterFlag) {
+            return this.truncation(videoIds, param.getForceTruncation());
+        }
+        // 2 开始过滤。
+        List<Long> videoIdNew = new ArrayList<>();
+        for (Long videoId : videoIds) {
+            if (!videosWithRisk.contains(videoId)) {
+                videoIdNew.add(videoId);
+            }
+        }
+        return this.truncation(videoIdNew, param.getForceTruncation());
+    }
+
+    private List<Long> truncation(List<Long> videoIds, int forceTruncation) {
+        if (forceTruncation == 0) {
+            return videoIds;
+        } else {
+            return videoIds.subList(0, Math.min(forceTruncation, videoIds.size()));
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/TagStrategy.java

@@ -0,0 +1,115 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author dyp
+ */
+@Component
+@Slf4j
+public class TagStrategy implements FilterStrategy {
+
+    @Autowired
+    private WxVideoTagRelRepository repository;
+
+    @Value("${video.filter.tagids:}")
+    private String videoFilterTagIds;
+
+    // 内存持久保存不淘汰
+    private LoadingCache<Long, Set<Long>> videoTagCache = CacheBuilder.newBuilder()
+            .maximumSize(100)
+            .refreshAfterWrite(60, TimeUnit.SECONDS)
+            .expireAfterWrite(60, TimeUnit.SECONDS)
+            .expireAfterAccess(60, TimeUnit.SECONDS)
+            .build(new CacheLoader<Long, Set<Long>>() {
+                @Override
+                public Set<Long> load(Long tagId) {
+                    List<WxVideoTagRel> rels = repository.findAllByTagId(tagId);
+                    return CommonCollectionUtils.toSet(rels, WxVideoTagRel::getVideoId);
+                }
+            });
+
+    @PostConstruct
+    public void init() {
+        if (StringUtils.isNotBlank(videoFilterTagIds)) {
+            // 只有涉政标签，用循环简化代码
+            String[] tags = videoFilterTagIds.split(",");
+            for (String tag : tags) {
+                if (StringUtils.isBlank(tag)) {
+                    continue;
+                }
+                videoTagCache.getUnchecked(NumberUtils.toLong(tag));
+            }
+        }
+    }
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+        // TODO 主要是涉政标签
+        if (param == null
+                || CollectionUtils.isEmpty(param.getVideoIds())) {
+            return Collections.emptyList();
+        }
+
+
+        List<Long> tagIds = new ArrayList<>();
+        if (StringUtils.isBlank(videoFilterTagIds)) {
+            return param.getVideoIds();
+        }
+        String[] tags = videoFilterTagIds.split(",");
+        for (String tag : tags) {
+            if (Objects.isNull(tag) || Objects.equals("", tag)) {
+                continue;
+            }
+            tagIds.add(Long.parseLong(tag));
+        }
+
+        Set<Long> retainVideoIds = new LinkedHashSet<>();
+        Iterator<Long> iterator = param.getVideoIds().iterator();
+        while (iterator.hasNext()) {
+            Long videoId = iterator.next();
+            for (Long tagId : tagIds) {
+                if (!hasVideoRelTagId(videoId, tagId)) {
+                    retainVideoIds.add(videoId);
+                }
+            }
+        }
+
+        if (CollectionUtils.isEmpty(retainVideoIds)) {
+            return Collections.emptyList();
+        }
+
+        return Lists.newArrayList(retainVideoIds);
+    }
+
+    private boolean hasVideoRelTagId(Long videoId, Long tagId) {
+        if (Objects.isNull(videoId) || videoId <= 0L || Objects.isNull(tagId) || tagId <= 0L) {
+            return false;
+        }
+        Set<Long> videos = videoTagCache.getUnchecked(tagId);
+        if (CollectionUtils.isEmpty(videos)) {
+            return false;
+        }
+        return videos.contains(videoId);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/VideoView.java

@@ -0,0 +1,35 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import lombok.Data;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.index.IndexDirection;
+import org.springframework.data.mongodb.core.index.Indexed;
+import org.springframework.data.mongodb.core.mapping.Document;
+import org.springframework.data.mongodb.core.mapping.Field;
+
+import java.util.Date;
+
+@Data
+@Document(collection = "video_view")
+public class VideoView {
+    @Id
+    private String id;
+    @Indexed(name = "video_id", direction = IndexDirection.ASCENDING)
+    @Field("video_id")
+    private Long videoId;
+    @Field("uid")
+    private String uid;
+    @Field("action_type")
+    private Integer actionType;
+    @Field("create_time")
+    private Long createTime;
+    @Field("create_date")
+    private Date createDate;
+    @Field("count")
+    private Integer count;
+    @Field("source")
+    private Integer source;
+    @Field("status")
+    private Integer status;
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/ViewedStrategy.java

@@ -0,0 +1,87 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.dao.DataAccessException;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.redis.core.RedisOperations;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.core.SessionCallback;
+import org.springframework.data.redis.core.SetOperations;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Component
+@Slf4j
+public class ViewedStrategy implements FilterStrategy {
+    @Autowired
+    @Qualifier("longVideoRedisTemplate")
+    private RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private MongoTemplate mongoTemplate;
+
+    private String keyFormat = "user:exclude:videoidset:%s";
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+        String user = StringUtils.isNotBlank(param.getUid()) ? param.getUid() : param.getMid();
+        if (StringUtils.isBlank(user)) {
+            return param.getVideoIds();
+        }
+        String key = String.format(keyFormat, param.getMid());
+        Set<String> viewedVideoIds = redisTemplate.opsForSet().members(key);
+        if (CollectionUtils.isEmpty(viewedVideoIds)) {
+            // 从mongo取曝光数据
+            Criteria criteria = new Criteria();
+            criteria.and("uid").is(user);
+            Query query = new Query();
+            query.addCriteria(criteria);
+            List<VideoView> list = mongoTemplate.find(query, VideoView.class);
+            //TODO 为什么限制最多10000条？是不是限制近几天更合适？
+            if (CollectionUtils.isNotEmpty(list)) {
+                int limit = 10000;
+                for (int i = list.size() - 1; i >= 0 && limit-- > 0; i--) {
+                    viewedVideoIds.add(String.valueOf(list.get(i).getVideoId()));
+                }
+
+                // 异步写Redis
+                ThreadPoolFactory.defaultPool().execute(() -> {
+                    redisTemplate.executePipelined(new SessionCallback<String>() {
+                        @Override
+                        public <A, B> String execute(RedisOperations<A, B> redisOperations) throws DataAccessException {
+                            SetOperations<String, String> operations =
+                                    (SetOperations<String, String>) redisOperations.opsForSet();
+                            operations.add(key, viewedVideoIds.toArray(new String[viewedVideoIds.size()]));
+                            redisTemplate.expire(key, 360 * 3600, TimeUnit.SECONDS);
+                            return null;
+                        }
+                    });
+                });
+            }
+        }
+
+        if (CollectionUtils.isEmpty(viewedVideoIds)) {
+            return param.getVideoIds();
+        }
+        return param.getVideoIds().stream()
+                .filter(vid -> !viewedVideoIds.contains(String.valueOf(vid)))
+                .collect(Collectors.toList());
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/FilterParamFactory.java

@@ -23,6 +23,7 @@ public class FilterParamFactory {
         filterParam.setVideosWithRisk(param.getVideosWithRisk());
         filterParam.setAppRegionFiltered(param.getAppRegionFiltered());
         filterParam.setRegionCode(param.getRegionCode());
+        filterParam.setAbExpCodes(param.getAbExpCodes());
         return filterParam;
     }
 
@@ -37,6 +38,7 @@ public class FilterParamFactory {
         filterParam.setVideosWithRisk(param.getVideosWithRisk());
         filterParam.setAppRegionFiltered(param.getAppRegionFiltered());
         filterParam.setRegionCode(param.getRegionCode());
+        filterParam.setAbExpCodes(param.getAbExpCodes());
         return filterParam;
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallParam.java

@@ -4,6 +4,7 @@ import lombok.Data;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * @author dyp
@@ -30,5 +31,6 @@ public class RecallParam {
     private Boolean riskFilterFlag;
     private Map<Integer, List<String>> appRegionFiltered;
     private List<Long> videosWithRisk;
+    private Set<String> abExpCodes;
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractRegionRecallStrategy.java

@@ -4,6 +4,7 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.collect.Lists;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
 import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
 import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
 import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
@@ -134,8 +135,10 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
                 lastVideoId = t.getValue();
                 videoMap.put(NumberUtils.toLong(t.getValue(), 0), t.getScore());
             }
-            filterService.forceTruncation = 20;
-            FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet())));
+            FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
+            filterParam.setForceTruncation(20);
+
+            FilterResult filterResult = filterService.filter(filterParam);
             // log.info("recall filterResult={}", JSONUtils.toJson(filterResult));
 
             if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/CommonCollectionUtils.java

@@ -2,10 +2,7 @@ package com.tzld.piaoquan.recommend.server.util;
 
 import org.apache.commons.collections4.CollectionUtils;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -32,7 +29,16 @@ public class CommonCollectionUtils {
         if (CollectionUtils.isEmpty(col)) {
             return Collections.emptyMap();
         }
-        return col.stream().collect(Collectors.toMap(keyFunc::apply, valueFunc::apply));
+        return col.stream().collect(Collectors.toMap(keyFunc::apply, valueFunc::apply, (u, v) -> v, HashMap::new));
+    }
+
+    public static <T, R> Set<R> toSet(Collection<T> col, Function<T, R> func) {
+        if (CollectionUtils.isEmpty(col)) {
+            return Collections.emptySet();
+        }
+        return col.stream()
+                .map(c -> func.apply(c))
+                .collect(Collectors.toSet());
     }
 
     public static <T> boolean contains(Collection<T> col, T ele) {

recommend-server-service/src/main/resources/application-dev.yml

@@ -35,7 +35,46 @@ spring:
         max-wait: -1
         max-idle: 8
         min-idle: 0
-
+  long-video-redis:
+    hostName: r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com
+    port: 6379
+    password: Wqsd@2019
+    timeout: 1000
+    lettuce:
+      pool:
+        max-active: 8
+        max-wait: -1
+        max-idle: 8
+        min-idle: 0
+  data:
+    mongodb:
+      host: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
+      port: 3717
+      database: longvideo
+      username: lv
+      password: lv@2018
+      replicaSetName: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com:3717,dds-bp1de4fc73029b242144.mongodb.rds.aliyuncs.com:3717
+      # 连接池
+      maxConnectionIdleTime: 180000
+      # maxConnectionLifeTime: 30000
+      maxWaitTime: 3000
+      minSize: 5
+      maxSize: 10
+  datasource:
+    url: jdbc:mysql://rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com:3306/longvideo?useSSL=true&useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+    username: wx2016_longvideo
+    password: wx2016_longvideoP@assword1234
+    driver-class-name: com.mysql.jdbc.Driver
+    hikari:
+      connection-timeout: 30000
+      minimum-idle: 5
+      maximum-pool-size: 10
+      auto-commit: true
+      idle-timeout: 30000
+  jpa:
+    hibernate:
+      ddl-auto: validate
+    database: mysql
 xxl:
   job:
     admin:

recommend-server-service/src/main/resources/application-pre.yml

@@ -35,6 +35,45 @@ spring:
         max-wait: -1
         max-idle: 8
         min-idle: 0
+  long-video-redis:
+    hostName: r-bp1oyhyx4mxgs6klyt561.redis.rds.aliyuncs.com
+    port: 6379
+    password: Wqsd@2019
+    timeout: 1000
+    lettuce:
+      pool:
+        max-active: 8
+        max-wait: -1
+        max-idle: 8
+        min-idle: 0
+  data:
+    mongodb:
+      host: dds-bp1982ea69508a541455.mongodb.rds.aliyuncs.com
+      port: 3717
+      database: longvideo
+      username: lv
+      password: lv@2018
+      # 连接池
+      maxConnectionIdleTime: 180000
+      # maxConnectionLifeTime: 30000
+      maxWaitTime: 5000
+      minSize: 5
+      maxSize: 100
+  datasource:
+    url: jdbc:mysql://rm-bp1jjv3jv98133plv285-vpc-rw.mysql.rds.aliyuncs.com:3306/longvideo?useSSL=true&useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+    username: wx2016_longvideo
+    password: wx2016_longvideoP@assword1234
+    driver-class-name: com.mysql.jdbc.Driver
+    hikari:
+      connection-timeout: 30000
+      minimum-idle: 5
+      maximum-pool-size: 10
+      auto-commit: true
+      idle-timeout: 30000
+  jpa:
+    hibernate:
+      ddl-auto: validate
+    database: mysql
 xxl:
   job:
     admin:

recommend-server-service/src/main/resources/application-prod.yml

@@ -35,6 +35,45 @@ spring:
         max-wait: -1
         max-idle: 8
         min-idle: 0
+  long-video-redis:
+    hostName: r-bp1oyhyx4mxgs6klyt561.redis.rds.aliyuncs.com
+    port: 6379
+    password: Wqsd@2019
+    timeout: 1000
+    lettuce:
+      pool:
+        max-active: 8
+        max-wait: -1
+        max-idle: 8
+        min-idle: 0
+  data:
+    mongodb:
+      host: dds-bp1982ea69508a541455.mongodb.rds.aliyuncs.com
+      port: 3717
+      database: longvideo
+      username: lv
+      password: lv@2018
+      # 连接池
+      maxConnectionIdleTime: 180000
+      # maxConnectionLifeTime: 30000
+      maxWaitTime: 5000
+      minSize: 5
+      maxSize: 100
+  datasource:
+    url: jdbc:mysql://rm-bp1jjv3jv98133plv285-vpc-rw.mysql.rds.aliyuncs.com:3306/longvideo?useSSL=true&useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+    username: wx2016_longvideo
+    password: wx2016_longvideoP@assword1234
+    driver-class-name: com.mysql.jdbc.Driver
+    hikari:
+      connection-timeout: 30000
+      minimum-idle: 5
+      maximum-pool-size: 100
+      auto-commit: true
+      idle-timeout: 30000
+  jpa:
+    hibernate:
+      ddl-auto: validate
+    database: mysql
 xxl:
   job:
     admin:

recommend-server-service/src/main/resources/application-test.yml

@@ -35,7 +35,46 @@ spring:
         max-wait: -1
         max-idle: 8
         min-idle: 0
-
+  long-video-redis:
+    hostName: r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com
+    port: 6379
+    password: Wqsd@2019
+    timeout: 1000
+    lettuce:
+      pool:
+        max-active: 8
+        max-wait: -1
+        max-idle: 8
+        min-idle: 0
+  data:
+    mongodb:
+      host: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
+      port: 3717
+      database: longvideo
+      username: lv
+      password: lv@2018
+      replicaSetName: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com:3717,dds-bp1de4fc73029b242144.mongodb.rds.aliyuncs.com:3717
+      # 连接池
+      maxConnectionIdleTime: 180000
+      # maxConnectionLifeTime: 30000
+      maxWaitTime: 3000
+      minSize: 5
+      maxSize: 10
+  datasource:
+    url: jdbc:mysql://rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com:3306/longvideo?useSSL=true&useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+    username: wx2016_longvideo
+    password: wx2016_longvideoP@assword1234
+    driver-class-name: com.mysql.jdbc.Driver
+    hikari:
+      connection-timeout: 30000
+      minimum-idle: 5
+      maximum-pool-size: 10
+      auto-commit: true
+      idle-timeout: 30000
+  jpa:
+    hibernate:
+      ddl-auto: validate
+    database: mysql
 xxl:
   job:
     admin: