compatible

recommend-server-service/pom.xml

@@ -187,8 +187,18 @@
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
         <dependency>
+<<<<<<< HEAD
             <groupId>com.xuxueli</groupId>
             <artifactId>xxl-job-core</artifactId>
+=======
+            <groupId>org.springframework.data</groupId>
+            <artifactId>spring-data-mongodb</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.mongodb</groupId>
+            <artifactId>mongo-java-driver</artifactId>
+            <version>3.8.0</version>
+>>>>>>> 2eab46b (compatible)
         </dependency>
     </dependencies>
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/CustomMongoProperties.java

@@ -0,0 +1,32 @@
+package com.tzld.piaoquan.recommend.server.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+/**
+ * @author dyp
+ */
+@ConfigurationProperties(
+        prefix = "spring.data.mongodb"
+)
+@Data
+public class CustomMongoProperties {
+    private String host;
+    private Integer port;
+    private String uri;
+    private String database;
+    private String username;
+    private char[] password;
+    private String replicaSetName;
+
+    // 连接池配置 单位ms
+    private long maxConnectionIdleTime;
+    private long maxConnectionLifeTime;
+    private long maxWaitTime;
+    private int minSize;
+    private int maxSize;
+
+    // socket
+    private int readTimeout;
+    private int connectTimeout;
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/MongoTemplateConfig.java

@@ -0,0 +1,70 @@
+package com.tzld.piaoquan.recommend.server.config;
+
+import com.mongodb.MongoClientSettings;
+import com.mongodb.MongoCredential;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.mongodb.MongoDatabaseFactory;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory;
+import org.springframework.data.mongodb.core.convert.DbRefResolver;
+import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver;
+import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper;
+import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
+import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author dyp
+ */
+@Configuration
+@EnableConfigurationProperties(CustomMongoProperties.class)
+public class MongoTemplateConfig {
+    @Bean
+    public MongoClient mongoClient(CustomMongoProperties properties) {
+        MongoClientSettings.Builder builder = MongoClientSettings.builder();
+        builder.credential(MongoCredential.createCredential(properties.getUsername(), properties.getDatabase(), properties.getPassword()));
+        builder.applyToConnectionPoolSettings(b -> {
+            b.minSize(properties.getMinSize());
+            b.maxSize(properties.getMaxSize());
+            b.maxWaitTime(properties.getMaxWaitTime(), TimeUnit.MILLISECONDS);
+            b.maxConnectionIdleTime(properties.getMaxConnectionIdleTime(), TimeUnit.MILLISECONDS);
+            b.maxConnectionLifeTime(properties.getMaxConnectionLifeTime(), TimeUnit.MILLISECONDS);
+//            b.(properties.getMaintenanceFrequency());
+//            b.maintenanceInitialDelay();
+        });
+//        builder.applyToServerSettings(b -> {
+//            b.heartbeatFrequency()
+//        });
+        builder.applyToSocketSettings(b -> {
+            b.connectTimeout(properties.getConnectTimeout(), TimeUnit.MILLISECONDS);
+            b.readTimeout(properties.getReadTimeout(), TimeUnit.MILLISECONDS);
+        });
+        return MongoClients.create(builder.build());
+    }
+
+    @Bean
+    public MongoDatabaseFactory mongoDatabaseFactory(MongoClient client, CustomMongoProperties properties) {
+        SimpleMongoClientDatabaseFactory mongoDbFactory = new SimpleMongoClientDatabaseFactory(client,
+                properties.getDatabase());
+
+        return mongoDbFactory;
+    }
+
+    @Bean(name = "mongoTemplate")
+    public MongoTemplate mongoTemplate(MongoDatabaseFactory factory) {
+
+        // 去掉 _class 字段
+        DbRefResolver dbRefResolver = new DefaultDbRefResolver(factory);
+        MappingMongoConverter converter = new MappingMongoConverter(dbRefResolver, new MongoMappingContext());
+        converter.setTypeMapper(new DefaultMongoTypeMapper(null));
+
+        MongoTemplate mongoTemplate = new MongoTemplate(factory, converter);
+        return mongoTemplate;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -1,7 +1,5 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
-import com.google.common.base.Stopwatch;
-import com.thoughtworks.xstream.mapper.Mapper;
 import com.tzld.piaoquan.recommend.server.service.PreViewedService;
 import com.tzld.piaoquan.recommend.server.service.ViewedService;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
@@ -21,7 +19,6 @@ import java.util.stream.Collectors;
  */
 @Slf4j
 public abstract class AbstractFilterService implements FilterService {
-    public Integer forceTruncation;
     @Autowired
     private PreViewedService preViewedService;
 
@@ -29,14 +26,18 @@ public abstract class AbstractFilterService implements FilterService {
     private ViewedService viewedService;
 
 
-
     protected List<Long> viewFilter(FilterParam param) {
         // log.info("filterParam={}", JSONUtils.toJson(param));
         // 风险过滤
         List<Long> videoIds = filterWithRiskVideo(param.getRiskFilterFlag(),
                 param.getAppType(), param.getRegionCode(), param.getAppRegionFiltered(), param.getVideosWithRisk(),
+<<<<<<< HEAD
                 param.getVideoIds());
         // log.info("filterByRiskVideos videoIds={}", JSONUtils.toJson(videoIds));
+=======
+                param.getVideoIds(), param.getForceTruncation());
+        log.info("filterByRiskVideos videoIds={}", JSONUtils.toJson(videoIds));
+>>>>>>> 5917e10 (compatible)
         videoIds = filterByPreViewed(param.getAppType(), param.getMid(), videoIds);
         // log.info("filterByPreViewed videoIds={}", JSONUtils.toJson(videoIds));
         videoIds = filterByViewed(param.getAppType(), param.getMid(), param.getUid(), videoIds);
@@ -73,22 +74,23 @@ public abstract class AbstractFilterService implements FilterService {
                                            String regionCode,
                                            Map<Integer, List<String>> rules,
                                            List<Long> videosWithRisk,
-                                           List<Long> videoIds){
-        if (!riskFlag){
-            return this.truncation(videoIds);
+                                           List<Long> videoIds,
+                                           int forceTruncation) {
+        if (!riskFlag) {
+            return this.truncation(videoIds, forceTruncation);
         }
         // 1 判断是否过滤,不展示的app+区域列表。
         boolean filterFlag;
-        if (rules.containsKey(appType)){
+        if (rules.containsKey(appType)) {
             filterFlag = false;
-            if (rules.get(appType).contains(regionCode)){
+            if (rules.get(appType).contains(regionCode)) {
                 filterFlag = true;
             }
         } else {
             filterFlag = true;
         }
-        if (!filterFlag){
-            return this.truncation(videoIds);
+        if (!filterFlag) {
+            return this.truncation(videoIds, forceTruncation);
         }
         // 2 开始过滤。
         List<Long> videoIdNew = new ArrayList<>();
@@ -97,13 +99,14 @@ public abstract class AbstractFilterService implements FilterService {
                 videoIdNew.add(videoId);
             }
         }
-        return this.truncation(videoIdNew);
+        return this.truncation(videoIdNew, forceTruncation);
     }
-    private List<Long> truncation(List<Long> videoIds){
-        if (this.forceTruncation == null){
+
+    private List<Long> truncation(List<Long> videoIds, int forceTruncation) {
+        if (forceTruncation == 0) {
             return videoIds;
-        }else{
-            return videoIds.subList(0, Math.min(this.forceTruncation, videoIds.size()));
+        } else {
+            return videoIds.subList(0, Math.min(forceTruncation, videoIds.size()));
         }
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterParam.java

@@ -21,4 +21,5 @@ public class FilterParam {
     private Map<Integer, List<String>> appRegionFiltered;
     private List<Long> videosWithRisk;
     private String regionCode;
+    private int forceTruncation;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterStrategy.java

@@ -0,0 +1,9 @@
+package com.tzld.piaoquan.recommend.server.service.filter;
+
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+
+import java.util.List;
+
+public interface FilterStrategy {
+    List<Long> filter(FilterParam param);
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/RegionFilterService.java

@@ -1,7 +1,5 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
-import lombok.Getter;
-import lombok.Setter;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
@@ -11,11 +9,6 @@ import java.util.List;
  */
 @Service
 public class RegionFilterService extends AbstractFilterService {
-
-    public RegionFilterService(Integer forceTruncation){
-        this.forceTruncation = forceTruncation;
-    }
-    public RegionFilterService(){}
     public FilterResult filter(FilterParam param) {
         List<Long> videoIds = viewFilter(param);
         return new FilterResult(videoIds);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/VideoView.java

@@ -0,0 +1,35 @@
+package com.tzld.piaoquan.recommend.server.service.filter;
+
+import lombok.Data;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.index.IndexDirection;
+import org.springframework.data.mongodb.core.index.Indexed;
+import org.springframework.data.mongodb.core.mapping.Document;
+import org.springframework.data.mongodb.core.mapping.Field;
+
+import java.util.Date;
+
+@Data
+@Document(collection = "video_view")
+public class VideoView {
+    @Id
+    private String id;
+    @Indexed(name = "video_id", direction = IndexDirection.ASCENDING)
+    @Field("video_id")
+    private Long videoId;
+    @Field("uid")
+    private String uid;
+    @Field("action_type")
+    private Integer actionType;
+    @Field("create_time")
+    private Long createTime;
+    @Field("create_date")
+    private Date createDate;
+    @Field("count")
+    private Integer count;
+    @Field("source")
+    private Integer source;
+    @Field("status")
+    private Integer status;
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/PreViewedStrategy.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.tzld.piaoquan.recommend.server.service.PreViewedService;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Component
+public class PreViewedStrategy implements FilterStrategy {
+    @Autowired
+    private PreViewedService preViewedService;
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+
+        if (param == null) {
+            return Collections.emptyList();
+        }
+        if (StringUtils.isBlank(param.getMid())
+                || CollectionUtils.isEmpty(param.getVideoIds())) {
+            return param.getVideoIds();
+        }
+        Set<Long> preViewedVideoIds = preViewedService.getVideoIds(param.getAppType(), param.getMid());
+        return param.getVideoIds().stream()
+                .filter(l -> !preViewedVideoIds.contains(l))
+                .collect(Collectors.toList());
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/RiskVideoStrategy.java

@@ -0,0 +1,66 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author dyp
+ */
+@Component
+public class RiskVideoStrategy implements FilterStrategy {
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+
+        if (CollectionUtils.isEmpty(param.getVideoIds())) {
+            return Collections.emptyList();
+        }
+        boolean riskFlag = param.getRiskFilterFlag();
+        int appType = param.getAppType();
+        String regionCode = param.getRegionCode();
+        Map<Integer, List<String>> rules = param.getAppRegionFiltered();
+        List<Long> videosWithRisk = param.getVideosWithRisk();
+        List<Long> videoIds = param.getVideoIds();
+
+
+        if (!riskFlag) {
+            return this.truncation(param.getVideoIds(), param.getForceTruncation());
+        }
+        // 1 判断是否过滤,不展示的app+区域列表。
+        boolean filterFlag = false;
+        if (rules.containsKey(appType)) {
+            filterFlag = false;
+            if (rules.get(appType).contains(regionCode)) {
+                filterFlag = true;
+            }
+        } else {
+            filterFlag = true;
+        }
+        if (!filterFlag) {
+            return this.truncation(videoIds, param.getForceTruncation());
+        }
+        // 2 开始过滤。
+        List<Long> videoIdNew = new ArrayList<>();
+        for (Long videoId : videoIds) {
+            if (!videosWithRisk.contains(videoId)) {
+                videoIdNew.add(videoId);
+            }
+        }
+        return this.truncation(videoIdNew, param.getForceTruncation());
+    }
+
+    private List<Long> truncation(List<Long> videoIds, int forceTruncation) {
+        if (forceTruncation == 0) {
+            return videoIds;
+        } else {
+            return videoIds.subList(0, Math.min(forceTruncation, videoIds.size()));
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/ViewedStrategy.java

@@ -0,0 +1,77 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import com.tzld.piaoquan.recommend.server.service.filter.VideoView;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.dao.DataAccessException;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.redis.core.RedisOperations;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.core.SessionCallback;
+import org.springframework.data.redis.core.SetOperations;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author dyp
+ */
+@Component
+public class ViewedStrategy implements FilterStrategy {
+    @Autowired
+    private RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private MongoTemplate mongoTemplate;
+
+    private String keyFormat = "user:exclude:videoidset:%s";
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+        String user = StringUtils.isNotBlank(param.getUid()) ? param.getUid() : param.getMid();
+        if (StringUtils.isBlank(user)) {
+            return param.getVideoIds();
+        }
+        String key = String.format(keyFormat, param.getMid());
+        Set<String> viewedVideoIds = redisTemplate.opsForSet().members(key);
+        if (CollectionUtils.isEmpty(viewedVideoIds)) {
+            // 从mongo取曝光数据
+            Criteria criteria = new Criteria();
+            criteria.and("uid").is(user);
+            Query query = new Query();
+            query.addCriteria(criteria);
+            List<VideoView> list = mongoTemplate.find(query, VideoView.class);
+            //TODO 为什么限制最多10000条？是不是限制近几天更合适？
+            if (CollectionUtils.isNotEmpty(list)) {
+                int limit = 10000;
+                for (int i = list.size() - 1; i >= 0 && limit-- > 0; i--) {
+                    viewedVideoIds.add(String.valueOf(list.get(i).getVideoId()));
+                }
+                redisTemplate.executePipelined(new SessionCallback<String>() {
+                    @Override
+                    public <A, B> String execute(RedisOperations<A, B> redisOperations) throws DataAccessException {
+                        SetOperations<String, String> operations =
+                                (SetOperations<String, String>) redisOperations.opsForSet();
+                        operations.add(key, viewedVideoIds.toArray(new String[viewedVideoIds.size()]));
+                        redisTemplate.expire(key, 360 * 3600, TimeUnit.SECONDS);
+                        return null;
+                    }
+                });
+            } else {
+
+            }
+
+//        logger.info("getUserViewVideos execute time = {}", stopwatch.stop());
+            return list;
+        }
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractRegionRecallStrategy.java

@@ -4,6 +4,7 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.collect.Lists;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
 import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
 import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
 import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
@@ -134,9 +135,17 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
                 lastVideoId = t.getValue();
                 videoMap.put(NumberUtils.toLong(t.getValue(), 0), t.getScore());
             }
+<<<<<<< HEAD
             filterService.forceTruncation = 20;
             FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet())));
             // log.info("recall filterResult={}", JSONUtils.toJson(filterResult));
+=======
+            FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
+            filterParam.setForceTruncation(20);
+            
+            FilterResult filterResult = filterService.filter(filterParam);
+            log.info("recall filterResult={}", JSONUtils.toJson(filterResult));
+>>>>>>> 5917e10 (compatible)
 
             if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
                 filterResult.getVideoIds().stream().forEach(vid -> {

recommend-server-service/src/main/resources/application-dev.yml

@@ -24,6 +24,7 @@ spring:
         max-wait: -1
         max-idle: 8
         min-idle: 0
+<<<<<<< HEAD
   feature-redis:
     hostName: r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com
     port: 6379
@@ -35,6 +36,22 @@ spring:
         max-wait: -1
         max-idle: 8
         min-idle: 0
+=======
+  data:
+    mongodb:
+      host: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
+      port: 3717
+      database: longvideo
+      username: lv
+      password: lv@2018
+      replicaSetName: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com:3717,dds-bp1de4fc73029b242144.mongodb.rds.aliyuncs.com:3717
+      # 连接池
+      maxConnectionIdleTime: 30000
+      maxConnectionLifeTime: 30000
+      maxWaitTime: 30000
+      minSize: 5
+      maxSize: 10
+>>>>>>> 2eab46b (compatible)
 
 xxl:
   job: