ソースを参照

Merge branch 'feature_filter' of algorithm/recommend-server into master

dingyunpeng 1 年間 前
コミット
6668fa2ad2

+ 5 - 1
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/repository/WxVideoTagRel.java

@@ -21,8 +21,12 @@ public class WxVideoTagRel implements Serializable {
 
     private Date createTime;
 
+    public WxVideoTagRel() {
+
+    }
+
     @Data
-    public class WxVideoTagRelId implements Serializable {
+    public static class WxVideoTagRelId implements Serializable {
         private Long videoId;
         private Long tagId;
 

+ 2 - 0
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/repository/WxVideoTagRelRepository.java

@@ -8,4 +8,6 @@ import java.util.List;
 @Repository
 public interface WxVideoTagRelRepository extends JpaRepository<WxVideoTagRel, Long> {
     List<WxVideoTagRel> findAllByTagId(Long tagId);
+
+    List<WxVideoTagRel> findAllByTagIdIn(List<Long> tagId);
 }

+ 4 - 4
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/ViewedService.java

@@ -1,6 +1,5 @@
 package com.tzld.piaoquan.recommend.server.service;
 
-import com.google.common.base.Stopwatch;
 import com.google.common.collect.Lists;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.util.HttpClientFactory;
@@ -29,6 +28,8 @@ public class ViewedService {
 
     @Value("${video.filter.url:}")
     private String videoFilterUrl;
+    @Value("${securityAbExpCode:}")
+    private String securityAbExpCode;
 
     private final CloseableHttpClient client = HttpClientFactory.create(1000, 1000, 200, 200, 0, 1000);
 
@@ -44,11 +45,10 @@ public class ViewedService {
 
     // TODO 如果过滤失败,那么认为所有视频都被过滤掉
     public List<Long> filterViewedVideo(int appType, String mid, String uid, List<Long> videoIds, String cityCode,
-                                        String abCode, Long hotSceneType) {
+                                        Set<String> abExpCodes, Long hotSceneType) {
 
-        Stopwatch stopwatch = Stopwatch.createStarted();
         List<Integer> viewedTypes = new ArrayList<>(viewedTypesMap.getOrDefault(appType, defaultViewedTypes));
-        if ("60106".equals(abCode)) {
+        if (abExpCodes.contains(securityAbExpCode)) {
             viewedTypes.add(8);
         }
         CloseableHttpResponse chr = null;

+ 3 - 3
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -119,7 +119,7 @@ public abstract class AbstractFilterService {
         for (final List<Long> ids : chunks) {
             Future<List<Long>> future = pool.submit(() ->
                     viewedService.filterViewedVideo(param.getAppType(), param.getMid(), param.getUid(), ids, param.getCityCode(),
-                            param.getAbCode(), param.getHotSceneType()));
+                            param.getAbExpCodes(), param.getHotSceneType()));
             futures.add(future);
         }
         try {
@@ -150,7 +150,7 @@ public abstract class AbstractFilterService {
             return videoIds;
         }
         return viewedService.filterViewedVideo(param.getAppType(), param.getMid(), param.getUid(), videoIds, param.getCityCode(),
-                param.getAbCode(), param.getHotSceneType());
+                param.getAbExpCodes(), param.getHotSceneType());
 
     }
 
@@ -227,7 +227,6 @@ public abstract class AbstractFilterService {
         List<Future<List<Long>>> futures = new ArrayList<>();
         for (final FilterStrategy strategy : strategies) {
             Future<List<Long>> future = pool.submit(() -> {
-                Stopwatch stopwatch = Stopwatch.createStarted();
                 List<Long> result = strategy.filter(param);
                 cdl.countDown();
                 return result;
@@ -261,6 +260,7 @@ public abstract class AbstractFilterService {
 
     private List<FilterStrategy> getStrategies(FilterParam param) {
         List<FilterStrategy> strategies = new ArrayList<>();
+        strategies.add(ServiceBeanFactory.getBean(SecurityStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(PreViewedStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(ViewedStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(RecommendStatusStrategy.class));

+ 110 - 0
recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/SecurityStrategy.java

@@ -0,0 +1,110 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
+import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Component
+@Slf4j
+public class SecurityStrategy implements FilterStrategy {
+    @Value("#{'${block.hotscenetype.list:}'.split(',')}")
+    private Set<Long> excludeScenes;
+    @ApolloJsonValue("${video.filter.city.tagid.json:{}}")
+    private Map<String, List<Long>> videoFilterCityTagIdMap;
+    @Value("${securityAbExpCode:625}")
+    private String securityAbExpCode;
+    @Value("${securityGlobalSwitch:false}")
+    private boolean securityGlobalSwitch;
+
+    @Autowired
+    private WxVideoTagRelRepository wxVideoTagRelRepository;
+
+    // 内存持久保存不淘汰
+    private LoadingCache<String, Set<Long>> videoCache = CacheBuilder.newBuilder()
+            .maximumSize(100)
+            .refreshAfterWrite(60, TimeUnit.SECONDS)
+            .expireAfterWrite(60, TimeUnit.SECONDS)
+            .expireAfterAccess(60, TimeUnit.SECONDS)
+            .build(new CacheLoader<String, Set<Long>>() {
+                @Override
+                public Set<Long> load(String cityCode) {
+                    if (MapUtils.isEmpty(videoFilterCityTagIdMap)) {
+                        return new HashSet<>();
+                    }
+                    List<Long> tagIds = videoFilterCityTagIdMap.get(cityCode);
+                    if (CollectionUtils.isEmpty(tagIds)) {
+                        return new HashSet<>();
+                    }
+                    List<WxVideoTagRel> rels = wxVideoTagRelRepository.findAllByTagIdIn(tagIds);
+                    return CommonCollectionUtils.toSet(rels, WxVideoTagRel::getVideoId);
+                }
+            });
+
+    @PostConstruct
+    public void init() {
+        if (MapUtils.isEmpty(videoFilterCityTagIdMap)) {
+            return;
+        }
+        for (String cityCode : videoFilterCityTagIdMap.keySet()) {
+            videoCache.getUnchecked(cityCode);
+        }
+    }
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+
+        if (param == null) {
+            return Collections.emptyList();
+        }
+        if (CollectionUtils.isEmpty(param.getVideoIds())) {
+            return param.getVideoIds();
+        }
+
+        boolean hit = securityGlobalSwitch
+                || CommonCollectionUtils.contains(param.getAbExpCodes(), securityAbExpCode);
+        if (!hit) {
+            return param.getVideoIds();
+        }
+
+        if (CollectionUtils.isEmpty(excludeScenes)
+                || !CommonCollectionUtils.contains(excludeScenes, param.getHotSceneType())) {
+
+            if (MapUtils.isEmpty(videoFilterCityTagIdMap)
+                    || !videoFilterCityTagIdMap.containsKey(param.getCityCode())) {
+                return param.getVideoIds();
+            }
+
+            Set<Long> filterVideos = videoCache.getUnchecked(param.getCityCode());
+            if (CollectionUtils.isEmpty(filterVideos)) {
+                return param.getVideoIds();
+            }
+
+            List<Long> result = param.getVideoIds().stream()
+                    .filter(l -> !filterVideos.contains(l))
+                    .collect(Collectors.toList());
+
+            return result;
+        }
+        return param.getVideoIds();
+    }
+}