filter

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/AllowListStrategy.java

@@ -1,67 +1,78 @@
 package com.tzld.piaoquan.recommend.server.service.filter.strategy;
 
+import com.google.common.collect.Lists;
 import com.google.common.hash.Hashing;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
-import com.tzld.piaoquan.recommend.server.service.PreViewedService;
 import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
 import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Component;
 
 import java.util.*;
-import java.util.stream.Collectors;
 
 /**
  * @author dyp
  */
 @Component
+@Slf4j
 public class AllowListStrategy implements FilterStrategy {
     @Autowired
     private RedisTemplate<String, String> redisTemplate;
 
     private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_PREFIX = "movie:videoid:allowlist:";
     private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX = "movie:videoid:allowSet:";
+    private static final String VIDEO_ALLOW_LIST_BITMAP_KEY = "movie.store.mp.allowlist.videoid.bitmap";
+    private static final String RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY = "mp:religion:allowlist:videoid:bitmap";
+    @Value("${movie.videoid.allowlist.compatible:1}")
+    private Integer mvalCompatible;
 
     @Override
+    // TODO 未找到优化方法 暂时保留原代码
     public List<Long> filter(FilterParam param) {
 
         if (param == null
-                || CollectionUtils.isEmpty(param.getVideoIds()) {
+                || CollectionUtils.isEmpty(param.getVideoIds())) {
             return Collections.emptyList();
         }
         //不是新小程序 在白名单则不显示
+        Set<Long> retainVideoIds = new LinkedHashSet<>();
         if (param.getAppType() != AppTypeEnum.WAN_NENG_VIDEO.getCode()
                 && param.getAppType() != AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
                 && param.getAppType() != AppTypeEnum.ZUI_JING_QI.getCode()
                 && param.getAppType() != AppTypeEnum.H5.getCode()) {
 
-            Iterator<Long> iterator = videoIds.iterator();
+            Iterator<Long> iterator = param.getVideoIds().iterator();
             while (iterator.hasNext()) {
                 Long videoId = iterator.next();
-                //如果在白名单 则移除
-                if (movieStoreMpService.isMemberOfAllowList(videoId)) {
-                    iterator.remove();
+                //如果不在新小程序白名单 则保留
+                if (!isMemberOfVideoAllowList(videoId)) {
+                    retainVideoIds.add(videoId);
                 }
             }
-        } else if (param.getAppType() != AppTypeEnum.WAN_NENG_VIDEO.getCode()
-                && param.getAppType() != AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
-                && param.getAppType() != AppTypeEnum.ZUI_JING_QI.getCode()
-                && param.getAppType() != AppTypeEnum.H5.getCode()) {
+        } else if (param.getAppType() == AppTypeEnum.WAN_NENG_VIDEO.getCode()
+                || param.getAppType() == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
+                || param.getAppType() == AppTypeEnum.ZUI_JING_QI.getCode()
+                || param.getAppType() == AppTypeEnum.H5.getCode()) {
+            Iterator<Long> iterator = param.getVideoIds().iterator();
             while (iterator.hasNext()) {
                 Long videoId = iterator.next();
-                //如果在宗教白名单 则移除
-                if (movieStoreMpService.isMemberOfReligionVideoAllowList(videoId)) {
-                    iterator.remove();
+                //如果不在宗教白名单 则保留
+                if (!isMemberOfReligionVideoAllowList(videoId)) {
+                    retainVideoIds.add(videoId);
                 }
             }
         }
-        return videoIds;
+        if (CollectionUtils.isEmpty(retainVideoIds)) {
+            return Collections.emptyList();
+        }
+
+        return Lists.newArrayList(retainVideoIds);
     }
 
-    // TODO 未找到优化方法 暂时保留源代码
     private boolean isMemberOfVideoAllowList(Long videoId) {
         if (Objects.isNull(videoId)) {
             return false;
@@ -73,14 +84,26 @@ public class AllowListStrategy implements FilterStrategy {
             // 兼容旧 key
             if (Objects.equals(1, mvalCompatible) && !result) {
                 int idx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 10;
-                result = redisUtils.getBit(VIDEO_ALLOW_LIST_BITMAP_KEY_PREFIX + idx, videoId);
+                result = redisTemplate.opsForValue().getBit(VIDEO_ALLOW_LIST_BITMAP_KEY_PREFIX + idx, videoId);
             }
             if (Objects.equals(1, mvalCompatible) && !result) {
-                result = redisUtils.getBit(VIDEO_ALLOW_LIST_BITMAP_KEY, videoId);
+                result = redisTemplate.opsForValue().getBit(VIDEO_ALLOW_LIST_BITMAP_KEY, videoId);
             }
             return result;
         } catch (Exception e) {
-            LOGGER.error("isMemberOfVideoAllowList error {}", videoId, e);
+            log.error("isMemberOfVideoAllowList error {}", videoId, e);
+        }
+        return false;
+    }
+
+    public boolean isMemberOfReligionVideoAllowList(Long videoId) {
+        if (Objects.isNull(videoId)) {
+            return false;
+        }
+        try {
+            return redisTemplate.opsForValue().getBit(RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY, videoId);
+        } catch (Exception e) {
+            log.error("isMemberOfReligionVideoAllowList error {}", e, videoId);
         }
         return false;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/TagStrategy.java

@@ -0,0 +1,120 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Component
+@Slf4j
+public class TagStrategy implements FilterStrategy {
+    @Autowired
+    private RedisTemplate<String, String> redisTemplate;
+
+    @Value("${video.filter.tagids:}")
+    private String videoFilterTagIds;
+
+    private Map<Long, Set<Long>> videoTagCache = new ConcurrentHashMap<>();
+
+    @PostConstruct
+    public void init() {
+        initCacheByValue();
+    }
+
+    @Scheduled(cron = "0 0 0/1 * * ? ")
+    public void cornInit() {
+        initCacheByValue();
+    }
+
+    public void initCacheByValue() {
+        long l = System.currentTimeMillis();
+        if (org.apache.commons.lang.StringUtils.isNotBlank(videoFilterTagIds)) {
+            Map<Long, Set<Long>> tmp = new ConcurrentHashMap<>();
+            String[] tags = videoFilterTagIds.split(",");
+            for (String tag : tags) {
+                if (StringUtils.isBlank(tag)) {
+                    continue;
+                }
+                Long tagId = Long.valueOf(tag);
+                WxVideoTagRelExample wxVideoTagRelExample = new WxVideoTagRelExample();
+                wxVideoTagRelExample.createCriteria().andTagIdEqualTo(tagId);
+                List<WxVideoTagRel> wxVideoTagRels = wxVideoTagRelMapper.selectByExample(wxVideoTagRelExample);
+                tmp.put(tagId,
+                        wxVideoTagRels.stream().map(WxVideoTagRel::getVideoId).collect(Collectors.toSet()));
+            }
+            videoTagCache = tmp;
+        } else {
+            videoTagCache = new ConcurrentHashMap<>();
+        }
+        long useTime = System.currentTimeMillis() - l;
+        log.info("update filter video tags data success,all data size:" +
+                videoTagCache.keySet().size() + ",time:" + useTime);
+    }
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+        // TODO 主要是涉政标签
+        if (param == null
+                || CollectionUtils.isEmpty(param.getVideoIds())) {
+            return Collections.emptyList();
+        }
+
+
+        List<Long> tagIds = new ArrayList<>();
+        if (StringUtils.isBlank(videoFilterTagIds)) {
+            return param.getVideoIds();
+        }
+        String[] tags = videoFilterTagIds.split(",");
+        for (String tag : tags) {
+            if (Objects.isNull(tag) || Objects.equals("", tag)) {
+                continue;
+            }
+            tagIds.add(Long.parseLong(tag));
+        }
+
+        Set<Long> retainVideoIds = new LinkedHashSet<>();
+        Iterator<Long> iterator = param.getVideoIds().iterator();
+        while (iterator.hasNext()) {
+            Long videoId = iterator.next();
+            for (Long tagId : tagIds) {
+                if (!hasVideoRelTagId(videoId, tagId)) {
+                    retainVideoIds.add(videoId);
+                }
+            }
+        }
+
+
+        if (CollectionUtils.isEmpty(retainVideoIds)) {
+            return Collections.emptyList();
+        }
+
+        return Lists.newArrayList(retainVideoIds);
+    }
+
+    private boolean hasVideoRelTagId(Long videoId, Long tagId) {
+        if (Objects.isNull(videoId) || videoId <= 0L || Objects.isNull(tagId) || tagId <= 0L) {
+            return false;
+        }
+        Set<Long> videosByTag = videoTagCache.get(tagId);
+        if (Objects.isNull(videosByTag) || videosByTag.isEmpty()) {
+            return false;
+        }
+        return videosByTag.contains(videoId);
+    }
+
+}