remove 影视黑名单过滤

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/Application.java

@@ -25,8 +25,6 @@ import org.springframework.scheduling.annotation.EnableScheduling;
         "com.tzld.piaoquan.recommend.server.util",
         "com.tzld.piaoquan.recommend.server.config",
         "com.tzld.piaoquan.recommend.server.service",
-        "com.tzld.piaoquan.recommend.server.implement",
-        "com.tzld.piaoquan.recommend.server.framework.utils",
         "com.tzld.piaoquan.recommend.server.grpcservice",
         "com.tzld.piaoquan.recommend.server.remote",
         "com.tzld.piaoquan.recommend.server.web",

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -70,18 +70,7 @@ public abstract class AbstractFilterService {
         strategies.add(ServiceBeanFactory.getBean(RecommendStatusStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(AppletVideoStatusStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(RiskVideoStrategy.class));
-        switch (param.getAppType()) {
-            case 0:
-            case 4:
-                strategies.add(ServiceBeanFactory.getBean(AllowListStrategy.class));
-                break;
-            case 13:
-                break;
-            default:
-                strategies.add(ServiceBeanFactory.getBean(AllowListStrategy.class));
-                strategies.add(ServiceBeanFactory.getBean(TagStrategy.class));
-                break;
-        }
+
         if (CollectionUtils.isNotEmpty(param.getAbExpCodes()) && param.getAbExpCodes().contains("697")) {
             strategies.add(ServiceBeanFactory.getBean(VideoSourceTypeStrategy.class));
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithLevelScoreFilterService.java

@@ -1,55 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.filter;
-
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolService;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * @author dyp
- */
-@Service
-@Slf4j
-public class FlowPoolWithLevelScoreFilterService extends AbstractFilterService {
-    @Autowired
-    private FlowPoolService flowPoolService;
-
-    public FilterResult filter(FilterParam param) {
-        List<Long> videoIds = viewFilter(param);
-
-        videoIds = filterDistributeCountAndDel(param, videoIds);
-        return new FilterResult(videoIds);
-    }
-
-    private List<Long> filterDistributeCountAndDel(FilterParam param, List<Long> videoIds) {
-        if (CollectionUtils.isEmpty(videoIds)
-                || MapUtils.isEmpty(param.getFlowPoolMap())) {
-            return videoIds;
-        }
-        Map<Long, String> flowPoolMap = videoIds.stream()
-                .filter(v -> param.getFlowPoolMap().containsKey(v)
-                        && StringUtils.isNotBlank(param.getFlowPoolMap().get(v)))
-                .collect(Collectors.toMap(
-                        v -> v,
-                        v -> param.getFlowPoolMap().get(v)));
-
-        Map<Long, Integer> distributeCountMap = flowPoolService.getDistributeCountWithLevelScore(flowPoolMap);
-        List<Long> remainVideoIds = new ArrayList<>();
-        for (Long videoId : videoIds) {
-            if (distributeCountMap.get(videoId) != null
-                    && distributeCountMap.get(videoId) > 0) {
-                remainVideoIds.add(videoId);
-            }
-        }
-        return remainVideoIds;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/AllowListStrategy.java

@@ -1,101 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.filter.strategy;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Lists;
-import com.google.common.hash.Hashing;
-import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.PostConstruct;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author dyp
- */
-@Component
-@Slf4j
-public class AllowListStrategy implements FilterStrategy {
-    @Autowired
-    @Qualifier("longVideoRedisTemplate")
-    private RedisTemplate<String, String> redisTemplate;
-
-    private LoadingCache<Integer, Set<String>> allowVideoCache = CacheBuilder.newBuilder()
-            .maximumSize(1000000) // TODO 评估容量
-            .refreshAfterWrite(600, TimeUnit.SECONDS)
-            .expireAfterWrite(600, TimeUnit.SECONDS)
-            .expireAfterAccess(600, TimeUnit.SECONDS)
-            .build(new CacheLoader<Integer, Set<String>>() {
-                @Override
-                public Set<String> load(Integer idx) {
-                    String key = VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX + idx;
-                    Set<String> result = redisTemplate.opsForSet().members(key);
-                    if (CollectionUtils.isEmpty(result)) {
-                        return Collections.emptySet();
-                    }
-                    return result;
-                }
-            });
-
-    private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX = "movie:videoid:allowSet:";
-
-
-    @PostConstruct
-    public void init() {
-        // 预加载本地缓存
-        for (int i = 0; i < 100; i++) {
-            int finalI = i;
-            allowVideoCache.getUnchecked(finalI);
-        }
-    }
-
-    @Override
-    public List<Long> filter(FilterParam param) {
-        if (param == null
-                || CollectionUtils.isEmpty(param.getVideoIds())) {
-            return Collections.emptyList();
-        }
-        //不是新小程序 在白名单则不显示
-        Set<Long> retainVideoIds = new LinkedHashSet<>();
-        for (Long videoId : param.getVideoIds()) {
-            //如果不在新小程序白名单 则保留
-            if (!isMemberOfVideoAllowList(videoId)) {
-                retainVideoIds.add(videoId);
-            }
-        }
-        if (CollectionUtils.isEmpty(retainVideoIds)) {
-            return Collections.emptyList();
-        }
-
-        return Lists.newArrayList(retainVideoIds);
-    }
-
-    private boolean isMemberOfVideoAllowList(Long videoId) {
-        // TODO 优化
-        // 1. 每个idx 2000条数据 总共20w 约1.6M 可放入本地内存
-        // 2. 部分视频已下推荐，可清理
-        if (Objects.isNull(videoId)) {
-            return false;
-        }
-        try {
-            int idx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 100;
-            Set<String> allowList = allowVideoCache.get(idx);
-            if (CollectionUtils.isEmpty(allowList)) {
-                return false;
-            }
-            return allowList.contains(String.valueOf(videoId));
-        } catch (Exception e) {
-            log.error("isMemberOfVideoAllowList error {}", videoId, e);
-        }
-        return false;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/TagStrategy.java

@@ -1,115 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.filter.strategy;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Lists;
-import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
-import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.math.NumberUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.PostConstruct;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author dyp
- */
-@Component
-@Slf4j
-public class TagStrategy implements FilterStrategy {
-
-    @Autowired
-    private WxVideoTagRelRepository repository;
-
-    @Value("${video.filter.tagids:}")
-    private String videoFilterTagIds;
-
-    // 内存持久保存不淘汰
-    private LoadingCache<Long, Set<Long>> videoTagCache = CacheBuilder.newBuilder()
-            .maximumSize(100)
-            .refreshAfterWrite(60, TimeUnit.SECONDS)
-            .expireAfterWrite(60, TimeUnit.SECONDS)
-            .expireAfterAccess(60, TimeUnit.SECONDS)
-            .build(new CacheLoader<Long, Set<Long>>() {
-                @Override
-                public Set<Long> load(Long tagId) {
-                    List<WxVideoTagRel> rels = repository.findAllByTagId(tagId);
-                    return CommonCollectionUtils.toSet(rels, WxVideoTagRel::getVideoId);
-                }
-            });
-
-    @PostConstruct
-    public void init() {
-        if (StringUtils.isNotBlank(videoFilterTagIds)) {
-            // 只有涉政标签，用循环简化代码
-            String[] tags = videoFilterTagIds.split(",");
-            for (String tag : tags) {
-                if (StringUtils.isBlank(tag)) {
-                    continue;
-                }
-                videoTagCache.getUnchecked(NumberUtils.toLong(tag));
-            }
-        }
-    }
-
-    @Override
-    public List<Long> filter(FilterParam param) {
-        // TODO 主要是涉政标签
-        if (param == null
-                || CollectionUtils.isEmpty(param.getVideoIds())) {
-            return Collections.emptyList();
-        }
-
-
-        List<Long> tagIds = new ArrayList<>();
-        if (StringUtils.isBlank(videoFilterTagIds)) {
-            return param.getVideoIds();
-        }
-        String[] tags = videoFilterTagIds.split(",");
-        for (String tag : tags) {
-            if (Objects.isNull(tag) || Objects.equals("", tag)) {
-                continue;
-            }
-            tagIds.add(Long.parseLong(tag));
-        }
-
-        Set<Long> retainVideoIds = new LinkedHashSet<>();
-        Iterator<Long> iterator = param.getVideoIds().iterator();
-        while (iterator.hasNext()) {
-            Long videoId = iterator.next();
-            for (Long tagId : tagIds) {
-                if (!hasVideoRelTagId(videoId, tagId)) {
-                    retainVideoIds.add(videoId);
-                }
-            }
-        }
-
-        if (CollectionUtils.isEmpty(retainVideoIds)) {
-            return Collections.emptyList();
-        }
-
-        return Lists.newArrayList(retainVideoIds);
-    }
-
-    private boolean hasVideoRelTagId(Long videoId, Long tagId) {
-        if (Objects.isNull(videoId) || videoId <= 0L || Objects.isNull(tagId) || tagId <= 0L) {
-            return false;
-        }
-        Set<Long> videos = videoTagCache.getUnchecked(tagId);
-        if (CollectionUtils.isEmpty(videos)) {
-            return false;
-        }
-        return videos.contains(videoId);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/flowpool/FlowPoolService.java

@@ -61,6 +61,7 @@ public class FlowPoolService {
                         e -> e.getKey(),
                         e -> e.getValue()
                 ));
+        log.info("dirties size:{}", dirties.size());
         asyncDelDistributeCountWithLevel(dirties);
 
         return result;
@@ -91,33 +92,6 @@ public class FlowPoolService {
 
     }
 
-    public Map<Long, Integer> getDistributeCountWithLevelScore(Map<Long, String> videoFlowPoolMap) {
-        if (MapUtils.isEmpty(videoFlowPoolMap)) {
-            return Collections.emptyMap();
-        }
-
-        Map<Long, Integer> result = getDistributeCount(videoFlowPoolMap);
-        // 处理脏数据：分发数<0
-        Map<Long, String> dirties = videoFlowPoolMap.entrySet().stream()
-                .filter(e -> result.get(e.getKey()) <= 0)
-                .collect(Collectors.toMap(
-                        e -> e.getKey(),
-                        e -> e.getValue()
-                ));
-        asyncLocalDistributeCountWithLevelScore(dirties);
-
-        return result;
-    }
-
-    public void updateDistributeCountWithLevelScore(List<Video> videos) {
-        if (CollectionUtils.isEmpty(videos)) {
-            return;
-        }
-        Map<Long, String> removeMap = updateDistributeCount(videos);
-
-        asyncLocalDistributeCountWithLevelScore(removeMap);
-    }
-
     private void asyncLocalDistributeCountWithLevelScore(Map<Long, String> videoFlowPoolMap) {
         if (flowPoolUpgradeSwitch) {
             asyncDelDistributeCountV2(videoFlowPoolMap, (appType, level, values) -> {