merge filter service

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/VovLowerStrategy.java

@@ -1,194 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.filter.strategy;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.TypeReference;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.Nonnull;
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-@Slf4j
-@Component
-public class VovLowerStrategy implements FilterStrategy {
-
-    @ApolloJsonValue("${vov.filter.condition:{}}")
-    private Map<String, Double> vovFilterCondition;
-
-    @Autowired
-    @Qualifier("redisTemplate")
-    private RedisTemplate<String, String> redisTemplate;
-
-    private static final String KEY_FORMAT = "redis:vid_vov_daily4filter:%d";
-
-    private static final String LOWER_VOV_VID_KEY_FORMAT = "redis:lower_vov_vid:%s";
-    private static final DateTimeFormatter YYYY_MM_DD = DateTimeFormatter.ofPattern("yyyyMMdd");
-
-    /**
-     * 低VOV视频ID集合缓存, 五分钟有效期
-     */
-    private final LoadingCache<String, Set<Long>> lowerVovVidCache = CacheBuilder.newBuilder()
-            .expireAfterWrite(5, TimeUnit.MINUTES)
-            .build(new CacheLoader<String, Set<Long>>() {
-                @Override
-                public Set<Long> load(@Nonnull String key) {
-                    Set<String> members = redisTemplate.opsForSet().members(key);
-                    if (CollectionUtils.isEmpty(members)) {
-                        return Collections.emptySet();
-                    }
-                    return members.stream()
-                            .map(Long::parseLong)
-                            .collect(Collectors.toSet());
-                }
-            });
-
-    @Override
-    public List<Long> filter(FilterParam param) {
-
-        List<Long> videoIds = param.getVideoIds();
-        if (CollectionUtils.isEmpty(videoIds)) {
-            log.info("VOV过滤 -- videoIds为空，跳过");
-            return new ArrayList<>();
-        }
-
-        List<Long> result = new ArrayList<>(videoIds.size());
-        List<Long> removeIds = new ArrayList<>();
-
-        String dt = LocalDate.now().format(YYYY_MM_DD);
-        String redisKey = String.format(LOWER_VOV_VID_KEY_FORMAT, dt);
-        try {
-            Set<Long> lowerVovVidSet = lowerVovVidCache.get(redisKey);
-
-            for (Long videoId : videoIds) {
-                if (lowerVovVidSet.contains(videoId)) {
-                    removeIds.add(videoId);
-                } else {
-                    result.add(videoId);
-                }
-            }
-
-        } catch (Exception e) {
-            log.error("VOV过滤V2 -- 从缓存中获取低VOV视频ID集合异常: ", e);
-        }
-
-        log.info("VOV过滤V2 -- 本次召回的ID列表为: {}, 被过滤掉的ID列表为: {}, 剩下的ID列表为: {}", videoIds, removeIds, result);
-
-        return result;
-    }
-
-    public List<Long> filterV1(List<Long> videoIds) {
-
-        List<Long> result = new ArrayList<>(videoIds.size());
-
-        List<String> redisKeys = videoIds.stream()
-                .map(r -> String.format(KEY_FORMAT, r))
-                .collect(Collectors.toList());
-        List<String> vovInfos = redisTemplate.opsForValue().multiGet(redisKeys);
-        if (CollectionUtils.isEmpty(vovInfos) || vovInfos.size() != videoIds.size()) {
-            log.info("VOV过滤 -- 获取到的视频VOV信息为空或长度与videoIds长度不一致，跳过: {}, {}", videoIds.size(), vovInfos.size());
-            return result;
-        }
-
-        List<Long> removeIds = new ArrayList<>();
-
-        for (int i = 0; i < videoIds.size(); i++) {
-            Long videoId = videoIds.get(i);
-            String vovInfo = vovInfos.get(i);
-            if (isShouldFilter(vovInfo)) {
-                removeIds.add(videoId);
-                continue;
-            }
-
-            result.add(videoId);
-        }
-
-        log.info("VOV过滤V1 -- 本次召回的ID列表为: {}, 被过滤掉的ID列表为: {}, 剩下的ID列表为: {}", videoIds, removeIds, result);
-
-        return result;
-    }
-
-    private boolean isShouldFilter(String vovInfo) {
-        try {
-
-            if (StringUtils.isEmpty(vovInfo)) {
-                return false;
-            }
-
-            Map<String, Double> vovInfoMap = JSON.parseObject(vovInfo, new TypeReference<Map<String, Double>>() {
-            });
-            if (MapUtils.isEmpty(vovInfoMap)) {
-                return false;
-            }
-
-            if (isNewVideo(vovInfoMap) || isUptrendVideo(vovInfoMap)) {
-                return false;
-            }
-
-            // 预估VOV
-            double t1Vov0 = vovInfoMap.getOrDefault("1_vov0", 0d);
-            double t2Vov0 = vovInfoMap.getOrDefault("2_vov0", 0d);
-
-            double estimated2Vov0Alpha = vovFilterCondition.getOrDefault("estimated_2_vov_0_alpha", 0.2d);
-            double estimated1Vov0Alpha = vovFilterCondition.getOrDefault("estimated_1_vov_0_alpha", 0.8d);
-            double estimatedVovThreshold = vovFilterCondition.getOrDefault("estimated_vov_threshold", 0.19d);
-
-            double estimatedVov = t1Vov0;
-            if (t2Vov0 != 0) {
-                estimatedVov = estimated2Vov0Alpha * t2Vov0 + estimated1Vov0Alpha * t1Vov0;
-            }
-
-            return estimatedVov < estimatedVovThreshold;
-        } catch (Exception e) {
-            log.info("VOV过滤 -- 异常: ", e);
-        }
-        return false;
-    }
-
-    // 三天曝光都小于阈值的 认定为新视频
-    private boolean isNewVideo(Map<String, Double> vovInfoMap) {
-        double t2ViewPvCondition = vovFilterCondition.getOrDefault("t_2_view_pv", 50d);
-        double t3ViewPvCondition = vovFilterCondition.getOrDefault("t_3_view_pv", 50d);
-        double t4ViewPvCondition = vovFilterCondition.getOrDefault("t_4_view_pv", 50d);
-        double t2ViewPv = vovInfoMap.getOrDefault("2_vov0_分母", 0d);
-        double t3ViewPv = vovInfoMap.getOrDefault("3_vov0_分母", 0d);
-        double t4ViewPv = vovInfoMap.getOrDefault("4_vov0_分母", 0d);
-
-        return t2ViewPv < t2ViewPvCondition && t3ViewPv < t3ViewPvCondition && t4ViewPv < t4ViewPvCondition;
-    }
-
-    // 最近两天的vov满足并值，且昨天的vov-前天的vov 达到一定的涨幅认定为上升趋势视频
-    private boolean isUptrendVideo(Map<String, Double> vovInfoMap) {
-
-        double t1VovCondition = vovFilterCondition.getOrDefault("t_1_vov", 0.2d);
-        double t2VovCondition = vovFilterCondition.getOrDefault("t_2_vov", 0d);
-        double vovDiff = vovFilterCondition.getOrDefault("12_vov_diff", 0.1);
-
-        double t1Vov0 = vovInfoMap.getOrDefault("1_vov0", 0d);
-        double t2Vov0 = vovInfoMap.getOrDefault("2_vov0", 0d);
-        double t12VovDiff = t1Vov0 - t2Vov0;
-
-        return t2Vov0 > t2VovCondition && t1Vov0 > t1VovCondition && t12VovDiff > vovDiff;
-
-    }
-
-    @Scheduled(cron = "0 0/5 * * * ? ")
-    public void cronSync() {
-    }
-}