|
@@ -1,194 +0,0 @@
|
|
|
-package com.tzld.piaoquan.recommend.server.service.filter.strategy;
|
|
|
-
|
|
|
-import com.alibaba.fastjson.JSON;
|
|
|
-import com.alibaba.fastjson.TypeReference;
|
|
|
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
-import com.google.common.cache.CacheBuilder;
|
|
|
-import com.google.common.cache.CacheLoader;
|
|
|
-import com.google.common.cache.LoadingCache;
|
|
|
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
|
|
|
-import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
|
|
|
-import lombok.extern.slf4j.Slf4j;
|
|
|
-import org.apache.commons.collections.CollectionUtils;
|
|
|
-import org.apache.commons.collections.MapUtils;
|
|
|
-import org.apache.commons.lang3.StringUtils;
|
|
|
-import org.springframework.beans.factory.annotation.Autowired;
|
|
|
-import org.springframework.beans.factory.annotation.Qualifier;
|
|
|
-import org.springframework.data.redis.core.RedisTemplate;
|
|
|
-import org.springframework.scheduling.annotation.Scheduled;
|
|
|
-import org.springframework.stereotype.Component;
|
|
|
-
|
|
|
-import javax.annotation.Nonnull;
|
|
|
-import java.time.LocalDate;
|
|
|
-import java.time.format.DateTimeFormatter;
|
|
|
-import java.util.*;
|
|
|
-import java.util.concurrent.TimeUnit;
|
|
|
-import java.util.stream.Collectors;
|
|
|
-
|
|
|
-@Slf4j
|
|
|
-@Component
|
|
|
-public class VovLowerStrategy implements FilterStrategy {
|
|
|
-
|
|
|
- @ApolloJsonValue("${vov.filter.condition:{}}")
|
|
|
- private Map<String, Double> vovFilterCondition;
|
|
|
-
|
|
|
- @Autowired
|
|
|
- @Qualifier("redisTemplate")
|
|
|
- private RedisTemplate<String, String> redisTemplate;
|
|
|
-
|
|
|
- private static final String KEY_FORMAT = "redis:vid_vov_daily4filter:%d";
|
|
|
-
|
|
|
- private static final String LOWER_VOV_VID_KEY_FORMAT = "redis:lower_vov_vid:%s";
|
|
|
- private static final DateTimeFormatter YYYY_MM_DD = DateTimeFormatter.ofPattern("yyyyMMdd");
|
|
|
-
|
|
|
- /**
|
|
|
- * 低VOV视频ID集合缓存, 五分钟有效期
|
|
|
- */
|
|
|
- private final LoadingCache<String, Set<Long>> lowerVovVidCache = CacheBuilder.newBuilder()
|
|
|
- .expireAfterWrite(5, TimeUnit.MINUTES)
|
|
|
- .build(new CacheLoader<String, Set<Long>>() {
|
|
|
- @Override
|
|
|
- public Set<Long> load(@Nonnull String key) {
|
|
|
- Set<String> members = redisTemplate.opsForSet().members(key);
|
|
|
- if (CollectionUtils.isEmpty(members)) {
|
|
|
- return Collections.emptySet();
|
|
|
- }
|
|
|
- return members.stream()
|
|
|
- .map(Long::parseLong)
|
|
|
- .collect(Collectors.toSet());
|
|
|
- }
|
|
|
- });
|
|
|
-
|
|
|
- @Override
|
|
|
- public List<Long> filter(FilterParam param) {
|
|
|
-
|
|
|
- List<Long> videoIds = param.getVideoIds();
|
|
|
- if (CollectionUtils.isEmpty(videoIds)) {
|
|
|
- log.info("VOV过滤 -- videoIds为空,跳过");
|
|
|
- return new ArrayList<>();
|
|
|
- }
|
|
|
-
|
|
|
- List<Long> result = new ArrayList<>(videoIds.size());
|
|
|
- List<Long> removeIds = new ArrayList<>();
|
|
|
-
|
|
|
- String dt = LocalDate.now().format(YYYY_MM_DD);
|
|
|
- String redisKey = String.format(LOWER_VOV_VID_KEY_FORMAT, dt);
|
|
|
- try {
|
|
|
- Set<Long> lowerVovVidSet = lowerVovVidCache.get(redisKey);
|
|
|
-
|
|
|
- for (Long videoId : videoIds) {
|
|
|
- if (lowerVovVidSet.contains(videoId)) {
|
|
|
- removeIds.add(videoId);
|
|
|
- } else {
|
|
|
- result.add(videoId);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("VOV过滤V2 -- 从缓存中获取低VOV视频ID集合异常: ", e);
|
|
|
- }
|
|
|
-
|
|
|
- log.info("VOV过滤V2 -- 本次召回的ID列表为: {}, 被过滤掉的ID列表为: {}, 剩下的ID列表为: {}", videoIds, removeIds, result);
|
|
|
-
|
|
|
- return result;
|
|
|
- }
|
|
|
-
|
|
|
- public List<Long> filterV1(List<Long> videoIds) {
|
|
|
-
|
|
|
- List<Long> result = new ArrayList<>(videoIds.size());
|
|
|
-
|
|
|
- List<String> redisKeys = videoIds.stream()
|
|
|
- .map(r -> String.format(KEY_FORMAT, r))
|
|
|
- .collect(Collectors.toList());
|
|
|
- List<String> vovInfos = redisTemplate.opsForValue().multiGet(redisKeys);
|
|
|
- if (CollectionUtils.isEmpty(vovInfos) || vovInfos.size() != videoIds.size()) {
|
|
|
- log.info("VOV过滤 -- 获取到的视频VOV信息为空或长度与videoIds长度不一致,跳过: {}, {}", videoIds.size(), vovInfos.size());
|
|
|
- return result;
|
|
|
- }
|
|
|
-
|
|
|
- List<Long> removeIds = new ArrayList<>();
|
|
|
-
|
|
|
- for (int i = 0; i < videoIds.size(); i++) {
|
|
|
- Long videoId = videoIds.get(i);
|
|
|
- String vovInfo = vovInfos.get(i);
|
|
|
- if (isShouldFilter(vovInfo)) {
|
|
|
- removeIds.add(videoId);
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- result.add(videoId);
|
|
|
- }
|
|
|
-
|
|
|
- log.info("VOV过滤V1 -- 本次召回的ID列表为: {}, 被过滤掉的ID列表为: {}, 剩下的ID列表为: {}", videoIds, removeIds, result);
|
|
|
-
|
|
|
- return result;
|
|
|
- }
|
|
|
-
|
|
|
- private boolean isShouldFilter(String vovInfo) {
|
|
|
- try {
|
|
|
-
|
|
|
- if (StringUtils.isEmpty(vovInfo)) {
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- Map<String, Double> vovInfoMap = JSON.parseObject(vovInfo, new TypeReference<Map<String, Double>>() {
|
|
|
- });
|
|
|
- if (MapUtils.isEmpty(vovInfoMap)) {
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- if (isNewVideo(vovInfoMap) || isUptrendVideo(vovInfoMap)) {
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- // 预估VOV
|
|
|
- double t1Vov0 = vovInfoMap.getOrDefault("1_vov0", 0d);
|
|
|
- double t2Vov0 = vovInfoMap.getOrDefault("2_vov0", 0d);
|
|
|
-
|
|
|
- double estimated2Vov0Alpha = vovFilterCondition.getOrDefault("estimated_2_vov_0_alpha", 0.2d);
|
|
|
- double estimated1Vov0Alpha = vovFilterCondition.getOrDefault("estimated_1_vov_0_alpha", 0.8d);
|
|
|
- double estimatedVovThreshold = vovFilterCondition.getOrDefault("estimated_vov_threshold", 0.19d);
|
|
|
-
|
|
|
- double estimatedVov = t1Vov0;
|
|
|
- if (t2Vov0 != 0) {
|
|
|
- estimatedVov = estimated2Vov0Alpha * t2Vov0 + estimated1Vov0Alpha * t1Vov0;
|
|
|
- }
|
|
|
-
|
|
|
- return estimatedVov < estimatedVovThreshold;
|
|
|
- } catch (Exception e) {
|
|
|
- log.info("VOV过滤 -- 异常: ", e);
|
|
|
- }
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- // 三天曝光都小于阈值的 认定为新视频
|
|
|
- private boolean isNewVideo(Map<String, Double> vovInfoMap) {
|
|
|
- double t2ViewPvCondition = vovFilterCondition.getOrDefault("t_2_view_pv", 50d);
|
|
|
- double t3ViewPvCondition = vovFilterCondition.getOrDefault("t_3_view_pv", 50d);
|
|
|
- double t4ViewPvCondition = vovFilterCondition.getOrDefault("t_4_view_pv", 50d);
|
|
|
- double t2ViewPv = vovInfoMap.getOrDefault("2_vov0_分母", 0d);
|
|
|
- double t3ViewPv = vovInfoMap.getOrDefault("3_vov0_分母", 0d);
|
|
|
- double t4ViewPv = vovInfoMap.getOrDefault("4_vov0_分母", 0d);
|
|
|
-
|
|
|
- return t2ViewPv < t2ViewPvCondition && t3ViewPv < t3ViewPvCondition && t4ViewPv < t4ViewPvCondition;
|
|
|
- }
|
|
|
-
|
|
|
- // 最近两天的vov满足并值,且昨天的vov-前天的vov 达到一定的涨幅认定为上升趋势视频
|
|
|
- private boolean isUptrendVideo(Map<String, Double> vovInfoMap) {
|
|
|
-
|
|
|
- double t1VovCondition = vovFilterCondition.getOrDefault("t_1_vov", 0.2d);
|
|
|
- double t2VovCondition = vovFilterCondition.getOrDefault("t_2_vov", 0d);
|
|
|
- double vovDiff = vovFilterCondition.getOrDefault("12_vov_diff", 0.1);
|
|
|
-
|
|
|
- double t1Vov0 = vovInfoMap.getOrDefault("1_vov0", 0d);
|
|
|
- double t2Vov0 = vovInfoMap.getOrDefault("2_vov0", 0d);
|
|
|
- double t12VovDiff = t1Vov0 - t2Vov0;
|
|
|
-
|
|
|
- return t2Vov0 > t2VovCondition && t1Vov0 > t1VovCondition && t12VovDiff > vovDiff;
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- @Scheduled(cron = "0 0/5 * * * ? ")
|
|
|
- public void cronSync() {
|
|
|
- }
|
|
|
-}
|