Merge branch 'master' into feature/zhangbo_rank

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/MergeUtils.java

@@ -94,10 +94,6 @@ public class MergeUtils {
                 mergeQueuesItems.put(mergeQueue, currentItem);
             }
         }
-//         debug log
-//        for (Map.Entry<String, RankItem> entry : mergeQueuesItems.entries()) {
-//            LOGGER.info("after distribute item queue info: entry: [{}]", entry);
-//        }
         for (StrategyQueue queue : strategyQueueList) {
             String mergeQueueName = queue.getStrategyQueueInfo().getQueueName();
             if (mergeQueuesItems.containsKey(mergeQueueName)) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueueConfig.java

@@ -110,7 +110,6 @@ public class StrategyQueueConfig {
         if (conf.hasPath(STRATEGY_QUEUE_CLASS_CONF_STRING)) {
             mergeQueueInfo.setQueueClass(conf.getString(STRATEGY_QUEUE_CLASS_CONF_STRING));
         } else {
-            LOGGER.info("No Class config was found for queue [{}], use default class [{}]", name, DEFAULT_STRATEGY_CLASS_NAME);
             mergeQueueInfo.setQueueClass(DEFAULT_STRATEGY_CLASS_NAME);
         }
 
@@ -202,8 +201,6 @@ public class StrategyQueueConfig {
 
             if (strategyQueueInfoMap.get(name).getChildren().contains(rule.queueName)) {
                 strategyQueueInfoMap.get(name).addRule(rule);
-            } else {
-                LOGGER.info("Skip rule={} for not exist queue={}", ruleName, rule.queueName);
             }
         }
         return true;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ThompsonSamplingModel.java

@@ -57,7 +57,6 @@ public class ThompsonSamplingModel extends Model {
         }
 
         this.thompsonSamplingModel = initModel;
-        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
         input.close();
         in.close();
         return true;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/BaseRecaller.java

@@ -127,7 +127,6 @@ public class BaseRecaller<Video> {
         try {
             // 格式
             queues = queueProvider.loads(Lists.newArrayList(queueNames), QUEUE_LOAD_TIMEOUT, TimeUnit.MILLISECONDS);
-            LOGGER.info("load queue success [{}]", queues.size());
         } catch (Exception e) {
             LOGGER.error("load queue occur error [{}]", ExceptionUtils.getFullStackTrace(e));
         }
@@ -187,21 +186,15 @@ public class BaseRecaller<Video> {
                         LOGGER.error("[fetch queue error] ex {}", ExceptionUtils.getStackTrace(e));
                     }
                     if (result != null) {
-//                        LOGGER.info("fetch queue success [{}]", JSONObject.toJSONString(result));
                         candidateQueueMap.putAll(result);
                     }
                 }
             }
         }
 
-        LOGGER.info("traceId: [{}], cost = {}, recall candidateQueueMap: [{}]", requestData.getRequestId(),
-                stopwatch.elapsed().toMillis(), JSONObject.toJSONString(candidateQueueMap));
-
         stopwatch.reset().start();
         List<RankItem> result = convertToRankItem(candidateQueueMap, requestData, user);
 
-        LOGGER.info("traceId: [{}], cost = {}, convertToRankItem result: [{}]", requestData.getRequestId(),
-                stopwatch.elapsed().toMillis(), JSONObject.toJSONString(result));
         return result;
     }
 
@@ -215,17 +208,12 @@ public class BaseRecaller<Video> {
 
         final FilterPipeline<Video> recallFilter = new FilterPipeline<>(filterConfig, requestData, user);
 
-        LOGGER.info("traceId: [{}], recallFilter: [{}]", requestData.getRequestId(),
-                JSONObject.toJSONString(recallFilter));
-
         final List<Callable<List<RankItem>>> callables = new ArrayList<>();
         for (final Map.Entry<Candidate, Queue<Video>> entry : candidateQueueMap.entrySet()) {
             callables.add(() -> {
                 List<RankItem> candidateHits = new ArrayList<>();
                 final Candidate candidate = entry.getKey();
                 List<Entry<Video>> entriesValue = CommonCollectionUtils.iterableToList(entry.getValue());
-                LOGGER.info("traceId: [{}], entry.getValue: [{}], entriesValue: [{}]", requestData.getRequestId(),
-                        JSONObject.toJSONString(entry.getValue()), JSONObject.toJSONString(entriesValue));
                 try {
                     final Map<Video, Entry<Video>> entryMap = new HashMap<>();
                     List<Video> entriesList = entriesValue.stream().map(e -> {
@@ -233,8 +221,6 @@ public class BaseRecaller<Video> {
                         return e.item;
                     }).collect(Collectors.toList());
                     List<Video> entriesAfterFilter = recallFilter.doFilter(candidate, entriesList);
-                    LOGGER.info("traceId: [{}] entriesAfterFilter: [{}]",
-                            requestData.getRequestId(), JSONObject.toJSONString(entriesAfterFilter));
                     // 1. filter  TODO 待后续增加自定义filter
                     if (CollectionUtils.isNotEmpty(entriesAfterFilter)) {
                         List<Entry<Video>> entries = entriesAfterFilter.stream()
@@ -245,9 +231,6 @@ public class BaseRecaller<Video> {
                         candidateHits.addAll(toHits(entries, candidate));
                     }
 
-                    // debug log for tracing
-                    LOGGER.info("recalled candidate [{}], queue length [{}], expected [{}], hit [{}]",
-                            candidate.getCandidateKey(), entry.getValue().size(), candidate.getCandidateNum(), candidateHits.size());
                 } catch (Exception e) {
                     LOGGER.error("recall filter queue occur error, queue [{}], error: [{}]", candidate.toString(), ExceptionUtils.getFullStackTrace(e));
                 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterConfig.java

@@ -44,14 +44,7 @@ public class FilterConfig {
         Config filterConf = recallConfig.getConfig("filter-config");
         try {
             loadFilters(filterConf);
-            int pos = 0;
-            for (FilterConfigInfo filterConfigInfo : filterConfigInfoList) {
-                LOGGER.info("filter at position [{}], priority [{}] filter name [{}] ",
-                        new Object[]{pos++, filterConfigInfo.getFilterPriority(), filterConfigInfo.getConfigName()});
-            }
-            LOGGER.debug("Load filter config success");
         } catch (Exception e) {
-            LOGGER.error("Load filter config failed, [{}]", ExceptionUtils.getFullStackTrace(e));
             return false;
         }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterPipeline.java

@@ -54,11 +54,9 @@ public class FilterPipeline<T> {
     }
 
     public List<T> doFilter(Candidate candidate, List<T> list) {
-        LOGGER.info("FilterPipeline doFilter start traceId:{}, before list size:{}", requestContext.getRequestId(), list.size());
         for (AbstractFilter filter : filters) {
             filter.doFilter(candidate, list);
         }
-        LOGGER.info("FilterPipeline doFilter end traceId:{}, after list size:{}", requestContext.getRequestId(), list.size());
         return list;
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/RedisBackedQueue.java

@@ -70,7 +70,6 @@ public class RedisBackedQueue implements QueueProvider<Video> {
     // redis读取队列内内容
     public Index get(QueueName name) throws Exception {
         String valueJson = client.get(name.toString());
-        logger.info("RedisBackedQueue get QueueName: [{}], value: [{}]", name, valueJson);
         Index index = new Index(name.toString(), valueJson);
         return index;
     }
@@ -86,8 +85,6 @@ public class RedisBackedQueue implements QueueProvider<Video> {
     public Queue<Video> load(QueueName name) throws ExecutionException {
 
         Pair<Long, Queue<Video>> cachedQueue = cache.get(name);
-//        logger.info("RedisBackedQueue load QueueName: [{}], cachedQueue size: [{}]", name,
-//                cachedQueue.getValue().size());
         if (cachedQueue == null) {
             // 清理本地缓存中当前key的数据
             cache.invalidate(name);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/HistoryLongPeriodFilter.java

@@ -51,15 +51,9 @@ public class HistoryLongPeriodFilter extends AbstractFilter<Video> {
         filterParam.setConcurrent(true);
         filterParam.setNotUsePreView(false);
         FilterResult filterResult = filterService.filter(filterParam);
-        LOGGER.info("HistoryLongPeriodFilter doFilter traceId:{}, filterResult:{}", requestContext.getRequestId(),
-                filterResult);
-        LOGGER.info("HistoryLongPeriodFilter doFilter traceId:{}, list:{}", requestContext.getRequestId(),
-                list.stream().map(Video::getVideoId).map(Object::toString).collect(Collectors.joining(",")));
         if (filterResult != null && filterResult.getVideoIds() != null) {
             list.removeIf(video -> !filterResult.getVideoIds().contains(video.getVideoId()));
         }
-        LOGGER.info("HistoryLongPeriodFilter doFilter end traceId:{}, list:{}", requestContext.getRequestId(),
-                list.stream().map(Video::getVideoId).map(Object::toString).collect(Collectors.joining(",")));
     }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/PreViewedFilter.java

@@ -35,27 +35,18 @@ public class PreViewedFilter extends AbstractFilter<Video> {
 
     @Override
     public void doFilter(Candidate candidate, List<Video> list) {
-        LOGGER.info("PreViewedFilter doFilter start traceId:{}, user:{}, apptype:{}", requestContext.getRequestId(),
-                user, requestContext.getAppType());
         if (StringUtils.isBlank(user.getMid())
                 || CollectionUtils.isEmpty(list)) {
             return;
         }
-        LOGGER.info("PreViewedFilter doFilter traceId:{}, preViewedVideoIds:{}", requestContext.getRequestId(),
-                preViewedVideoIds);
 
         if (CollectionUtils.isEmpty(preViewedVideoIds)) {
             return;
         }
-        LOGGER.info("PreViewedFilter doFilter traceId:{}, list:{}", requestContext.getRequestId(),
-                list.stream().map(Video::getVideoId).map(Objects::toString).collect(Collectors.joining(",")));
         list.removeIf(video -> preViewedVideoIds.contains(video.getVideoId()));
-        LOGGER.info("PreViewedFilter doFilter end traceId:{}, list:{}", requestContext.getRequestId(),
-                list.stream().map(Video::getVideoId).map(Objects::toString).collect(Collectors.joining(",")));
     }
 
 
-
 }
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/ViewedHistoryFilter.java

@@ -53,18 +53,12 @@ public class ViewedHistoryFilter extends AbstractFilter<Video> {
         if (CollectionUtils.isEmpty(videoList)) {
             return;
         }
-        LOGGER.info("ViewedHistoryFilter doFilter start traceId:{}, user:{}, apptype:{}", requestContext.getRequestId(),
-                user, requestContext.getAppType());
         videoList.removeIf(video -> this.historySet.contains(String.valueOf(video.getVideoId())));
-        LOGGER.info("ViewedHistoryFilter doFilter end traceId:{}, list:{}", requestContext.getRequestId(),
-                videoList.stream().map(Video::getVideoId).map(Object::toString).collect(Collectors.joining(",")));
     }
 
 
     public Set<String> loadUserHistory(User user) {
         String userid = StringUtils.isNotBlank(user.getUid()) ? user.getUid() : user.getMid();
-        LOGGER.info("ViewedHistoryFilter loadUserHistory start traceId:{}, userid:{}, apptype:{}", requestContext.getRequestId(),
-                userid, requestContext.getAppType());
         if (StringUtils.isBlank(userid)) {
             return new HashSet<>();
         }
@@ -93,8 +87,6 @@ public class ViewedHistoryFilter extends AbstractFilter<Video> {
                 viewedVideoIds.add("-1");
             }
             stopWatch.stop();
-            LOGGER.info("ViewedHistoryFilter loadUserHistory end traceId:{}, userid:{}, apptype:{}, viewedVideoIds:{}, time:{}",
-                    requestContext.getRequestId(), userid, requestContext.getAppType(), viewedVideoIds, stopWatch.getTime());
 
             redisTemplate.opsForSet().add(key, viewedVideoIds.toArray(new String[0]));
             redisTemplate.expire(key, 1, TimeUnit.DAYS);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -83,6 +83,12 @@ public class RankService {
             return new RankResult(results);
         }
 
+        if (param.getAppType() == AppTypeEnum.APP_SPEED.getCode()) {
+            List<Video> results = new ArrayList<>(extractAndSort(param, AppFallbackRecallStrategy.PUSH_FORM));
+            removeDuplicate(results);
+            return new RankResult(results);
+        }
+
         // 2 正常走分发 排序+冷启动
         List<Video> rovRecallRank = mergeAndRankRovRecall(param);
         List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -88,6 +88,9 @@ public class RecallService implements ApplicationContextAware {
         } else if (param.getAppType() == AppTypeEnum.PIAO_QUAN_MEIHAO_ZHUFU.getCode() && Objects.equals(param.getCategoryId(), 100)) {
             strategies.add(strategyMap.get(BlessRecallStrategy.class.getSimpleName()));
             return strategies;
+        } else if (param.getAppType() == AppTypeEnum.APP_SPEED.getCode()) {
+            strategies.add(strategyMap.get(AppFallbackRecallStrategy.class.getSimpleName()));
+            return strategies;
         } else {
             switch (abCode) {
                 case "60121": // 536

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AppFallbackRecallStrategy.java

@@ -0,0 +1,82 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ */
+@Component
+public class AppFallbackRecallStrategy implements RecallStrategy {
+
+    public static final String PUSH_FORM = "app_fallback_recall_strategy";
+
+    @Autowired
+    private RegionFilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    private RedisTemplate<String, String> redisTemplate;
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        // 1 获取省份key 放入参数map中
+        String regionCode = param.getRegionCode();
+        String redisKey;
+        if (StringUtils.isBlank(regionCode) || StringUtils.equals(regionCode, "110100")
+                || StringUtils.equals(regionCode, "430100")) {
+            // 北京和长沙
+            redisKey = "app_fallback_recall_strategy_safe_videos";
+        } else {
+            // 其他
+            redisKey = "app_fallback_recall_unsafe_strategy_videos";
+        }
+        String videoIds = redisTemplate.opsForValue().get(redisKey);
+        if (StringUtils.isBlank(videoIds)) {
+            return Collections.emptyList();
+        }
+        List<Long> videoIdList = Arrays.asList(videoIds.split(",")).stream()
+                .map(Long::parseLong)
+                .collect(Collectors.toList());
+        FilterParam filterParam = FilterParamFactory.create(param, videoIdList);
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        List<Video> videosResult = new ArrayList<>();
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            filterResult.getVideoIds().stream().limit(500)
+                    .forEach(vid -> {
+                        Video video = new Video();
+                        video.setVideoId(vid);
+                        video.setAbCode(param.getAbCode());
+                        video.setPushFrom(pushFrom());
+                        videosResult.add(video);
+                    });
+        }
+        return videosResult;
+    }
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+
+
+}

recommend-server-service/src/main/resources/application-test.yml

@@ -14,10 +14,8 @@ eureka:
 
 spring:
   redis:
-    clusterHost: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
-    clusterPort: 3717
-    secondaryHost: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
-    secondaryPort: 3717
+    hostName: r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com
+    port: 6379
     password: Wqsd@2019
     timeout: 1000
     lettuce:
@@ -61,8 +59,10 @@ spring:
         min-idle: 0
   data:
     mongodb:
-      host: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
-      port: 3717
+      clusterHost: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
+      clusterPort: 3717
+      secondaryHost: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
+      secondaryPort: 3717
       database: longvideo
       username: lv
       password: lv@2018