homepage recommend

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -25,12 +25,12 @@ import com.tzld.piaoquan.recommend.server.service.recall.strategy.RegionRelative
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
 
 import javax.annotation.PostConstruct;
 import java.util.*;
@@ -128,6 +128,13 @@ public class RecommendService {
     }
 
     private RecommendResponse specialMidRecommend(RecommendRequest request) {
+        log.info("hit special mid recommend request={}", JSONUtils.toJson(request));
+        if (request == null) {
+            return RecommendResponse.newBuilder()
+                    .setResult(Result.newBuilder().setCode(1).setMessage("success"))
+                    .build();
+        }
+
         RecallParam recallParam = new RecallParam();
         recallParam.setAppType(request.getAppType());
         recallParam.setMid(request.getMid());
@@ -135,7 +142,7 @@ public class RecommendService {
         recallParam.setSize(request.getSize());
 
         RecallResult recallResult = recallService.recall(recallParam);
-        log.info("recallResult {}", recallResult);
+        log.info("recallParam {}, recallResult {}", JSONUtils.toJson(recallParam), JSONUtils.toJson(recallResult));
 
         RankParam rankParam = new RankParam();
         rankParam.setRecallResult(recallResult);
@@ -143,7 +150,7 @@ public class RecommendService {
         rankParam.setSpecialRecommend(true);
 
         RankResult rankResult = rankService.rank(rankParam);
-        log.info("rankResult {}", rankResult);
+        log.info("rankParam, rankResult {}", JSONUtils.toJson(rankParam), JSONUtils.toJson(rankResult));
 
         if (rankResult == null || CollectionUtils.isEmpty(rankResult.getVideos())) {
             return RecommendResponse.newBuilder()
@@ -156,10 +163,11 @@ public class RecommendService {
         if (recallParam.getSize() < rankResult.getVideos().size()) {
             videos = rankResult.getVideos().subList(0, recallParam.getSize());
         }
-        if (StringUtils.isNotBlank(request.getMid()) && !CollectionUtils.isEmpty(videos)) {
+        if (CollectionUtils.isNotEmpty(videos)) {
             Video lastVideo = videos.get(videos.size() - 1);
-            redisTemplate.opsForValue().set(lastVideo.getLastVideoKey(), String.valueOf(lastVideo.getVideoId()), 1
-                    , TimeUnit.DAYS);
+            log.info("update recall last video  lastVideo={}", JSONUtils.toJson(lastVideo));
+            redisTemplate.opsForValue().set(lastVideo.getLastVideoKey(), String.valueOf(lastVideo.getVideoId()),
+                    1, TimeUnit.DAYS);
         }
 
         return RecommendResponse.newBuilder()

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithLevelFilterService.java

@@ -45,6 +45,7 @@ public class FlowPoolWithLevelFilterService extends AbstractFilterService {
                         v -> param.getFlowPoolMap().get(v)));
 
         Map<Long, Integer> distributeCountMap = flowPoolService.getDistributeCountWithLevel(flowPoolMap);
+        log.info("getDistributeCountWithLevel distributeCountMap={}", JSONUtils.toJson(distributeCountMap));
 
         List<Long> remainVideoIds = new ArrayList<>();
         for (Long videoId : videoIds) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithLevelScoreFilterService.java

@@ -44,7 +44,7 @@ public class FlowPoolWithLevelScoreFilterService extends AbstractFilterService {
                         v -> param.getFlowPoolMap().get(v)));
 
         Map<Long, Integer> distributeCountMap = flowPoolService.getDistributeCountWithLevelScore(flowPoolMap);
-        log.info("filterDistributeCountAndDel distributeCountMap={}", JSONUtils.toJson(distributeCountMap));
+        log.info("getDistributeCountWithLevelScore distributeCountMap={}", JSONUtils.toJson(distributeCountMap));
         List<Long> remainVideoIds = new ArrayList<>();
         for (Long videoId : videoIds) {
             if (distributeCountMap.get(videoId) != null

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FlowPoolWithScoreFilterService.java

@@ -44,7 +44,7 @@ public class FlowPoolWithScoreFilterService extends AbstractFilterService {
                         v -> param.getFlowPoolMap().get(v)));
 
         Map<Long, Integer> distributeCountMap = flowPoolService.getDistributeCountWithScore(flowPoolMap);
-
+        log.info("getDistributeCountWithScore distributeCountMap={}", JSONUtils.toJson(distributeCountMap));
         List<Long> remainVideoIds = new ArrayList<>();
         for (Long videoId : videoIds) {
             if (distributeCountMap.get(videoId) != null

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractRegionRecallStrategy.java

@@ -58,27 +58,37 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
         // recordKey: 每小时生成一个召回源，用来决定用哪个召回源
         // poolKey: 召回源的key
         // lastVideoLKey: 上次返回的最后一个视频
+        /**
+         * 1、如果当前小时数据已生成，那么召回源用当前小时数据
+         * 2、如果当前小时数据未生成，那么召回源用前一小时数据
+         *
+         * 1、如果recordKey是当前小时数据，用lastVideo作为cursor
+         * 2、如果recordKey是前一小时数据，重置lastVideo
+         *
+         *
+         */
+
         String record = redisTemplate.opsForValue().get(recordKey);
         if (StringUtils.isNotBlank(record)) {
             Map<String, String> recordMap = JSONUtils.fromJson(record, new TypeToken<Map<String, String>>() {
             }, Collections.emptyMap());
             if (MapUtils.isNotEmpty(recordMap)) {
                 String record_dt = recordMap.get("date");
-                String record_h = recordMap.get("h");
+                int record_h = NumberUtils.toInt(recordMap.get("h"), 0);
                 String now_dt = DateUtils.getCurrentDateStr("yyyyMMdd");
                 int h = DateUtils.getCurrentHour();
-                if (record_dt.equals(now_dt) && Integer.parseInt(record_h) == h) {
-                    poolKey = poolKey(param, now_dt, h);
+                if (record_dt.equals(now_dt) && record_h == h) {
+                    poolKey = poolKey(param, record_dt, record_h);
                     idx = getIdx(lastVideoKey, poolKey);
-                } else if ((record_dt.equals(now_dt) && h - Integer.parseInt(record_h) == 1)
-                        || (h == 0 && Integer.parseInt(record_h) == 23)) {
+                } else if ((record_dt.equals(now_dt) && h - record_h == 1)
+                        || (h == 0 && record_h == 23)) {
                     poolKey = poolKey(param, now_dt, h);
                     if (redisTemplate.hasKey(poolKey)) {
                         redisTemplate.opsForValue().set(recordKey, String.format(recordKeyFormat, now_dt,
                                 h), 2, TimeUnit.HOURS);
                         redisTemplate.delete(lastVideoKey);
                     } else {
-                        poolKey = poolKey(param, record_dt, Integer.parseInt(record_h));
+                        poolKey = poolKey(param, record_dt, record_h);
                         idx = getIdx(lastVideoKey, poolKey);
                     }
                 } else {
@@ -134,9 +144,11 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
             }
         }
 
+        // 如果不限制返回size个数据，可以迁移到排序阶段
+        // Collections.sort(results, Comparator.comparingDouble(o -> -o.getRovScore()));
 
-        Collections.sort(results, Comparator.comparingDouble(o -> -o.getRovScore()));
-
+        // 这里是一个优化：如果所有数据都被过滤，说明下一次召回可以直接从lastVideo开始
+        // TODO 可以优化成异步更新
         if (StringUtils.isNotBlank(lastVideoId)
                 && CollectionUtils.isEmpty(results)
                 && StringUtils.isNotBlank(param.getMid())) {
@@ -144,7 +156,8 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
             redisTemplate.opsForValue().set(lastVideoKey, lastVideoId, 2, TimeUnit.HOURS);
         }
 
-        return results.subList(0, results.size() < param.getSize() ? results.size() : param.getSize());
+        // return results.subList(0, results.size() < param.getSize() ? results.size() : param.getSize());
+        return results;
     }
 
     private int getIdx(String lastVideoKey, String poolKey) {
@@ -160,7 +173,6 @@ public abstract class AbstractRegionRecallStrategy implements RecallStrategy {
         return idx;
     }
 
-    // TODO 和 推荐排序后的lastVideo更新有什么区别?
     private String updateLastVideoRecord(String recordKey, RecallParam param) {
         String currentDateStr = DateUtils.getCurrentDateStr("yyyyMMdd");
         int h = DateUtils.getCurrentHour();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractVideoRecallStrategy.java

@@ -1,7 +1,7 @@
 package com.tzld.piaoquan.recommend.server.service.recall.strategy;
 
-import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
+import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
 import com.tzld.piaoquan.recommend.server.service.filter.VideoFilterService;
@@ -13,11 +13,13 @@ import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.redis.core.RedisTemplate;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 
 /**
  * @author dyp
@@ -44,18 +46,13 @@ public abstract class AbstractVideoRecallStrategy implements RecallStrategy {
         if (StringUtils.isBlank(value)) {
             return null;
         }
-        List<Item> items = JSONObject.parseArray(value, Item.class);
+        Map<Long, Double> videoScoreMap = JSONUtils.fromJson(value, new TypeToken<Map<Long, Double>>() {
+        }, Collections.emptyMap());
         List<Video> results = new ArrayList<>();
 
-        Map<Long, Double> videoScoreMap = new HashMap<>();
-        for (int i = 0; i < items.size() && i < 20; i++) {
-            Item item = items.get(i);
-            videoScoreMap.put(NumberUtils.toLong(item.getVideoId(), 0),
-                    NumberUtils.toDouble(item.getScore(), 0));
-        }
-
         FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, Lists.newArrayList(videoScoreMap.keySet())));
-
+        log.info("recall filterResult={}", JSONUtils.toJson(filterResult));
+        
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
             filterResult.getVideoIds().stream().forEach(vid -> {
                 Video recallData = new Video();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/SpecialRecallStrategy.java

@@ -50,7 +50,8 @@ public class SpecialRecallStrategy implements RecallStrategy {
                 idx += 1;
             }
         }
-        log.info("lastSpecialRecallKey={}, value={}, specialKeyName={},idx={}", lastSpecialRecallKey, value, specialKeyName, idx);
+        log.info("lastSpecialRecallKey={}, value={}, specialKeyName={}, idx={}", lastSpecialRecallKey, value,
+                specialKeyName, idx);
 
         int getSize = param.getSize() * 5;
         int freq = 0;