扩展city召回

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -441,6 +441,7 @@ public class RecommendService {
         recallParam.setAbExpCodes(param.getAbExpCodes());
 
         recallParam.setProvince(param.getProvince());
+        recallParam.setCity(param.getCity());
         recallParam.setCategoryId(param.getCategoryId());
 
         recallParam.setCityCode(param.getCityCode());

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV567.java

@@ -14,9 +14,7 @@ import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractVideoMer
 import com.tzld.piaoquan.recommend.server.service.rank.tansform.FeatureV6;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.FeatureBucketUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import com.tzld.piaoquan.recommend.server.util.*;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -26,7 +24,6 @@ import org.springframework.stereotype.Service;
 import java.util.*;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 @Service
 @Slf4j
@@ -47,65 +44,30 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         //-------------------辑-------------------
 
         long currentMs = System.currentTimeMillis();
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
         Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v0);
-        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        List<Video> rovRecallRank = new ArrayList<>(v0);
+        List<Video> rovRecallRank = new ArrayList<>();
+        // -------------------5路特殊旧召回------------------
+        RecallUtils.extractOldSpecialRecall(param, setVideo, rovRecallRank);
         //-------------------return相似召回------------------
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
-        rovRecallRank.addAll(v6);
-        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("v6", 5.0).intValue(), param, ReturnVideoRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
         //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("v1", 5.0).intValue(), param, RegionRealtimeRecallStrategyV1.PUSH_FORM, setVideo, rovRecallRank);
         //-------------------scene cf rovn------------------
-        List<Video> sceneCFRovn = extractAndSort(param, SceneCFRovnRecallStrategy.PUSH_FORM);
-        sceneCFRovn = sceneCFRovn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        sceneCFRovn = sceneCFRovn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), sceneCFRovn.size()));
-        rovRecallRank.addAll(sceneCFRovn);
-        setVideo.addAll(sceneCFRovn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), param, SceneCFRovnRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
         //-------------------scene cf rosn------------------
-        List<Video> sceneCFRosn = extractAndSort(param, SceneCFRosnRecallStrategy.PUSH_FORM);
-        sceneCFRosn = sceneCFRosn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
-        rovRecallRank.addAll(sceneCFRosn);
-        setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        // -------------------cate1------------------
-        int cate1RecallN = mergeWeight.getOrDefault("cate1RecallN", 5.0).intValue();
-        addRecall(param, cate1RecallN, UserCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
-        // -------------------cate2------------------
-        int cate2RecallN = mergeWeight.getOrDefault("cate2RecallN", 5.0).intValue();
-        addRecall(param, cate2RecallN, UserCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), param, SceneCFRosnRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+        // -------------------user cate1------------------
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("cate1RecallN", 5.0).intValue(), param, UserCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+        // -------------------user cate2------------------
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("cate2RecallN", 5.0).intValue(), param, UserCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
         // -------------------head province cate1------------------
-        int headCate1RecallN = mergeWeight.getOrDefault("headCate1RecallN", 3.0).intValue();
-        addRecall(param, headCate1RecallN, HeadProvinceCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate1RecallN", 3.0).intValue(), param, HeadProvinceCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
         // -------------------head province cate2------------------
-        int headCate2RecallN = mergeWeight.getOrDefault("headCate2RecallN", 3.0).intValue();
-        addRecall(param, headCate2RecallN, HeadProvinceCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate2RecallN", 3.0).intValue(), param, HeadProvinceCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
         //-------------------head cate2 of rovn------------------
-        List<Video> headCate2Rov = extractAndSort(param, HeadCate2RovRecallStrategy.PUSH_FROM);
-        // 视频去重
-        removeDuplicate(headCate2Rov);
-        headCate2Rov = headCate2Rov.stream().filter(o -> !setVideo.contains(o.getVideoId())).collect(Collectors.toList());
-        headCate2Rov = headCate2Rov.subList(0, Math.min(mergeWeight.getOrDefault("headCate2Rov", 5.0).intValue(), headCate2Rov.size()));
-        rovRecallRank.addAll(headCate2Rov);
-        setVideo.addAll(headCate2Rov.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate2Rov", 5.0).intValue(), param, HeadCate2RovRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        //-------------------city rovn------------------
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("cityRov", 5.0).intValue(), param, CityRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -136,7 +98,7 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
 
         // 4. 排序模型计算
         Map<String, Float> sceneFeatureMap = new HashMap<>(0);
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250317.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250423.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
 
         // 5. 排序公式特征
         double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.059);
@@ -370,6 +332,13 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
                 baseInfo.put("system", system);
             }
         }
+        String userChannel = param.getChannelName();
+        if (null != userChannel && !userChannel.isEmpty()) {
+            baseInfo.put("user_channel", userChannel);
+        }
+        if (FeatureUtils.firstLevel(param.getUserShareDepth())) {
+            baseInfo.put("user_level", "1st");
+        }
         return baseInfo;
     }
 
@@ -383,16 +352,6 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         return newScore;
     }
 
-    private void addRecall(RankParam param, int recallNum, String recallName, Set<Long> setVideo, List<Video> rovRecallRank) {
-        if (recallNum > 0) {
-            List<Video> list = extractAndSort(param, recallName);
-            list = list.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-            list = list.subList(0, Math.min(recallNum, list.size()));
-            rovRecallRank.addAll(list);
-            setVideo.addAll(list.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        }
-    }
-
     private Map<String, Double> findSimCateScore(String headCate2, int length) {
         if (StringUtils.isBlank(headCate2)) {
             return new HashMap<>();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV568.java

@@ -14,10 +14,7 @@ import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractVideoMer
 import com.tzld.piaoquan.recommend.server.service.rank.tansform.FeatureV6;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.FeatureBucketUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import com.tzld.piaoquan.recommend.server.util.RecallUtils;
+import com.tzld.piaoquan.recommend.server.util.*;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -101,7 +98,7 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
 
         // 4. 排序模型计算
         Map<String, Float> sceneFeatureMap = new HashMap<>(0);
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250317.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250423.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
 
         // 5. 排序公式特征
         double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.059);
@@ -343,6 +340,13 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
                 baseInfo.put("system", system);
             }
         }
+        String userChannel = param.getChannelName();
+        if (null != userChannel && !userChannel.isEmpty()) {
+            baseInfo.put("user_channel", userChannel);
+        }
+        if (FeatureUtils.firstLevel(param.getUserShareDepth())) {
+            baseInfo.put("user_level", "1st");
+        }
         return baseInfo;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallParam.java

@@ -18,6 +18,7 @@ public class RecallParam {
     private String dataKey;
     private String ruleKey;
     private String province;
+    private String city;
 
     private int size;
     private Long videoId;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -114,6 +114,9 @@ public class RecallService implements ApplicationContextAware {
         strategies.add(strategyMap.get(HeadProvinceCate2RecallStrategy.class.getSimpleName()));
         strategies.add(strategyMap.get(HeadCate2RovRecallStrategy.class.getSimpleName()));
         Set<String> abExpCodes = param.getAbExpCodes();
+        if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("567")) {
+            strategies.add(strategyMap.get(CityRovnRecallStrategy.class.getSimpleName()));
+        }
         if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("568")) {
             strategies.add(strategyMap.get(PremiumROVRecallStrategy.class.getSimpleName()));
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/CityRovnRecallStrategy.java

@@ -0,0 +1,102 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class CityRovnRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    public static final String PUSH_FROM = "city_rovn";
+    public static final String redisKeyPrefix = "city_rovn_recall";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FROM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            String city = param.getCity().replaceAll("市$", "");
+            String redisKey = String.format("%s:%s", redisKeyPrefix, city);
+            String redisValue = redisTemplate.opsForValue().get(redisKey);
+            if (null == redisValue || redisValue.isEmpty()) {
+                return videosResult;
+            }
+            Pair<List<Long>, Map<Long, Double>> pair = parsePair(redisValue, param.getVideoId(), 100);
+            fillVideoResult(param, pair, videosResult);
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private Pair<List<Long>, Map<Long, Double>> parsePair(String data, long headVid, int size) {
+        List<Long> idsList = new ArrayList<>();
+        Map<Long, Double> scoresMap = new HashMap<>();
+        if (!StringUtils.isBlank(data)) {
+            String[] cells = data.split("\t");
+            if (2 == cells.length) {
+                List<Long> ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                List<Double> scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty() && ids.size() == scores.size()) {
+                    int minSize = Math.min(size, ids.size());
+                    for (int i = 0; i < minSize; ++i) {
+                        long id = ids.get(i);
+                        double score = scores.get(i);
+                        if (headVid != id && !scoresMap.containsKey(id)) {
+                            idsList.add(id);
+                            scoresMap.put(id, score);
+                        }
+                    }
+                }
+            }
+        }
+        return new MutablePair<>(idsList, scoresMap);
+    }
+
+    private void fillVideoResult(RecallParam param, Pair<List<Long>, Map<Long, Double>> pair, List<Video> videosResult) {
+        if (null != pair) {
+            List<Long> ids = pair.getLeft();
+            Map<Long, Double> scoresMap = pair.getRight();
+            if (null != ids && null != scoresMap && !ids.isEmpty()) {
+                FilterParam filterParam = FilterParamFactory.create(param, ids);
+                FilterResult filterResult = filterService.filter(filterParam);
+                if (null != filterResult && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                    filterResult.getVideoIds().forEach(vid -> {
+                        Video video = new Video();
+                        video.setVideoId(vid);
+                        video.setRovScore(scoresMap.getOrDefault(vid, 0D));
+                        video.setPushFrom(pushFrom());
+                        videosResult.add(video);
+                    });
+                }
+            }
+        }
+    }
+}