|
|
@@ -0,0 +1,181 @@
|
|
|
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
|
|
|
+
|
|
|
+import com.tzld.piaoquan.recommend.server.model.Video;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
|
|
|
+import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.collections4.CollectionUtils;
|
|
|
+import org.apache.commons.collections4.MapUtils;
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
+import org.apache.commons.lang3.math.NumberUtils;
|
|
|
+import org.apache.commons.lang3.tuple.Pair;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.beans.factory.annotation.Qualifier;
|
|
|
+import org.springframework.data.redis.core.RedisTemplate;
|
|
|
+import org.springframework.stereotype.Component;
|
|
|
+
|
|
|
+import java.util.*;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+
|
|
|
+/**
|
|
|
+ * 用户画像 实质元素 rovn 召回
|
|
|
+ * 数据源: param.userNetworkSeqFeature 里的 s_z_y_s (元素列表) + zt_gyf (归一分列表)
|
|
|
+ * 上游 alg_user_network_seq_feature 已新增, 来自 user_element_profile_hot
|
|
|
+ * top_elements = UNION ALL(positive_ranked, negative_ranked), 所以归一分可能为负
|
|
|
+ *
|
|
|
+ * 逻辑: (element, score) pair 按 score DESC 取前 topN=30 正向元素 -> 一次 multiGet
|
|
|
+ * elements_rovn_recall:{kw} 倒排 -> 同 vid 取 max score -> 排序 -> filter
|
|
|
+ * 只取正向 (score > 0), 避免召回用户厌恶元素
|
|
|
+ *
|
|
|
+ * 跟 YearShareDkElementsRecallStrategy 共用 Redis 倒排 key, 仅用户兴趣源 + 取法不同
|
|
|
+ */
|
|
|
+@Slf4j
|
|
|
+@Component
|
|
|
+public class UserProfileDkElementsFXRecallStrategy implements RecallStrategy {
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ @Qualifier("redisTemplate")
|
|
|
+ private RedisTemplate<String, String> redisTemplate;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private FilterService filterService;
|
|
|
+
|
|
|
+ private final String CLASS_NAME = this.getClass().getSimpleName();
|
|
|
+
|
|
|
+ public static final int topN = 30;
|
|
|
+ public static final String PUSH_FROM = "recall_user_profile_dk_elements_fx";
|
|
|
+ public static final String redisKeyPrefix = "elements_rovn_recall";
|
|
|
+
|
|
|
+ public static final String KEY_ELEMENTS = "s_z_y_s_fx";
|
|
|
+ public static final String KEY_SCORES = "fx_gyf";
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public String pushFrom() {
|
|
|
+ return PUSH_FROM;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public List<Video> recall(RecallParam param) {
|
|
|
+ List<Video> videosResult = new ArrayList<>();
|
|
|
+ try {
|
|
|
+ if (MapUtils.isEmpty(param.getUserNetworkSeqFeature())) {
|
|
|
+ return videosResult;
|
|
|
+ }
|
|
|
+
|
|
|
+ List<String> elements = FeatureUtils.extractVidsFromUserNetworkSeqFeature(param.getUserNetworkSeqFeature(), KEY_ELEMENTS);
|
|
|
+ List<String> scores = FeatureUtils.extractVidsFromUserNetworkSeqFeature(param.getUserNetworkSeqFeature(), KEY_SCORES);
|
|
|
+ if (CollectionUtils.isEmpty(elements) || elements.size() != scores.size()) {
|
|
|
+ return videosResult;
|
|
|
+ }
|
|
|
+
|
|
|
+ List<String> topElements = pickTopPositiveElements(elements, scores);
|
|
|
+ if (CollectionUtils.isEmpty(topElements)) {
|
|
|
+ return videosResult;
|
|
|
+ }
|
|
|
+
|
|
|
+ List<String> keys = getRedisKey(topElements);
|
|
|
+ List<String> values = redisTemplate.opsForValue().multiGet(keys);
|
|
|
+
|
|
|
+ // 保留 Redis 倒排的真实 rovn 分 (而非位置分): scoresMap 的 score 会写到 Video.rovScore,
|
|
|
+ // 粗排截断 coarseMap miss 的 vid 会 fallback 用 Video.rovScore 排序, 真实分更有信号.
|
|
|
+ Map<Long, Double> scoresMap = recall(param.getVideoId(), values);
|
|
|
+ List<Long> ids = scoresMap.entrySet().stream()
|
|
|
+ .sorted(Comparator.comparingDouble((Map.Entry<Long, Double> e) -> e.getValue()).reversed())
|
|
|
+ .map(Map.Entry::getKey)
|
|
|
+ .collect(Collectors.toList());
|
|
|
+
|
|
|
+ FilterParam filterParam = FilterParamFactory.create(param, ids, pushFrom(), scoresMap);
|
|
|
+ FilterResult filterResult = filterService.filter(filterParam);
|
|
|
+ if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
|
|
|
+ for (Long vid : filterResult.getVideoIds()) {
|
|
|
+ Video video = new Video();
|
|
|
+ video.setVideoId(vid);
|
|
|
+ video.setRovScore(scoresMap.getOrDefault(vid, 0.0));
|
|
|
+ video.setPushFrom(pushFrom());
|
|
|
+ videosResult.add(video);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
|
|
|
+ }
|
|
|
+ return videosResult;
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 组对 + 过滤负向 + 按归一分降序 + 取前 topN 个 element */
|
|
|
+ private List<String> pickTopPositiveElements(List<String> elements, List<String> scores) {
|
|
|
+ List<Pair<String, Double>> pairs = new ArrayList<>();
|
|
|
+ for (int i = 0; i < elements.size(); i++) {
|
|
|
+ String element = elements.get(i);
|
|
|
+ if (StringUtils.isBlank(element)) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ double score = NumberUtils.toDouble(scores.get(i), 0.0);
|
|
|
+ if (score <= 0) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ pairs.add(Pair.of(element, score));
|
|
|
+ }
|
|
|
+ if (pairs.isEmpty()) {
|
|
|
+ return Collections.emptyList();
|
|
|
+ }
|
|
|
+ return pairs.stream()
|
|
|
+ .sorted(Comparator.comparingDouble((Pair<String, Double> p) -> p.getValue()).reversed())
|
|
|
+ .map(Pair::getKey)
|
|
|
+ .distinct()
|
|
|
+ .limit(topN)
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ }
|
|
|
+
|
|
|
+ private List<String> getRedisKey(List<String> elementList) {
|
|
|
+ List<String> keys = new ArrayList<>();
|
|
|
+ for (String element : elementList) {
|
|
|
+ keys.add(String.format("%s:%s", redisKeyPrefix, element));
|
|
|
+ }
|
|
|
+ return keys;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 解析 multiGet 拿到的 N 个 Redis value, 拼成 vid -> 真实 score map.
|
|
|
+ * value 格式: vid1,vid2,...\tscore1,score2,... (rovn 真实分)
|
|
|
+ * 同 vid 在多个 element 倒排里出现时, 取 max score (跟 AbstractRedisRecallStrategy 一致).
|
|
|
+ */
|
|
|
+ private Map<Long, Double> recall(Long headVid, List<String> values) {
|
|
|
+ Map<Long, Double> scoresMap = new HashMap<>();
|
|
|
+ if (CollectionUtils.isEmpty(values)) {
|
|
|
+ return scoresMap;
|
|
|
+ }
|
|
|
+ for (String value : values) {
|
|
|
+ if (StringUtils.isBlank(value)) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ String[] cells = value.split("\t");
|
|
|
+ if (cells.length != 2) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ List<Long> ids;
|
|
|
+ List<Double> scores;
|
|
|
+ try {
|
|
|
+ ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
|
|
|
+ scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
|
|
|
+ } catch (NumberFormatException nfe) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (ids.isEmpty() || ids.size() != scores.size()) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ for (int i = 0; i < ids.size(); i++) {
|
|
|
+ long id = ids.get(i);
|
|
|
+ if (headVid != null && headVid == id) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ scoresMap.merge(id, scores.get(i), Math::max);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return scoresMap;
|
|
|
+ }
|
|
|
+}
|
