|
@@ -10,6 +10,7 @@ import com.tzld.videoVector.common.constant.VectorConstants;
|
|
|
import com.tzld.videoVector.common.enums.Modality;
|
|
import com.tzld.videoVector.common.enums.Modality;
|
|
|
import com.tzld.videoVector.dao.mapper.pgVector.DeconstructVectorConfigMapper;
|
|
import com.tzld.videoVector.dao.mapper.pgVector.DeconstructVectorConfigMapper;
|
|
|
import com.tzld.videoVector.dao.mapper.pgVector.ext.ArticleDeconstructResultMapperExt;
|
|
import com.tzld.videoVector.dao.mapper.pgVector.ext.ArticleDeconstructResultMapperExt;
|
|
|
|
|
+import com.tzld.videoVector.dao.mapper.pgVector.ext.ArticleQualityMapperExt;
|
|
|
import com.tzld.videoVector.dao.mapper.pgVector.ext.MaterialDeconstructResultMapperExt;
|
|
import com.tzld.videoVector.dao.mapper.pgVector.ext.MaterialDeconstructResultMapperExt;
|
|
|
import com.tzld.videoVector.dao.mapper.pgVector.ext.MaterialQualityMapperExt;
|
|
import com.tzld.videoVector.dao.mapper.pgVector.ext.MaterialQualityMapperExt;
|
|
|
import com.tzld.videoVector.model.entity.ArticleMatch;
|
|
import com.tzld.videoVector.model.entity.ArticleMatch;
|
|
@@ -23,6 +24,7 @@ import com.tzld.videoVector.model.param.recall.MatchByMaterialIdParam;
|
|
|
import com.tzld.videoVector.model.param.recall.MatchByTextParam;
|
|
import com.tzld.videoVector.model.param.recall.MatchByTextParam;
|
|
|
import com.tzld.videoVector.model.param.recall.MatchByVideoIdParam;
|
|
import com.tzld.videoVector.model.param.recall.MatchByVideoIdParam;
|
|
|
import com.tzld.videoVector.model.po.pgVector.ArticleDeconstructResult;
|
|
import com.tzld.videoVector.model.po.pgVector.ArticleDeconstructResult;
|
|
|
|
|
+import com.tzld.videoVector.model.po.pgVector.ArticleQuality;
|
|
|
import com.tzld.videoVector.model.po.pgVector.DeconstructVectorConfig;
|
|
import com.tzld.videoVector.model.po.pgVector.DeconstructVectorConfig;
|
|
|
import com.tzld.videoVector.model.po.pgVector.DeconstructVectorConfigExample;
|
|
import com.tzld.videoVector.model.po.pgVector.DeconstructVectorConfigExample;
|
|
|
import com.tzld.videoVector.model.po.pgVector.MaterialDeconstructResult;
|
|
import com.tzld.videoVector.model.po.pgVector.MaterialDeconstructResult;
|
|
@@ -112,6 +114,9 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
@Autowired
|
|
@Autowired
|
|
|
private ArticleDeconstructResultMapperExt articleDeconstructResultMapperExt;
|
|
private ArticleDeconstructResultMapperExt articleDeconstructResultMapperExt;
|
|
|
|
|
|
|
|
|
|
+ @Autowired(required = false)
|
|
|
|
|
+ private ArticleQualityMapperExt articleQualityMapperExt;
|
|
|
|
|
+
|
|
|
@Autowired
|
|
@Autowired
|
|
|
private DeconstructVectorConfigMapper deconstructVectorConfigMapper;
|
|
private DeconstructVectorConfigMapper deconstructVectorConfigMapper;
|
|
|
|
|
|
|
@@ -287,7 +292,7 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
articleItems = Collections.emptyList();
|
|
articleItems = Collections.emptyList();
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- List<VideoMatchEnrichedVO> videoItems = enrichVideoMatches(videoMatches, configCode);
|
|
|
|
|
|
|
+ List<VideoMatchEnrichedVO> videoItems = enrichVideoMatches(videoMatches, configCode, param.getDays());
|
|
|
|
|
|
|
|
// WP3 召回前筛选:模态 + 来源(精排前先筛,减少无效候选进入 ranker)
|
|
// WP3 召回前筛选:模态 + 来源(精排前先筛,减少无效候选进入 ranker)
|
|
|
int beforeV = videoItems.size(), beforeM = materialItems.size(), beforeA = articleItems.size();
|
|
int beforeV = videoItems.size(), beforeM = materialItems.size(), beforeA = articleItems.size();
|
|
@@ -457,7 +462,7 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
.limit(enrichK).collect(Collectors.toList());
|
|
.limit(enrichK).collect(Collectors.toList());
|
|
|
|
|
|
|
|
// enrich
|
|
// enrich
|
|
|
- List<VideoMatchEnrichedVO> videoItems = enrichVideoMatches(topVideo, configCodes.get(0));
|
|
|
|
|
|
|
+ List<VideoMatchEnrichedVO> videoItems = enrichVideoMatches(topVideo, configCodes.get(0), param.getDays());
|
|
|
for (int i = 0; i < videoItems.size() && i < topVideo.size(); i++) {
|
|
for (int i = 0; i < videoItems.size() && i < topVideo.size(); i++) {
|
|
|
String cc = topVideo.get(i).getConfigCode();
|
|
String cc = topVideo.get(i).getConfigCode();
|
|
|
if (cc != null) videoItems.get(i).setConfigCode(cc);
|
|
if (cc != null) videoItems.get(i).setConfigCode(cc);
|
|
@@ -797,7 +802,8 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
/**
|
|
/**
|
|
|
* 视频召回结果 enrich
|
|
* 视频召回结果 enrich
|
|
|
*/
|
|
*/
|
|
|
- private List<VideoMatchEnrichedVO> enrichVideoMatches(List<VideoMatchResult> matches, String requestConfigCode) {
|
|
|
|
|
|
|
+ private List<VideoMatchEnrichedVO> enrichVideoMatches(List<VideoMatchResult> matches, String requestConfigCode,
|
|
|
|
|
+ Integer days) {
|
|
|
if (CollectionUtils.isEmpty(matches)) {
|
|
if (CollectionUtils.isEmpty(matches)) {
|
|
|
return Collections.emptyList();
|
|
return Collections.emptyList();
|
|
|
}
|
|
}
|
|
@@ -811,7 +817,7 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
: videoApiService.getVideoDetail(videoIds);
|
|
: videoApiService.getVideoDetail(videoIds);
|
|
|
|
|
|
|
|
// 从 Redis 加载视频详情(含 ROV 等指标数据)用于精排打分
|
|
// 从 Redis 加载视频详情(含 ROV 等指标数据)用于精排打分
|
|
|
- Map<Long, Map<String, Object>> redisDetails = loadRedisVideoDetails(matches);
|
|
|
|
|
|
|
+ Map<Long, Map<String, Object>> redisDetails = loadRedisVideoDetails(matches, days);
|
|
|
|
|
|
|
|
List<VideoMatchEnrichedVO> items = new ArrayList<>(matches.size());
|
|
List<VideoMatchEnrichedVO> items = new ArrayList<>(matches.size());
|
|
|
for (VideoMatchResult m : matches) {
|
|
for (VideoMatchResult m : matches) {
|
|
@@ -853,16 +859,18 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
* 从 Redis 批量加载视频详情(key: video:detail:{metricsDays}d:{videoId})
|
|
* 从 Redis 批量加载视频详情(key: video:detail:{metricsDays}d:{videoId})
|
|
|
* 用于提取 ROV 等指标数据做精排打分。
|
|
* 用于提取 ROV 等指标数据做精排打分。
|
|
|
*/
|
|
*/
|
|
|
- private Map<Long, Map<String, Object>> loadRedisVideoDetails(List<VideoMatchResult> matches) {
|
|
|
|
|
|
|
+ private Map<Long, Map<String, Object>> loadRedisVideoDetails(List<VideoMatchResult> matches, Integer days) {
|
|
|
Map<Long, Map<String, Object>> result = new HashMap<>();
|
|
Map<Long, Map<String, Object>> result = new HashMap<>();
|
|
|
if (CollectionUtils.isEmpty(matches)) return result;
|
|
if (CollectionUtils.isEmpty(matches)) return result;
|
|
|
|
|
|
|
|
|
|
+ int effectiveDays = days != null ? days : metricsDays;
|
|
|
|
|
+
|
|
|
List<Long> orderedIds = new ArrayList<>();
|
|
List<Long> orderedIds = new ArrayList<>();
|
|
|
List<String> keys = new ArrayList<>();
|
|
List<String> keys = new ArrayList<>();
|
|
|
for (VideoMatchResult m : matches) {
|
|
for (VideoMatchResult m : matches) {
|
|
|
if (m != null && m.getVideoId() != null) {
|
|
if (m != null && m.getVideoId() != null) {
|
|
|
orderedIds.add(m.getVideoId());
|
|
orderedIds.add(m.getVideoId());
|
|
|
- keys.add(VectorConstants.VIDEO_DETAIL_DAYS_KEY_PREFIX + metricsDays + "d:" + m.getVideoId());
|
|
|
|
|
|
|
+ keys.add(VectorConstants.VIDEO_DETAIL_DAYS_KEY_PREFIX + effectiveDays + "d:" + m.getVideoId());
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
if (keys.isEmpty()) return result;
|
|
if (keys.isEmpty()) return result;
|
|
@@ -1082,6 +1090,7 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
.filter(java.util.Objects::nonNull)
|
|
.filter(java.util.Objects::nonNull)
|
|
|
.collect(Collectors.toList());
|
|
.collect(Collectors.toList());
|
|
|
Map<String, ArticleDeconstructResult> rowByArticleId = loadArticleDeconstructRows(articleIds);
|
|
Map<String, ArticleDeconstructResult> rowByArticleId = loadArticleDeconstructRows(articleIds);
|
|
|
|
|
+ Map<String, ArticleQuality> qualityByArticleId = loadArticleQualityRows(articleIds);
|
|
|
|
|
|
|
|
List<VideoMatchEnrichedVO> items = new ArrayList<>(matches.size());
|
|
List<VideoMatchEnrichedVO> items = new ArrayList<>(matches.size());
|
|
|
for (ArticleMatch m : matches) {
|
|
for (ArticleMatch m : matches) {
|
|
@@ -1125,6 +1134,27 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
|
|
|
|
|
applyCompatibilityFields(vo);
|
|
applyCompatibilityFields(vo);
|
|
|
applySignals(vo, requestConfigCode, "ann");
|
|
applySignals(vo, requestConfigCode, "ann");
|
|
|
|
|
+
|
|
|
|
|
+ // 文章质量分 enrich
|
|
|
|
|
+ ArticleQuality aq = qualityByArticleId.get(m.getArticleId());
|
|
|
|
|
+ if (aq != null && vo.getSignals() != null) {
|
|
|
|
|
+ RecallSignalsVO.QualitySignal qs = new RecallSignalsVO.QualitySignal();
|
|
|
|
|
+ qs.setHasData(true);
|
|
|
|
|
+ qs.setReadScore(aq.getReadScore());
|
|
|
|
|
+ qs.setOpenScore(aq.getOpenScore());
|
|
|
|
|
+ qs.setFissionScore(aq.getFissionScore());
|
|
|
|
|
+ qs.setConfidence(aq.getConfidence());
|
|
|
|
|
+ qs.setTotalRead(aq.getTotalRead());
|
|
|
|
|
+ qs.setAvgRead(aq.getAvgRead());
|
|
|
|
|
+ if (aq.getAvgRead() != null && aq.getAvgRead() > 0) {
|
|
|
|
|
+ qs.setReadMultiplier((double) aq.getTotalRead() / aq.getAvgRead());
|
|
|
|
|
+ }
|
|
|
|
|
+ qs.setOpenRate(aq.getOpenRate());
|
|
|
|
|
+ qs.setFissionRate(aq.getFissionRate());
|
|
|
|
|
+ qs.setPublishCount(aq.getPublishCount());
|
|
|
|
|
+ vo.getSignals().setQuality(qs);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
items.add(vo);
|
|
items.add(vo);
|
|
|
}
|
|
}
|
|
|
return items;
|
|
return items;
|
|
@@ -1153,6 +1183,31 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
return result;
|
|
return result;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ private Map<String, ArticleQuality> loadArticleQualityRows(List<String> articleIds) {
|
|
|
|
|
+ if (CollectionUtils.isEmpty(articleIds)) {
|
|
|
|
|
+ return Collections.emptyMap();
|
|
|
|
|
+ }
|
|
|
|
|
+ if (articleQualityMapperExt == null) {
|
|
|
|
|
+ return Collections.emptyMap();
|
|
|
|
|
+ }
|
|
|
|
|
+ Map<String, ArticleQuality> result = new HashMap<>();
|
|
|
|
|
+ try {
|
|
|
|
|
+ List<ArticleQuality> rows = articleQualityMapperExt.selectByContentIds(articleIds);
|
|
|
|
|
+ if (CollectionUtils.isEmpty(rows)) {
|
|
|
|
|
+ return result;
|
|
|
|
|
+ }
|
|
|
|
|
+ for (ArticleQuality row : rows) {
|
|
|
|
|
+ if (row == null || !StringUtils.hasText(row.getContentId())) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+ result.putIfAbsent(row.getContentId(), row);
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
|
+ log.error("批量加载 article_quality 失败: {}", e.getMessage(), e);
|
|
|
|
|
+ }
|
|
|
|
|
+ return result;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
private JSONObject parseArticleResultJson(ArticleDeconstructResult row) {
|
|
private JSONObject parseArticleResultJson(ArticleDeconstructResult row) {
|
|
|
if (row == null || !StringUtils.hasText(row.getResult())) {
|
|
if (row == null || !StringUtils.hasText(row.getResult())) {
|
|
|
return null;
|
|
return null;
|
|
@@ -1657,6 +1712,9 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
if (spec.getWViral() != null) p.setWViral(spec.getWViral());
|
|
if (spec.getWViral() != null) p.setWViral(spec.getWViral());
|
|
|
if (spec.getWRoi() != null) p.setWRoi(spec.getWRoi());
|
|
if (spec.getWRoi() != null) p.setWRoi(spec.getWRoi());
|
|
|
if (spec.getMaterialMissingStrategy() != null) p.setMaterialMissingStrategy(spec.getMaterialMissingStrategy());
|
|
if (spec.getMaterialMissingStrategy() != null) p.setMaterialMissingStrategy(spec.getMaterialMissingStrategy());
|
|
|
|
|
+ if (spec.getWRead() != null) p.setWRead(spec.getWRead());
|
|
|
|
|
+ if (spec.getWOpen() != null) p.setWOpen(spec.getWOpen());
|
|
|
|
|
+ if (spec.getWFission() != null) p.setWFission(spec.getWFission());
|
|
|
return p;
|
|
return p;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -1824,7 +1882,7 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
}
|
|
}
|
|
|
String configCode = StringUtils.hasText(param.getConfigCode())
|
|
String configCode = StringUtils.hasText(param.getConfigCode())
|
|
|
? param.getConfigCode() : VectorConstants.DEFAULT_CONFIG_CODE;
|
|
? param.getConfigCode() : VectorConstants.DEFAULT_CONFIG_CODE;
|
|
|
- List<VideoMatchEnrichedVO> videoItems = enrichVideoMatches(rawMatches, configCode);
|
|
|
|
|
|
|
+ List<VideoMatchEnrichedVO> videoItems = enrichVideoMatches(rawMatches, configCode, null);
|
|
|
// displayK 截断
|
|
// displayK 截断
|
|
|
if (videoItems.size() > displayK) {
|
|
if (videoItems.size() > displayK) {
|
|
|
videoItems = limitEnrichedItemsByScore(videoItems, displayK);
|
|
videoItems = limitEnrichedItemsByScore(videoItems, displayK);
|
|
@@ -1900,7 +1958,7 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
if (!deduped.isEmpty()) {
|
|
if (!deduped.isEmpty()) {
|
|
|
List<VideoMatchResult> videoResults = toVideoMatchResults(deduped, cc);
|
|
List<VideoMatchResult> videoResults = toVideoMatchResults(deduped, cc);
|
|
|
populateVideoMatchResultDetails(videoResults);
|
|
populateVideoMatchResultDetails(videoResults);
|
|
|
- allResults.addAll(enrichVideoMatches(videoResults, cc));
|
|
|
|
|
|
|
+ allResults.addAll(enrichVideoMatches(videoResults, cc, null));
|
|
|
}
|
|
}
|
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
|
log.error("matchByMaterialId 视频搜索失败 configCode={}: {}", cc, e.getMessage(), e);
|
|
log.error("matchByMaterialId 视频搜索失败 configCode={}: {}", cc, e.getMessage(), e);
|
|
@@ -2293,7 +2351,7 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
if (!deduped.isEmpty()) {
|
|
if (!deduped.isEmpty()) {
|
|
|
List<VideoMatchResult> videoResults = toVideoMatchResults(deduped, cc);
|
|
List<VideoMatchResult> videoResults = toVideoMatchResults(deduped, cc);
|
|
|
populateVideoMatchResultDetails(videoResults);
|
|
populateVideoMatchResultDetails(videoResults);
|
|
|
- allResults.addAll(enrichVideoMatches(videoResults, cc));
|
|
|
|
|
|
|
+ allResults.addAll(enrichVideoMatches(videoResults, cc, null));
|
|
|
}
|
|
}
|
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
|
log.error("matchByArticleId 视频搜索失败 configCode={}: {}", cc, e.getMessage(), e);
|
|
log.error("matchByArticleId 视频搜索失败 configCode={}: {}", cc, e.getMessage(), e);
|
|
@@ -2422,6 +2480,25 @@ public class VectorRecallTestServiceImpl implements VectorRecallTestService {
|
|
|
|
|
|
|
|
applyCompatibilityFields(vo);
|
|
applyCompatibilityFields(vo);
|
|
|
applySignals(vo, configCode, "self");
|
|
applySignals(vo, configCode, "self");
|
|
|
|
|
+
|
|
|
|
|
+ // 文章质量分 enrich
|
|
|
|
|
+ Map<String, ArticleQuality> selfQualityMap = loadArticleQualityRows(Collections.singletonList(articleId));
|
|
|
|
|
+ ArticleQuality aq = selfQualityMap.get(articleId);
|
|
|
|
|
+ if (aq != null && vo.getSignals() != null) {
|
|
|
|
|
+ RecallSignalsVO.QualitySignal qs = new RecallSignalsVO.QualitySignal();
|
|
|
|
|
+ qs.setHasData(true);
|
|
|
|
|
+ qs.setReadScore(aq.getReadScore());
|
|
|
|
|
+ qs.setOpenScore(aq.getOpenScore());
|
|
|
|
|
+ qs.setFissionScore(aq.getFissionScore());
|
|
|
|
|
+ qs.setConfidence(aq.getConfidence());
|
|
|
|
|
+ qs.setTotalRead(aq.getTotalRead());
|
|
|
|
|
+ qs.setAvgRead(aq.getAvgRead());
|
|
|
|
|
+ qs.setOpenRate(aq.getOpenRate());
|
|
|
|
|
+ qs.setFissionRate(aq.getFissionRate());
|
|
|
|
|
+ qs.setPublishCount(aq.getPublishCount());
|
|
|
|
|
+ vo.getSignals().setQuality(qs);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
return vo;
|
|
return vo;
|
|
|
}
|
|
}
|
|
|
|
|
|