|
|
@@ -17,7 +17,6 @@ import org.springframework.stereotype.Component;
|
|
|
|
|
|
import java.util.*;
|
|
|
import java.util.concurrent.*;
|
|
|
-import java.util.stream.Collectors;
|
|
|
|
|
|
@Component
|
|
|
@Slf4j
|
|
|
@@ -38,48 +37,62 @@ public class TodayPublishStrategy implements FilterStrategy {
|
|
|
public FilterResult filter(FilterParam param) {
|
|
|
Long start = System.currentTimeMillis();
|
|
|
FilterResult filterResult = new FilterResult();
|
|
|
- List<String> result = Collections.synchronizedList(param.getContents().stream().map(Content::getId).collect(Collectors.toList()));
|
|
|
Long todayStart = DateUtils.getTodayStart();
|
|
|
+
|
|
|
List<PublishContent> todayPublishContentList = aigcBaseMapper.getTodayPublishContentList(param.getAccountId(), todayStart);
|
|
|
List<PublishContent> todayPublishFailList = aigcBaseMapper.getTodayPublishFailContentList(param.getAccountId(), todayStart);
|
|
|
+
|
|
|
+ // 构建已发布/失败 ID 集合,O(1) 查重,同时收集 ID 用于标题查询
|
|
|
+ Set<String> todayPublishIdSet = new HashSet<>();
|
|
|
+ List<String> todayPublishContentIds = new ArrayList<>();
|
|
|
if (CollectionUtils.isNotEmpty(todayPublishContentList)) {
|
|
|
- List<String> todayPublishContentIdList = todayPublishContentList.stream().map(PublishContent::getId).collect(Collectors.toList());
|
|
|
- result.removeAll(todayPublishContentIdList);
|
|
|
+ for (PublishContent pc : todayPublishContentList) {
|
|
|
+ todayPublishIdSet.add(pc.getId());
|
|
|
+ todayPublishContentIds.add(pc.getId());
|
|
|
+ }
|
|
|
}
|
|
|
if (CollectionUtils.isNotEmpty(todayPublishFailList)) {
|
|
|
- List<String> todayPublishFailIdList = todayPublishFailList.stream().map(PublishContent::getId).collect(Collectors.toList());
|
|
|
- result.removeAll(todayPublishFailIdList);
|
|
|
- }
|
|
|
- // 标题相似度过滤
|
|
|
- if (CollectionUtils.isNotEmpty(result)) {
|
|
|
- List<String> todayPublishContentIds = new ArrayList<>();
|
|
|
- if (CollectionUtils.isNotEmpty(todayPublishContentList)) {
|
|
|
- todayPublishContentList.stream().map(PublishContent::getId).forEach(todayPublishContentIds::add);
|
|
|
- }
|
|
|
- if (CollectionUtils.isNotEmpty(todayPublishFailList)) {
|
|
|
- todayPublishFailList.stream().map(PublishContent::getId).forEach(todayPublishContentIds::add);
|
|
|
+ for (PublishContent pc : todayPublishFailList) {
|
|
|
+ todayPublishIdSet.add(pc.getId());
|
|
|
+ todayPublishContentIds.add(pc.getId());
|
|
|
}
|
|
|
+ }
|
|
|
+
|
|
|
+ // 预构建标题相似度缓存(参照 HistoryTitleStrategy)
|
|
|
+ List<Set<Character>> todayTitleCache = Collections.emptyList();
|
|
|
+ if (CollectionUtils.isNotEmpty(todayPublishContentIds)) {
|
|
|
List<String> todayPublishTitles = aigcBaseMapper.getTodayPublishContentTitles(todayPublishContentIds);
|
|
|
if (CollectionUtils.isNotEmpty(todayPublishTitles)) {
|
|
|
- Map<String, Content> contentMap = param.getContents().stream()
|
|
|
- .collect(Collectors.toMap(Content::getId, c -> c, (a, b) -> a));
|
|
|
- List<Set<Character>> todayTitleCache = TitleSimilarCheckUtil.makeCache(todayPublishTitles);
|
|
|
- List<String> titleFilterResult = new ArrayList<>();
|
|
|
- List<Content> filteredContents = new ArrayList<>();
|
|
|
- for (String contentId : result) {
|
|
|
- Content content = contentMap.get(contentId);
|
|
|
- if (content != null && content.getTitle() != null
|
|
|
- && TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), todayTitleCache, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD)) {
|
|
|
- filteredContents.add(content);
|
|
|
- } else {
|
|
|
- titleFilterResult.add(contentId);
|
|
|
- }
|
|
|
- }
|
|
|
- result = titleFilterResult;
|
|
|
- filterResult.setFilterContent(filteredContents);
|
|
|
+ todayTitleCache = TitleSimilarCheckUtil.makeCache(todayPublishTitles);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 单次遍历:先 ID 精确匹配,再标题相似度过滤(参照 HistoryTitleStrategy 模式)
|
|
|
+ List<String> result = new ArrayList<>(param.getContents().size());
|
|
|
+ List<Content> filterContents = new ArrayList<>();
|
|
|
+ boolean hasTitleCache = CollectionUtils.isNotEmpty(todayTitleCache);
|
|
|
+
|
|
|
+ for (Content content : param.getContents()) {
|
|
|
+ String contentId = content.getId();
|
|
|
+
|
|
|
+ // 今日已发布/失败,精确 ID 排除
|
|
|
+ if (todayPublishIdSet.contains(contentId)) {
|
|
|
+ continue;
|
|
|
}
|
|
|
+
|
|
|
+ // 标题相似度过滤
|
|
|
+ if (hasTitleCache
|
|
|
+ && content.getTitle() != null
|
|
|
+ && TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), todayTitleCache, TitleSimilarCheckUtil.SIMILARITY_THRESHOLD)) {
|
|
|
+ filterContents.add(content);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ result.add(contentId);
|
|
|
}
|
|
|
+
|
|
|
filterResult.setContentIds(result);
|
|
|
+ filterResult.setFilterContent(filterContents);
|
|
|
log.info("TodayPublishStrategy filter cost time:{}", System.currentTimeMillis() - start);
|
|
|
return filterResult;
|
|
|
}
|