|
@@ -82,13 +82,13 @@ public class DataFlushService {
|
|
|
list = list.stream().filter(o -> StringUtils.hasText(o.getVideoPath())).collect(Collectors.toList());
|
|
|
Map<String, List<ArticleMatchVideos>> map = list.stream().collect(Collectors.groupingBy(ArticleMatchVideos::getContentId));
|
|
|
List<LongArticlesCrawlerVideos> batchSaveList = new ArrayList<>();
|
|
|
- for (ArticleMatchVideos video : list) {
|
|
|
- List<ArticleMatchVideos> mapList = map.get(video.getContentId());
|
|
|
+ for (ArticleMatchVideos articleMatchVideos : list) {
|
|
|
+ List<ArticleMatchVideos> mapList = map.get(articleMatchVideos.getContentId());
|
|
|
List<Date> orderDate = mapList.stream().map(ArticleMatchVideos::getUpdateTime)
|
|
|
.sorted().collect(Collectors.toList());
|
|
|
double score = 0.2;
|
|
|
for (int i = 0; i < orderDate.size(); i++) {
|
|
|
- if (orderDate.get(i).equals(video.getUpdateTime())) {
|
|
|
+ if (orderDate.get(i).equals(articleMatchVideos.getUpdateTime())) {
|
|
|
if (i == 0) {
|
|
|
score = 1;
|
|
|
} else if (i == 1) {
|
|
@@ -99,14 +99,14 @@ public class DataFlushService {
|
|
|
}
|
|
|
}
|
|
|
LongArticlesCrawlerVideos saveItem = new LongArticlesCrawlerVideos();
|
|
|
- saveItem.setContentId(video.getContentId());
|
|
|
- saveItem.setPlatform(video.getPlatform());
|
|
|
- saveItem.setVideoTitle(video.getVideoTitle());
|
|
|
- saveItem.setCrawlerTime(video.getUpdateTime());
|
|
|
- saveItem.setVideoOssPath(video.getVideoPath());
|
|
|
- saveItem.setCoverOssPath(video.getCoverPath());
|
|
|
- saveItem.setUserId(video.getUid());
|
|
|
- saveItem.setTraceId(video.getTraceId());
|
|
|
+ saveItem.setContentId(articleMatchVideos.getContentId());
|
|
|
+ saveItem.setPlatform(articleMatchVideos.getPlatform());
|
|
|
+ saveItem.setVideoTitle(articleMatchVideos.getVideoTitle());
|
|
|
+ saveItem.setCrawlerTime(articleMatchVideos.getUpdateTime());
|
|
|
+ saveItem.setVideoOssPath(articleMatchVideos.getVideoPath());
|
|
|
+ saveItem.setCoverOssPath(articleMatchVideos.getCoverPath());
|
|
|
+ saveItem.setUserId(articleMatchVideos.getUid());
|
|
|
+ saveItem.setTraceId(articleMatchVideos.getTraceId());
|
|
|
saveItem.setDownloadStatus(2);
|
|
|
saveItem.setScore(score / 1000);
|
|
|
batchSaveList.add(saveItem);
|
|
@@ -142,44 +142,44 @@ public class DataFlushService {
|
|
|
List<LongArticlesText> batchSaveLongArticlesTextList = new ArrayList<>();
|
|
|
List<LongArticlesMatchVideos> batchSaveLongArticlesMatchVideosList = new ArrayList<>();
|
|
|
Set<String> existsIdSet = new HashSet<>();
|
|
|
- for (LongArticlesVideo video : list) {
|
|
|
- if (video.getContentId().endsWith("lehuo")) {
|
|
|
+ for (LongArticlesVideo longArticlesVideo : list) {
|
|
|
+ if (longArticlesVideo.getContentId().endsWith("lehuo")) {
|
|
|
continue;
|
|
|
}
|
|
|
- if (!existsIdSet.contains(video.getContentId())) {
|
|
|
+ if (!existsIdSet.contains(longArticlesVideo.getContentId())) {
|
|
|
LongArticlesText longArticlesText = new LongArticlesText();
|
|
|
- longArticlesText.setContentId(video.getContentId());
|
|
|
- longArticlesText.setArticleTitle(video.getArticleTitle());
|
|
|
- longArticlesText.setArticleText(video.getArticleText());
|
|
|
- if (StringUtils.hasText(video.getArticleText())) {
|
|
|
- longArticlesText.setKimiTitle(video.getKimiTitle().replace("\"", ""));
|
|
|
+ longArticlesText.setContentId(longArticlesVideo.getContentId());
|
|
|
+ longArticlesText.setArticleTitle(longArticlesVideo.getArticleTitle());
|
|
|
+ longArticlesText.setArticleText(longArticlesVideo.getArticleText());
|
|
|
+ if (StringUtils.hasText(longArticlesVideo.getArticleText())) {
|
|
|
+ longArticlesText.setKimiTitle(longArticlesVideo.getKimiTitle().replace("\"", ""));
|
|
|
}
|
|
|
- if (StringUtils.hasText(video.getKimiSummary())) {
|
|
|
- longArticlesText.setKimiSummary(video.getKimiSummary());
|
|
|
- longArticlesText.setKimiKeys(video.getKimiKeys());
|
|
|
+ if (StringUtils.hasText(longArticlesVideo.getKimiSummary())) {
|
|
|
+ longArticlesText.setKimiSummary(longArticlesVideo.getKimiSummary());
|
|
|
+ longArticlesText.setKimiKeys(longArticlesVideo.getKimiKeys());
|
|
|
} else {
|
|
|
- LongArticlesText text = kimiTitleMap.get(video.getContentId());
|
|
|
+ LongArticlesText text = kimiTitleMap.get(longArticlesVideo.getContentId());
|
|
|
longArticlesText.setKimiSummary(text.getKimiSummary());
|
|
|
longArticlesText.setKimiKeys(text.getKimiKeys());
|
|
|
}
|
|
|
longArticlesText.setKimiStatus(1);
|
|
|
batchSaveLongArticlesTextList.add(longArticlesText);
|
|
|
- existsIdSet.add(video.getContentId());
|
|
|
+ existsIdSet.add(longArticlesVideo.getContentId());
|
|
|
}
|
|
|
- if (Objects.isNull(video.getRequestTimeStamp())) {
|
|
|
+ if (Objects.isNull(longArticlesVideo.getRequestTimeStamp())) {
|
|
|
continue;
|
|
|
}
|
|
|
LongArticlesMatchVideos longArticlesMatchVideos = new LongArticlesMatchVideos();
|
|
|
- longArticlesMatchVideos.setContentId(video.getContentId());
|
|
|
- longArticlesMatchVideos.setTraceId(video.getTraceId());
|
|
|
- longArticlesMatchVideos.setGhId(video.getGhId());
|
|
|
- longArticlesMatchVideos.setAccountName(video.getAccountName());
|
|
|
- longArticlesMatchVideos.setContentStatus(video.getContentStatus());
|
|
|
- longArticlesMatchVideos.setSuccessStatus(video.getSuccess());
|
|
|
- longArticlesMatchVideos.setRequestTimestamp(video.getRequestTimeStamp());
|
|
|
- longArticlesMatchVideos.setUpdateTime(video.getUpdateTime());
|
|
|
- longArticlesMatchVideos.setProcessTimes(video.getProcessTimes());
|
|
|
- longArticlesMatchVideos.setResponse(getLongArticleVideoResponse(video));
|
|
|
+ longArticlesMatchVideos.setContentId(longArticlesVideo.getContentId());
|
|
|
+ longArticlesMatchVideos.setTraceId(longArticlesVideo.getTraceId());
|
|
|
+ longArticlesMatchVideos.setGhId(longArticlesVideo.getGhId());
|
|
|
+ longArticlesMatchVideos.setAccountName(longArticlesVideo.getAccountName());
|
|
|
+ longArticlesMatchVideos.setContentStatus(longArticlesVideo.getContentStatus());
|
|
|
+ longArticlesMatchVideos.setSuccessStatus(longArticlesVideo.getSuccess());
|
|
|
+ longArticlesMatchVideos.setRequestTimestamp(longArticlesVideo.getRequestTimeStamp());
|
|
|
+ longArticlesMatchVideos.setUpdateTime(longArticlesVideo.getUpdateTime());
|
|
|
+ longArticlesMatchVideos.setProcessTimes(longArticlesVideo.getProcessTimes());
|
|
|
+ longArticlesMatchVideos.setResponse(getLongArticleVideoResponse(longArticlesVideo));
|
|
|
batchSaveLongArticlesMatchVideosList.add(longArticlesMatchVideos);
|
|
|
}
|
|
|
if (CollectionUtil.isNotEmpty(batchSaveLongArticlesTextList)) {
|
|
@@ -210,25 +210,28 @@ public class DataFlushService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- private String getLongArticleVideoResponse(LongArticlesVideo video) {
|
|
|
+ private String getLongArticleVideoResponse(LongArticlesVideo longArticlesVideo) {
|
|
|
JSONArray jsonArray = new JSONArray();
|
|
|
- if (StringUtils.hasText(video.getResult1())) {
|
|
|
- if (StringUtils.hasText(video.getKimiTitle()) && video.getKimiTitle().contains("\"")) {
|
|
|
- video.setResult1(video.getResult1().replace(video.getKimiTitle(), video.getKimiTitle().replace("\"", "")));
|
|
|
+ if (StringUtils.hasText(longArticlesVideo.getResult1())) {
|
|
|
+ if (StringUtils.hasText(longArticlesVideo.getKimiTitle()) && longArticlesVideo.getKimiTitle().contains("\"")) {
|
|
|
+ longArticlesVideo.setResult1(longArticlesVideo.getResult1().replace(longArticlesVideo.getKimiTitle(),
|
|
|
+ longArticlesVideo.getKimiTitle().replace("\"", "")));
|
|
|
}
|
|
|
- jsonArray.add(resultToResponse(video.getResult1()));
|
|
|
+ jsonArray.add(resultToResponse(longArticlesVideo.getResult1()));
|
|
|
}
|
|
|
- if (StringUtils.hasText(video.getResult2())) {
|
|
|
- if (StringUtils.hasText(video.getKimiTitle()) && video.getKimiTitle().contains("\"")) {
|
|
|
- video.setResult2(video.getResult2().replace(video.getKimiTitle(), video.getKimiTitle().replace("\"", "")));
|
|
|
+ if (StringUtils.hasText(longArticlesVideo.getResult2())) {
|
|
|
+ if (StringUtils.hasText(longArticlesVideo.getKimiTitle()) && longArticlesVideo.getKimiTitle().contains("\"")) {
|
|
|
+ longArticlesVideo.setResult2(longArticlesVideo.getResult2().replace(longArticlesVideo.getKimiTitle(),
|
|
|
+ longArticlesVideo.getKimiTitle().replace("\"", "")));
|
|
|
}
|
|
|
- jsonArray.add(resultToResponse(video.getResult2()));
|
|
|
+ jsonArray.add(resultToResponse(longArticlesVideo.getResult2()));
|
|
|
}
|
|
|
- if (StringUtils.hasText(video.getResult3())) {
|
|
|
- if (StringUtils.hasText(video.getKimiTitle()) && video.getKimiTitle().contains("\"")) {
|
|
|
- video.setResult3(video.getResult3().replace(video.getKimiTitle(), video.getKimiTitle().replace("\"", "")));
|
|
|
+ if (StringUtils.hasText(longArticlesVideo.getResult3())) {
|
|
|
+ if (StringUtils.hasText(longArticlesVideo.getKimiTitle()) && longArticlesVideo.getKimiTitle().contains("\"")) {
|
|
|
+ longArticlesVideo.setResult3(longArticlesVideo.getResult3().replace(longArticlesVideo.getKimiTitle(),
|
|
|
+ longArticlesVideo.getKimiTitle().replace("\"", "")));
|
|
|
}
|
|
|
- jsonArray.add(resultToResponse(video.getResult3()));
|
|
|
+ jsonArray.add(resultToResponse(longArticlesVideo.getResult3()));
|
|
|
}
|
|
|
return JSONObject.toJSONString(jsonArray);
|
|
|
}
|