|
|
@@ -2,27 +2,28 @@ package com.tzld.piaoquan.longarticle.service.local.impl;
|
|
|
|
|
|
import com.alibaba.fastjson.JSONArray;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.tzld.piaoquan.longarticle.component.RedisLock;
|
|
|
import com.tzld.piaoquan.longarticle.dao.mapper.longarticle.CrawlerVideoMapper;
|
|
|
+import com.tzld.piaoquan.longarticle.dao.mapper.longarticle.SingleVideoSourceMapper;
|
|
|
import com.tzld.piaoquan.longarticle.dao.mapper.longarticle.TitleAuditMapper;
|
|
|
-import com.tzld.piaoquan.longarticle.model.dto.AuditContentRequest;
|
|
|
+import com.tzld.piaoquan.longarticle.model.bo.VideoDetail;
|
|
|
import com.tzld.piaoquan.longarticle.model.po.longarticle.*;
|
|
|
import com.tzld.piaoquan.longarticle.service.remote.MatchService;
|
|
|
+import com.tzld.piaoquan.longarticle.service.remote.VideoService;
|
|
|
import com.tzld.piaoquan.longarticle.utils.LarkRobotUtil;
|
|
|
+import com.tzld.piaoquan.longarticle.utils.VideoUtils;
|
|
|
import com.tzld.piaoquan.longarticle.utils.other.*;
|
|
|
import com.tzld.piaoquan.longarticle.utils.page.Page;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
-import org.springframework.data.redis.core.RedisTemplate;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
import org.springframework.util.CollectionUtils;
|
|
|
|
|
|
import java.nio.file.Files;
|
|
|
import java.nio.file.Paths;
|
|
|
-import java.util.ArrayList;
|
|
|
-import java.util.Date;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Random;
|
|
|
+import java.util.*;
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
@Slf4j
|
|
|
@@ -38,6 +39,15 @@ public class CrawlerVideoServiceImpl {
|
|
|
@Autowired
|
|
|
private MatchService matchService;
|
|
|
|
|
|
+ @Autowired
|
|
|
+ private RedisLock redisLock;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private SingleVideoSourceMapper singleVideoSourceMapper;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private VideoService videoService;
|
|
|
+
|
|
|
private static final String default_user_id = "69637498";
|
|
|
|
|
|
private static final Double NLP_SIMILARITY_THRESHOLD = 0.55;
|
|
|
@@ -46,6 +56,8 @@ public class CrawlerVideoServiceImpl {
|
|
|
|
|
|
private static final int MIN_NUM = 1;
|
|
|
|
|
|
+ private static final String SINGLE_VIDEO_LOCK_KEY = "single_video_lock_key_%s";
|
|
|
+
|
|
|
private static final List<String> sensitiveWords = new ArrayList<String>() {{
|
|
|
add("人民");
|
|
|
add("必胜");
|
|
|
@@ -59,7 +71,7 @@ public class CrawlerVideoServiceImpl {
|
|
|
add("习");
|
|
|
}};
|
|
|
|
|
|
- public boolean addCrawlerVideo(String contentId, String rootContentId, LongArticlesText kimiText) {
|
|
|
+ public boolean addCrawlerVideo(String contentId, String videoPoolTraceId, Long pqVideoId, String rootContentId, LongArticlesText kimiText) {
|
|
|
if (existCrawlerVideo(contentId)) {
|
|
|
return true;
|
|
|
}
|
|
|
@@ -77,6 +89,29 @@ public class CrawlerVideoServiceImpl {
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
+ //// 查询视频池是否存在该视频
|
|
|
+ //if (StringUtils.isNotEmpty(videoPoolTraceId)) {
|
|
|
+ // String lockKey = String.format(SINGLE_VIDEO_LOCK_KEY, contentId);
|
|
|
+ // String lockValue = UUID.randomUUID().toString();
|
|
|
+ // boolean lock = redisLock.tryLock(lockKey, lockValue, 300, TimeUnit.SECONDS);
|
|
|
+ // if (lock) {
|
|
|
+ // log.info("singleCrawlerVideo lock lockKey={}", lockKey);
|
|
|
+ // Boolean res = addCrawlerVideoByVideoPoolTraceId(contentId, videoPoolTraceId);
|
|
|
+ // log.info("singleCrawlerVideo unlock lockKey={}", lockKey);
|
|
|
+ // redisLock.unlock(lockKey, lockValue);
|
|
|
+ // if (res) {
|
|
|
+ // return true;
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ //}
|
|
|
+ //if (Objects.nonNull(pqVideoId)) {
|
|
|
+ // JSONObject videoDetail = VideoUtils.getVideoDetail(pqVideoId);;
|
|
|
+ // if (Objects.nonNull(videoDetail)) {
|
|
|
+ // CrawlerVideo crawlerVideo = piaoquanVideoProduce(contentId, videoDetail);
|
|
|
+ // crawlerVideo.setCrawlerTime(new Date());
|
|
|
+ // crawlerVideoMapper.insertSelective(crawlerVideo);
|
|
|
+ // }
|
|
|
+ //}
|
|
|
log.info("addCrawlerVideo contentId={} rootContentId={} kimiText={}", contentId, rootContentId, kimiText);
|
|
|
//爬虫爬取视频
|
|
|
int count = 0;
|
|
|
@@ -109,6 +144,65 @@ public class CrawlerVideoServiceImpl {
|
|
|
return count >= MIN_NUM;
|
|
|
}
|
|
|
|
|
|
+ public CrawlerVideo piaoquanVideoProduce(String contentId, JSONObject videoDetail) {
|
|
|
+ CrawlerVideo crawlerVideo = new CrawlerVideo();
|
|
|
+ crawlerVideo.setPlatform("piaoquan");
|
|
|
+ crawlerVideo.setDownloadStatus(2);
|
|
|
+ crawlerVideo.setVideoTitle(videoDetail.getString("title"));
|
|
|
+ crawlerVideo.setOutVideoId(String.valueOf(videoDetail.getLong("id")));
|
|
|
+ crawlerVideo.setVideoUrl(videoDetail.getString("videoPath"));
|
|
|
+ crawlerVideo.setVideoOssPath(videoDetail.getString("ossVideoPath"));
|
|
|
+ String shareImgPath = videoDetail.getString("shareImgPath");
|
|
|
+ if (StringUtils.isNotEmpty(shareImgPath)) {
|
|
|
+ String coverUrl = shareImgPath.substring(0, shareImgPath.indexOf("?"));
|
|
|
+ crawlerVideo.setCoverUrl(coverUrl);
|
|
|
+ String coverPath = VideoDownloader.downloadCover(crawlerVideo.getOutVideoId(), crawlerVideo.getPlatform(), crawlerVideo.getCoverUrl());
|
|
|
+ try {
|
|
|
+ String coverOssPath = OSSUploader.uploadToOSS(coverPath, "image");
|
|
|
+ crawlerVideo.setCoverOssPath(coverOssPath);
|
|
|
+ if (StringUtils.isNotEmpty(coverPath)) {
|
|
|
+ Files.delete(Paths.get(coverPath));
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("pushOss del file error", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ crawlerVideo.setContentId(contentId);
|
|
|
+ crawlerVideo.setCrawlerTime(new Date());
|
|
|
+ crawlerVideo.setUserId(default_user_id);
|
|
|
+ return crawlerVideo;
|
|
|
+ }
|
|
|
+
|
|
|
+ public Boolean addCrawlerVideoByVideoPoolTraceId(String contentId, String videoPoolTraceId) {
|
|
|
+ //判断数据是否已经存在
|
|
|
+ CrawlerVideoExample crawlerVideoExample = new CrawlerVideoExample();
|
|
|
+ crawlerVideoExample.createCriteria().andContentIdEqualTo(contentId);
|
|
|
+ long l = crawlerVideoMapper.countByExample(crawlerVideoExample);
|
|
|
+ if (l > 0) {
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+
|
|
|
+ //数据准备
|
|
|
+ SingleVideoSourceExample example = new SingleVideoSourceExample();
|
|
|
+ example.createCriteria().andContentTraceIdEqualTo(videoPoolTraceId);
|
|
|
+ List<SingleVideoSource> singleVideoSources = singleVideoSourceMapper.selectByExample(example);
|
|
|
+ if (CollectionUtils.isEmpty(singleVideoSources)) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ SingleVideoSource singleVideoSource = singleVideoSources.get(0);
|
|
|
+
|
|
|
+ //视频抓取表插入
|
|
|
+ CrawlerVideo crawlerVideo = new CrawlerVideo();
|
|
|
+ crawlerVideo.setDownloadStatus(2);
|
|
|
+ crawlerVideo.setVideoTitle(singleVideoSource.getArticleTitle());
|
|
|
+ crawlerVideo.setCoverUrl(singleVideoSource.getCoverUrl());
|
|
|
+ crawlerVideo.setVideoOssPath(singleVideoSource.getVideoOssPath());
|
|
|
+ crawlerVideo.setCrawlerTime(new Date(singleVideoSource.getCrawlerTimestamp() * 1000));
|
|
|
+ crawlerVideo.setContentId(contentId);
|
|
|
+ crawlerVideo.setPlatform(singleVideoSource.getPlatform());
|
|
|
+ crawlerVideoMapper.insertSelective(crawlerVideo);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
|
|
|
public boolean uploadCrawlerVideo(String contentId, String flowPoolLeve) {
|
|
|
if (existUploadCrawlerVideo(contentId)) {
|