Merge branch 'master' into wyp/1217-articleCategory

# Conflicts:
#	long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java
#	long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/cgi/PQVideoAuditResultEnum.java

@@ -0,0 +1,32 @@
+package com.tzld.longarticle.recommend.server.common.enums.cgi;
+
+import lombok.Getter;
+
+import java.util.Objects;
+
+@Getter
+public enum PQVideoAuditResultEnum {
+    WAITING(0, "审核中"),
+    PASS(1, "审核通过"),
+    REJECT(2, "审核不通过"),
+
+    other(999, "其他"),
+    ;
+
+    private final Integer status;
+    private final String description;
+
+    PQVideoAuditResultEnum(Integer status, String description) {
+        this.status = status;
+        this.description = description;
+    }
+
+    public static PQVideoAuditResultEnum from(Integer status) {
+        for (PQVideoAuditResultEnum statusEnum : PQVideoAuditResultEnum.values()) {
+            if (Objects.equals(statusEnum.status, status)) {
+                return statusEnum;
+            }
+        }
+        return other;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/cgi/PQVideoSensitiveLevelEnum.java

@@ -0,0 +1,34 @@
+package com.tzld.longarticle.recommend.server.common.enums.cgi;
+
+import lombok.Getter;
+
+import java.util.Objects;
+
+@Getter
+public enum PQVideoSensitiveLevelEnum {
+    NOT_CHECK(-1, "未检测"),
+    NORMAL(0, "正常"),
+    LOW_SENSITIVE(1, "初级敏感"),
+    MID_SENSITIVE(2, "中级敏感"),
+    HIGH_SENSITIVE(3, "严重敏感"),
+
+    other(999, "其他"),
+    ;
+    // -1 未检测 0 正常 1 初级敏感 2中级敏感 3 严重敏感
+    private final Integer level;
+    private final String description;
+
+    PQVideoSensitiveLevelEnum(Integer level, String description) {
+        this.level = level;
+        this.description = description;
+    }
+
+    public static PQVideoSensitiveLevelEnum from(Integer level) {
+        for (PQVideoSensitiveLevelEnum levelEnum : PQVideoSensitiveLevelEnum.values()) {
+            if (Objects.equals(levelEnum.level, level)) {
+                return levelEnum;
+            }
+        }
+        return other;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -69,6 +69,8 @@ public interface LongArticleBaseMapper {
 
     List<String> getFilterColdLongArticleTitle();
 
+    List<String> getExistsOssPath();
+
     Integer articleCategoryCount(String title);
 
     List<ArticleCategoryListVO> articleCategoryList(String title, Integer offset, Integer pageSize);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/cgi/PQVideoAuditResult.java

@@ -0,0 +1,11 @@
+package com.tzld.longarticle.recommend.server.model.cgi;
+
+import lombok.Data;
+
+@Data
+public class PQVideoAuditResult {
+    private Long taskId;
+    private Integer auditResult;
+    private Integer sensitiveLevel;
+    private String reason;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/DatastatScore.java

@@ -83,4 +83,7 @@ public class DatastatScore {
     @Column(name = "source_id")
     private String sourceId;
 
+    @Column(name = "publish_timestamp")
+    private Long publishTimestamp;
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/LongArticleVideoAudit.java

@@ -29,6 +29,9 @@ public class LongArticleVideoAudit {
     @Column(name = "status")
     private Integer status;
 
+    @Column(name = "task_id")
+    private Long taskId;
+
     @Column(name = "fail_reason")
     private String failReason;
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/PQResponse.java

@@ -0,0 +1,15 @@
+package com.tzld.longarticle.recommend.server.model.param;
+
+import lombok.Data;
+
+@Data
+public class PQResponse<T> {
+
+    private long code;
+    private String msg;
+    private T data;
+
+}
+
+
+

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/aigc/AIGCWaitingPublishContentService.java

@@ -29,7 +29,19 @@ public class AIGCWaitingPublishContentService {
 
 
     public List<Content> getAllContent(RecallParam param) {
-        List<Content> result = new ArrayList<>();
+        int retryTimes = 2;
+        while (retryTimes > 0) {
+            List<Content> result = post(param);
+            if (Objects.nonNull(result)) {
+                return result;
+            }
+            retryTimes--;
+        }
+        return new ArrayList<>();
+    }
+
+    public List<Content> post(RecallParam param) {
+        List<Content> result = null;
         JSONObject bodyParam = new JSONObject();
         JSONObject bodyParamParams = new JSONObject();
         bodyParamParams.put("accountId", param.getAccountId());
@@ -50,7 +62,7 @@ public class AIGCWaitingPublishContentService {
                     JSONObject jsonObject = JSONObject.parseObject(responseBody);
                     if (jsonObject.getInteger("code") == 0) {
                         JSONArray data = jsonObject.getJSONArray("data");
-                        result.addAll(JSONArray.parseArray(data.toJSONString(), Content.class));
+                        result = JSONArray.parseArray(data.toJSONString(), Content.class);
                     }
                 }
             }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/pq/PQVideoAuditResultService.java

@@ -0,0 +1,82 @@
+package com.tzld.longarticle.recommend.server.remote.pq;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.tzld.longarticle.recommend.server.common.HttpPoolFactory;
+import com.tzld.longarticle.recommend.server.model.cgi.PQVideoAuditResult;
+import com.tzld.longarticle.recommend.server.model.param.PQResponse;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+@Service
+@Slf4j
+public class PQVideoAuditResultService {
+
+    @Autowired
+    ArticleRepository articleRepository;
+
+    @Value("${pq.host:}")
+    private String host;
+
+    private final CloseableHttpClient client = HttpPoolFactory.aigcPool();
+    private static final String url = "/longvideoapi/openapi/audit/queryTaskResult";
+
+    public List<PQVideoAuditResult> getResult(List<Long> taskIds) {
+        int retryTimes = 3;
+        while (retryTimes > 0) {
+            List<PQVideoAuditResult> result = post(taskIds);
+            if (CollectionUtils.isNotEmpty(result)) {
+                return result;
+            }
+            retryTimes--;
+        }
+        return new ArrayList<>();
+    }
+
+    public List<PQVideoAuditResult> post(List<Long> taskIds) {
+        JSONObject params = new JSONObject();
+        params.put("taskIds", taskIds);
+        try {
+            HttpPost httpPost = new HttpPost(host + url);
+            StringEntity stringEntity = new StringEntity(params.toJSONString(), StandardCharsets.UTF_8);
+            httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
+            httpPost.setEntity(stringEntity);
+            CloseableHttpResponse response = client.execute(httpPost);
+            StatusLine statusLine = response.getStatusLine();
+            if (statusLine.getStatusCode() == 200) {
+                HttpEntity responseEntity = response.getEntity();
+                if (Objects.nonNull(responseEntity)) {
+                    String responseBody = EntityUtils.toString(responseEntity, "UTF-8");
+                    PQResponse pqResponse = JSONObject.parseObject(responseBody, PQResponse.class);
+                    if (pqResponse.getCode() == 0 && Objects.nonNull(pqResponse.getData())) {
+                        return JSONArray.parseArray(JSONObject.toJSONString(pqResponse.getData()), PQVideoAuditResult.class);
+                    } else {
+                        return null;
+                    }
+                }
+            }
+        } catch (IOException e) {
+            log.error("PQVideoAuditResultService error", e);
+        }
+        return null;
+    }
+
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/pq/PQVideoAuditStartProcessService.java

@@ -0,0 +1,79 @@
+package com.tzld.longarticle.recommend.server.remote.pq;
+
+import com.alibaba.fastjson.JSONObject;
+import com.tzld.longarticle.recommend.server.common.HttpPoolFactory;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+@Service
+@Slf4j
+public class PQVideoAuditStartProcessService {
+
+    @Autowired
+    ArticleRepository articleRepository;
+
+    @Value("${pq.host:}")
+    private String host;
+
+    private final CloseableHttpClient client = HttpPoolFactory.aigcPool();
+    private static final String url = "/longvideoapi/openapi/audit/startProcess";
+
+    public Long startProcess(Long videoId) {
+        int retryTimes = 3;
+        while (retryTimes > 0) {
+            Long taskId = post(videoId);
+            if (Objects.nonNull(taskId)) {
+                return taskId;
+            }
+            retryTimes--;
+        }
+        return null;
+    }
+
+    public Long post(Long videoId) {
+        JSONObject params = new JSONObject();
+        params.put("videoId", videoId);
+        try {
+            HttpPost httpPost = new HttpPost(host + url);
+            StringEntity stringEntity = new StringEntity(params.toJSONString(), StandardCharsets.UTF_8);
+            httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
+            httpPost.setEntity(stringEntity);
+            CloseableHttpResponse response = client.execute(httpPost);
+            StatusLine statusLine = response.getStatusLine();
+            if (statusLine.getStatusCode() == 200) {
+                HttpEntity responseEntity = response.getEntity();
+                if (Objects.nonNull(responseEntity)) {
+                    String responseBody = EntityUtils.toString(responseEntity, "UTF-8");
+                    JSONObject jsonObject = JSONObject.parseObject(responseBody);
+                    if (jsonObject.getInteger("code") == 0) {
+                        JSONObject data = jsonObject.getJSONObject("data");
+                        if (data.containsKey("taskId")) {
+                            return data.getLong("taskId");
+                        }
+                    } else {
+                        return null;
+                    }
+                }
+            }
+        } catch (IOException e) {
+            log.error("PQVideoAuditStartProcessService error", e);
+        }
+        return null;
+    }
+
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/LongArticleVideoAuditRepository.java

@@ -11,4 +11,6 @@ public interface LongArticleVideoAuditRepository extends JpaRepository<LongArtic
     LongArticleVideoAudit getByVideoId(Long videoId);
 
     List<LongArticleVideoAudit> getByVideoIdIn(List<Long> videoIds);
+
+    List<LongArticleVideoAudit> getByStatusAndTaskIdIsNotNull(Integer status);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleAuditService.java

@@ -5,18 +5,23 @@ import com.alibaba.fastjson.JSONArray;
 import com.google.common.collect.Lists;
 import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PushTypeEnum;
+import com.tzld.longarticle.recommend.server.common.enums.cgi.PQVideoAuditResultEnum;
+import com.tzld.longarticle.recommend.server.common.enums.cgi.PQVideoSensitiveLevelEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleDeleteStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
 import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
+import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
+import com.tzld.longarticle.recommend.server.model.cgi.PQVideoAuditResult;
 import com.tzld.longarticle.recommend.server.model.dto.*;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
 import com.tzld.longarticle.recommend.server.model.param.ArticleDangerFindDeleteParam;
-import com.tzld.longarticle.recommend.server.model.param.ArticleVideoAuditResultParam;
 import com.tzld.longarticle.recommend.server.model.param.PublishContentParam;
 import com.tzld.longarticle.recommend.server.remote.WxAccessTokenRemoteService;
 import com.tzld.longarticle.recommend.server.remote.WxArticleDeleteService;
+import com.tzld.longarticle.recommend.server.remote.pq.PQVideoAuditResultService;
+import com.tzld.longarticle.recommend.server.remote.pq.PQVideoAuditStartProcessService;
 import com.tzld.longarticle.recommend.server.repository.aigc.PublishAccountRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.*;
@@ -31,6 +36,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
+import java.net.URL;
 import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -61,11 +67,17 @@ public class ArticleAuditService {
     private ArticleRepository articleRepository;
     @Autowired
     private ArticleUnsafeTitleRepository articleUnsafeTitleRepository;
+    @Autowired
+    private PQVideoAuditStartProcessService pqVideoAuditStartProcessService;
+    @Autowired
+    private PQVideoAuditResultService pqVideoAuditResultService;
+    @Autowired
+    private LongArticleBaseMapper longArticleBaseMapper;
 
 
     @XxlJob("articleVideoAudit")
     public ReturnT<String> articleVideoAudit(String param) {
-        long publishTime = DateUtils.getTodayStart();
+        long publishTime = DateUtils.getTodayStart() / 1000;
         if (StringUtils.hasText(param)) {
             publishTime = DateUtils.getStartOfDay(param, "yyyyMMdd");
         }
@@ -77,50 +89,102 @@ public class ArticleAuditService {
         List<LongArticlesMatchVideo> longArticlesMatchVideoList = longArticlesMatchVideoRepository.getByTraceIdIn(traceIds);
         Map<String, LongArticlesMatchVideo> longarticlesMatchVideoMap = longArticlesMatchVideoList.stream()
                 .collect(Collectors.toMap(LongArticlesMatchVideo::getTraceId, Function.identity()));
+        List<String> existsOssPath = longArticleBaseMapper.getExistsOssPath();
         for (String traceId : traceIds) {
             LongArticlesMatchVideo longArticlesMatchVideo = longarticlesMatchVideoMap.get(traceId);
             List<LongArticlesMatchVideoResponse> responseList = JSONArray.parseArray(longArticlesMatchVideo.getResponse()
                     , LongArticlesMatchVideoResponse.class);
-            List<Long> videoIds = responseList.stream().map(LongArticlesMatchVideoResponse::getVideoID).collect(Collectors.toList());
-            List<LongArticleVideoAudit> existsList = longArticleVideoAuditRepository.getByVideoIdIn(videoIds);
-            List<Long> existsVideoIds = existsList.stream().map(LongArticleVideoAudit::getVideoId).collect(Collectors.toList());
             for (LongArticlesMatchVideoResponse response : responseList) {
-                if (existsVideoIds.contains(response.getVideoID())) {
+                String ossPath = getOssPath(response.getVideoPath());
+                if (existsOssPath.contains(ossPath)) {
                     continue;
                 }
+                existsOssPath.add(ossPath);
                 LongArticleVideoAudit videoAudit = new LongArticleVideoAudit();
                 videoAudit.setVideoId(response.getVideoID());
                 videoAudit.setTraceId(traceId);
                 videoAudit.setContentId(longArticlesMatchVideo.getContentId());
-                videoAudit.setOssPath(response.getVideoOSS());
+                videoAudit.setOssPath(ossPath);
                 videoAudit.setStatus(StatusEnum.ZERO.getCode());
                 videoAudit.setCreateTimestamp(System.currentTimeMillis());
                 longArticleVideoAuditRepository.save(videoAudit);
-                // todo 调用PQ 审核视频
+                // 调用PQ 审核视频
+                try {
+                    Long taskId = pqVideoAuditStartProcessService.startProcess(response.getVideoID());
+                    if (Objects.nonNull(taskId)) {
+                        videoAudit.setTaskId(taskId);
+                        longArticleVideoAuditRepository.save(videoAudit);
+                    } else {
+                        log.error("PQVideoAuditStartProcess start process error videoId:{} ", response.getVideoID());
+                    }
+                } catch (Exception e) {
+                    log.error("PQVideoAuditStartProcess start process videoId:{} error:{}", response.getVideoID(), e.getMessage());
+                }
             }
         }
 
         return ReturnT.SUCCESS;
     }
 
-    /**
-     * 视频审核结果处理
-     */
-    public void articleVideoAuditResult(ArticleVideoAuditResultParam param) {
-        LongArticleVideoAudit longArticleVideoAudit = longArticleVideoAuditRepository.getByVideoId(param.getVideoId());
-        if (param.getResult() == 1) {
+    private String getOssPath(String ossUrl) {
+        String result = "";
+        if (ossUrl == null || ossUrl.isEmpty()) {
+            return result;
+        }
+        try {
+            URL url = new URL(ossUrl);
+            // 去除域名
+            result = url.getPath();
+            // 获取后缀.位置
+            int endIndex = result.lastIndexOf(".");
+            if (endIndex == -1) {
+                endIndex = result.length();
+            }
+            // 去除 / 及 文件后缀
+            result = result.substring(1, endIndex);
+        } catch (Exception e) {
+            log.error("getOssPath error:{}", e.getMessage());
+        }
+        return result;
+    }
+
+    @XxlJob("articleVideoAuditResult")
+    public ReturnT<String> articleVideoAuditResult(String param) {
+        List<LongArticleVideoAudit> list = longArticleVideoAuditRepository.getByStatusAndTaskIdIsNotNull(StatusEnum.ZERO.getCode());
+        for (List<LongArticleVideoAudit> partition : Lists.partition(list, 10)) {
+            List<Long> taskIds = partition.stream().map(LongArticleVideoAudit::getTaskId).collect(Collectors.toList());
+            Map<Long, LongArticleVideoAudit> map = partition.stream().collect(Collectors.toMap(LongArticleVideoAudit::getTaskId, Function.identity()));
+            List<PQVideoAuditResult> resultList = pqVideoAuditResultService.getResult(taskIds);
+            resultList.forEach(result -> saveVideoAuditResult(map.get(result.getTaskId()), result));
+        }
+        return ReturnT.SUCCESS;
+    }
+
+
+    public void saveVideoAuditResult(LongArticleVideoAudit longArticleVideoAudit, PQVideoAuditResult result) {
+
+        if (Objects.equals(result.getAuditResult(), PQVideoAuditResultEnum.PASS.getStatus())) {
             // 审核通过，更新文章状态
-            longArticleVideoAudit.setStatus(ArticleDeleteStatusEnum.SUCCESS.getCode());
+            longArticleVideoAudit.setStatus(PQVideoAuditResultEnum.PASS.getStatus());
             longArticleVideoAudit.setFinishTimestamp(System.currentTimeMillis());
             longArticleVideoAuditRepository.save(longArticleVideoAudit);
-        } else {
+        } else if (Objects.equals(result.getAuditResult(), PQVideoAuditResultEnum.REJECT.getStatus())) {
             // 审核不通过，删除文章
-            longArticleVideoAudit.setStatus(ArticleDeleteStatusEnum.FAIL.getCode());
-            longArticleVideoAudit.setFailReason(param.getFailReason());
+            longArticleVideoAudit.setStatus(PQVideoAuditResultEnum.REJECT.getStatus());
+            PQVideoSensitiveLevelEnum sensitiveLevelEnum = PQVideoSensitiveLevelEnum.from(result.getSensitiveLevel());
+            longArticleVideoAudit.setFailReason(sensitiveLevelEnum.getDescription());
             longArticleVideoAudit.setFinishTimestamp(System.currentTimeMillis());
             longArticleVideoAuditRepository.save(longArticleVideoAudit);
             // 构建删除文章记录 并保存
-            saveDeleteRecord(longArticleVideoAudit.getOssPath());
+            // saveDeleteRecord(longArticleVideoAudit.getOssPath());
+            // 暂时不做删除 先发送通知
+            FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.ARTICLE_DELETE.getRobotId(),
+                    "视频审核不通过\n" +
+                            "traceId：" + longArticleVideoAudit.getTraceId() + "\n" +
+                            "视频id：" + longArticleVideoAudit.getVideoId() + "\n" +
+                            "视频path：" + longArticleVideoAudit.getOssPath() + "\n" +
+                            "敏感等级：" + sensitiveLevelEnum.getDescription() + "\n" +
+                            "管理后台地址：https://admin.piaoquantv.com/cms/post-detail/" + longArticleVideoAudit.getVideoId() + "/detail");
         }
     }
 
@@ -150,7 +214,7 @@ public class ArticleAuditService {
         Map<String, String> publishPushIdMap = pushContentRelList.stream()
                 .collect(Collectors.toMap(PublishGzhPushContentRelDTO::getPublishContentId,
                         PublishGzhPushContentRelDTO::getPushId,
-                         (o1, o2) -> o2));
+                        (o1, o2) -> o2));
         List<PublishGzhPushDTO> pushList = aigcBaseMapper.getPushByPushIdIn(pushIds);
         Map<String, PublishGzhPushDTO> pushDTOMap = pushList.stream()
                 .collect(Collectors.toMap(PublishGzhPushDTO::getPushId, Function.identity()));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/DataDashboardService.java

@@ -45,6 +45,7 @@ import com.xxl.job.core.biz.model.ReturnT;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.util.Pair;
 import org.springframework.http.*;
@@ -339,7 +340,7 @@ public class DataDashboardService {
             setObjArticleDetailInfo(article, obj, articleDetailInfos);
             Article firstArticle = articleMap.get(article.getGhId()).get(article.getAppMsgId()).get(1);
             Map<String, Map<String, PublishSortLog>> dateStrategy = sortStrategyMap.get(article.getGhId());
-            AccountAvgInfo avgInfo = getAccountAvgInfo(accountAvgInfoIndexMap, article.getGhId(),
+            Map<String, AccountAvgInfo> indexAvgInfoMap = getDateAccountAvgInfo(accountAvgInfoIndexMap, article.getGhId(),
                     article.getPublishTimestamp(), article.getItemIndex());
             AccountAvgInfo firstAvgInfo = getAccountAvgInfo(accountAvgInfoIndexMap, article.getGhId(),
                     article.getPublishTimestamp(), 1);
@@ -377,7 +378,7 @@ public class DataDashboardService {
                     Objects.equals(articleType, ArticleTypeEnum.WUXIANLIU.getVal())) {
                 obj.setStrategy(RankStrategyEnum.INFINITE_STRATEGY.getStrategy());
             }
-            setObjAvgInfo(article, obj, avgInfo);
+            setObjAvgInfo(article, obj, indexAvgInfoMap);
             setObjHisRateInfo(article, obj, hisArticleMap, accountAvgInfoIndexMap, hisArticleDetailInfoMap);
             // aigc 数据
             setObjAigcInfo(article, obj, date, publishAccountMap, publishContentMap, publishContentLayoutMap,
@@ -601,14 +602,24 @@ public class DataDashboardService {
         }
     }
 
-    private void setObjAvgInfo(Article article, DatastatSortStrategy obj, AccountAvgInfo avgInfo) {
+    private void setObjAvgInfo(Article article, DatastatSortStrategy obj, Map<String, AccountAvgInfo> indexAvgInfoMap) {
+        if (MapUtils.isNotEmpty(indexAvgInfoMap)) {
+            AccountAvgInfo info = null;
+            for (String index : indexAvgInfoMap.keySet()) {
+                info = indexAvgInfoMap.get(index);
+                break;
+            }
+            if (Objects.nonNull(info)) {
+                obj.setAccountMode(info.getAccountMode());
+                obj.setAccountSource(info.getAccountSource());
+                obj.setAccountType(info.getAccountType());
+                obj.setAccountStatus(info.getAccountStatus());
+                obj.setBusinessType(AccountBusinessTypeEnum.from(info.getBusinessType()).getDescription());
+                obj.setFans(info.getFans());
+            }
+        }
+        AccountAvgInfo avgInfo = indexAvgInfoMap.get(article.getItemIndex().toString());
         if (Objects.nonNull(avgInfo)) {
-            obj.setAccountMode(avgInfo.getAccountMode());
-            obj.setAccountSource(avgInfo.getAccountSource());
-            obj.setAccountType(avgInfo.getAccountType());
-            obj.setAccountStatus(avgInfo.getAccountStatus());
-            obj.setBusinessType(AccountBusinessTypeEnum.from(avgInfo.getBusinessType()).getDescription());
-            obj.setFans(avgInfo.getFans());
             obj.setAvgViewCount(avgInfo.getReadAvg());
             if (avgInfo.getReadAvg() > 0) {
                 obj.setReadRate((article.getShowViewCount() * 1.0) / avgInfo.getReadAvg());
@@ -1066,6 +1077,21 @@ public class DataDashboardService {
         return avgInfo;
     }
 
+    private Map<String, AccountAvgInfo> getDateAccountAvgInfo(Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap,
+                                                              String ghId, Long updateTime, Integer itemIndex) {
+        Map<String, Map<String, AccountAvgInfo>> dateAvgMap = accountAvgInfoIndexMap.get(ghId);
+        String hisPublishDate = DateUtils.timestampToYMDStr(updateTime, "yyyy-MM-dd");
+        if (Objects.nonNull(dateAvgMap)) {
+            List<String> avgMapDateList = new ArrayList<>(dateAvgMap.keySet());
+            hisPublishDate = DateUtils.findNearestDate(avgMapDateList, hisPublishDate, "yyyy-MM-dd");
+            Map<String, AccountAvgInfo> accountAvgInfoMap = dateAvgMap.get(hisPublishDate);
+            if (Objects.nonNull(accountAvgInfoMap)) {
+                return accountAvgInfoMap;
+            }
+        }
+        return new HashMap<>();
+    }
+
     private void setPublishFuturePerformance(IntermediateIndicatorsExport item, DatastatSortStrategy data, Integer poolLevel,
                                              Map<String, ArticlePoolPromotionSource> promotionSourceMap,
                                              Map<String, List<Article>> futurePublishMap,
@@ -1205,11 +1231,11 @@ public class DataDashboardService {
             DatastatSortStrategy obj = new DatastatSortStrategy();
             List<ArticleDetailInfo> articleDetailInfos = articleDetailInfoMap.get(article.getWxSn());
             setObjArticleDetailInfo(article, obj, articleDetailInfos);
-            AccountAvgInfo avgInfo = getAccountAvgInfo(accountAvgInfoIndexMap, article.getGhId(),
+            Map<String, AccountAvgInfo> indexAvgInfoMap = getDateAccountAvgInfo(accountAvgInfoIndexMap, article.getGhId(),
                     article.getPublishTimestamp(), article.getItemIndex());
             String date = DateUtils.timestampToYMDStr(article.getPublishTimestamp(), "yyyyMMdd");
             setObjBaseInfo(article, obj, date);
-            setObjAvgInfo(article, obj, avgInfo);
+            setObjAvgInfo(article, obj, indexAvgInfoMap);
             result.add(obj);
         }
         return result;
@@ -1633,7 +1659,7 @@ public class DataDashboardService {
                 item.setFirstExplorationIntervalAvg(explorationInterval);
             }
             result.add(item);
-            saveList.add(buildDatastatScore(publishSortLog, item));
+            saveList.add(buildDatastatScore(publishSortLog, item, article));
         }
         saveDatastatScore(dateStrList, saveList);
         result = result.stream().filter(o -> o.getIndex() == 1).collect(Collectors.toList());
@@ -1642,7 +1668,7 @@ public class DataDashboardService {
         return result;
     }
 
-    private DatastatScore buildDatastatScore(PublishSortLog publishSortLog, FirstContentScoreExport value) {
+    private DatastatScore buildDatastatScore(PublishSortLog publishSortLog, FirstContentScoreExport value, Article article) {
         DatastatScore item = new DatastatScore();
         item.setDt(value.getDateStr());
         item.setGhId(value.getGhId());
@@ -1665,6 +1691,7 @@ public class DataDashboardService {
         item.setPublishContentId(publishSortLog.getPublishContentId());
         item.setCrawlerChannelContentId(publishSortLog.getCrawlerChannelContentId());
         item.setSourceId(publishSortLog.getSourceId());
+        item.setPublishTimestamp(article.getPublishTimestamp());
         return item;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV15Strategy.java

@@ -2,6 +2,7 @@ package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
@@ -19,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * @author dyp
@@ -123,7 +125,21 @@ public class RankV15Strategy implements RankStrategy {
         RankService.commonAddSecondContent(param, result, publishPool, contentPools, contentMap, indexReplacePoolConfigMap);
 
         // 3-8
-        RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
+        // RankService.commonAdd38Content(param, result, contentPools, contentMap, param.getStrategy());
+        List<Content> pool = contentMap.get(contentPools[2]);
+        Integer videoSourceType = PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal();
+        Queue<Content> videoPoolQueue = pool.stream().filter(o -> Objects.equals(o.getSourceType(), videoSourceType))
+                .collect(Collectors.toCollection(LinkedList::new));
+        Queue<Content> otherPoolQueue = pool.stream().filter(o -> !Objects.equals(o.getSourceType(), videoSourceType))
+                .collect(Collectors.toCollection(LinkedList::new));
+        for (int i = 3; i < param.getSize() + 1; i++) {
+            Integer sourceType = RankService.getStrategyPoolSourceType(param.getStrategy(), i);
+            if (Objects.equals(sourceType, videoSourceType) && !videoPoolQueue.isEmpty()) {
+                result.add(videoPoolQueue.poll());
+            } else if (!otherPoolQueue.isEmpty()) {
+                result.add(otherPoolQueue.poll());
+            }
+        }
 
         RankStrategy.deduplication(result, contentMap, publishPool);
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -341,6 +341,13 @@ public class RecallService implements ApplicationContextAware {
             if (CollectionUtils.isEmpty(cacheList)) {
                 return;
             }
+            List<String> sourceIds = cacheList.stream().map(ArticleTitleHisCache::getSourceId).collect(Collectors.toList());
+            List<ArticleTitleHisCache> existsList = articleTitleHisCacheRepository.getBySourceIdInAndType(sourceIds, type);
+            Set<String> existsIds = existsList.stream().map(ArticleTitleHisCache::getSourceId).collect(Collectors.toSet());
+            cacheList = cacheList.stream().filter(o -> !existsIds.contains(o.getSourceId())).collect(Collectors.toList());
+            if (CollectionUtils.isEmpty(cacheList)) {
+                return;
+            }
             longArticleBaseMapper.batchInsertArticleTitleHisCache(cacheList);
         } catch (Exception e) {
             log.error("saveArticleTitleHisCache error:{}", e.getMessage());

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/recommend/ArticleAuditController.java

@@ -2,7 +2,6 @@ package com.tzld.longarticle.recommend.server.web.recommend;
 
 import com.tzld.longarticle.recommend.server.common.response.CommonResponse;
 import com.tzld.longarticle.recommend.server.model.param.ArticleDangerFindDeleteParam;
-import com.tzld.longarticle.recommend.server.model.param.ArticleVideoAuditResultParam;
 import com.tzld.longarticle.recommend.server.service.recommend.ArticleAuditService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -22,8 +21,8 @@ public class ArticleAuditController {
         service.articleVideoAudit(dateStr);
     }
 
-    @PostMapping("/articleVideoAuditResult")
-    public CommonResponse<Void> articleVideoAuditResult(@RequestBody ArticleVideoAuditResultParam param) {
+    @GetMapping("/articleVideoAuditResult")
+    public CommonResponse<Void> articleVideoAuditResult(String param) {
         service.articleVideoAuditResult(param);
         return CommonResponse.success();
     }

long-article-recommend-service/src/main/resources/application-dev.yml

@@ -131,4 +131,7 @@ pushMessage:
     groupId: GID_3RD_PARTY_PUSH_MESSAGE_CALLBACK_DEV
     tag: mini
 
-small_page_url: https://testapi.piaoquantv.com
+small_page_url: https://testapi.piaoquantv.com
+
+pq:
+  host: https://videotest.yishihui.com

long-article-recommend-service/src/main/resources/application-pre.yml

@@ -163,4 +163,7 @@ aliyun:
         project: wqsd-video
         store: video_blacklist_security_filter_log
 
-small_page_url: https://testapi.piaoquantv.com
+small_page_url: https://testapi.piaoquantv.com
+
+pq:
+  host: https://videopre.piaoquantv.com

long-article-recommend-service/src/main/resources/application-prod.yml

@@ -121,4 +121,7 @@ pushMessage:
     groupId: GID_3RD_PARTY_PUSH_MESSAGE_CALLBACK_PROD
     tag: mini
 
-small_page_url: https://api.piaoquantv.com
+small_page_url: https://api.piaoquantv.com
+
+pq:
+  host: https://longvideoapi.piaoquantv.com

long-article-recommend-service/src/main/resources/application-test.yml

@@ -153,4 +153,7 @@ aliyun:
         project: wqsd-video-test
         store: video_blacklist_security_filter_log
 
-small_page_url: https://testapi.piaoquantv.com
+small_page_url: https://testapi.piaoquantv.com
+
+pq:
+  host: https://videotest.yishihui.com

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -74,14 +74,14 @@
         (dt, gh_id, account_name, `index`, title, strategy, score, similarity, view_count_rate,
         his_fission_avg_read_rate_rate, his_fission_avg_read_sum_rate, his_fission_de_weight_avg_read_sum_rate,
         read_count, read_avg, read_avg_rate, category, category_score, first_pub_interval, publish_content_id,
-        crawler_channel_content_id, source_id)
+        crawler_channel_content_id, source_id, publish_timestamp)
         VALUES
         <foreach collection="list" item="item" separator=",">
             (#{item.dt}, #{item.ghId}, #{item.accountName}, #{item.index}, #{item.title}, #{item.strategy},
             #{item.score}, #{item.similarity}, #{item.viewCountRate}, #{item.hisFissionAvgReadRateRate},
             #{item.hisFissionAvgReadSumRate}, #{item.hisFissionDeWeightAvgReadSumRate}, #{item.readCount},
             #{item.readAvg}, #{item.readAvgRate}, #{item.category}, #{item.categoryScore}, #{item.firstPubInterval},
-            #{item.publishContentId}, #{item.crawlerChannelContentId}, #{item.sourceId})
+            #{item.publishContentId}, #{item.crawlerChannelContentId}, #{item.sourceId}, #{item.publishTimestamp})
         </foreach>
     </insert>
 
@@ -256,6 +256,10 @@
         </foreach>
     </insert>
 
+    <select id="getExistsOssPath" resultType="java.lang.String">
+        select distinct oss_path from long_articles_video_audit
+    </select>
+
     <select id="articleCategoryList"
             resultType="com.tzld.longarticle.recommend.server.model.vo.ArticleCategoryListVO">
         select produce_content_id, title, category