Merge branch 'wyp/1220-articleDeleteByOssPath' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/aigc/PublishContentStatusEnum.java

@@ -0,0 +1,38 @@
+package com.tzld.longarticle.recommend.server.common.enums.aigc;
+
+import lombok.Getter;
+
+import java.util.Objects;
+
+@Getter
+public enum PublishContentStatusEnum {
+    not_publish(0, "未发布"),
+    waiting_publish(1, "待发布"),
+    published(2, "已发布"),
+    no_publish(3, "不发布"),
+    publishing(4, "发布中"),
+    publish_fail(5, "发布失败"),
+    platformReview(6, "平台审核中"),
+    input_delete(7, "输入删除"),
+    input_deprecated(8, "输入废弃"),
+    other(999, "其他"),
+    ;
+
+    private final Integer val;
+    private final String description;
+
+    PublishContentStatusEnum(Integer val, String description) {
+        this.val = val;
+        this.description = description;
+    }
+
+    public static PublishContentStatusEnum from(Integer val) {
+        for (PublishContentStatusEnum statusEnum : PublishContentStatusEnum.values()) {
+            if (Objects.equals(statusEnum.val, val)) {
+                return statusEnum;
+            }
+        }
+        return other;
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/ArticleMatchContentStatusEnum.java

@@ -0,0 +1,35 @@
+package com.tzld.longarticle.recommend.server.common.enums.recommend;
+
+import lombok.Getter;
+
+@Getter
+public enum ArticleMatchContentStatusEnum {
+    PROCESSING_STATUS(101, "任务处理中"),
+    INIT_STATUS(0, "任务初始化状态"),
+    KIMI_FINISHED_STATUS(1, "Kimi 执行完成状态"),
+    SPIDER_FINISHED_STATUS(2, "爬虫执行完成状态"),
+    ETL_COMPLETE_STATUS(3, "任务视频ETL 完成状态"),
+    PUBLISHED_STATUS(4, "任务发布完成状态"),
+    MISMATCH_STATUS(96, "文章品类不匹配"),
+    EXIT_STATUS(97, "文章已经退场 or 晋级"),
+    FAIL_STATUS(99, "失败"),
+    ;
+
+    private Integer code;
+    private String desc;
+
+    ArticleMatchContentStatusEnum(Integer code, String desc) {
+        this.code = code;
+        this.desc = desc;
+    }
+
+    public static ArticleMatchContentStatusEnum from(Integer code) {
+        for (ArticleMatchContentStatusEnum status : ArticleMatchContentStatusEnum.values()) {
+            if (status.getCode().equals(code)) {
+                return status;
+            }
+        }
+        return null;
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/crawler/CrawlerBaseMapper.java

@@ -2,7 +2,7 @@ package com.tzld.longarticle.recommend.server.mapper.crawler;
 
 import com.tzld.longarticle.recommend.server.model.dto.ArticleMatchVideos;
 import com.tzld.longarticle.recommend.server.model.dto.GetOffVideos;
-import com.tzld.longarticle.recommend.server.model.dto.LongArticlesText;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesText;
 import com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideoDTO;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountCorrelation;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -3,7 +3,7 @@ package com.tzld.longarticle.recommend.server.mapper.longArticle;
 import com.tzld.longarticle.recommend.server.model.dto.GetOffVideos;
 import com.tzld.longarticle.recommend.server.model.dto.LongArticlesCrawlerVideos;
 import com.tzld.longarticle.recommend.server.model.dto.LongArticlesMatchVideos;
-import com.tzld.longarticle.recommend.server.model.dto.LongArticlesText;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesText;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
 import com.tzld.longarticle.recommend.server.model.param.ArticleVideoPoolSourceParam;
 import com.tzld.longarticle.recommend.server.model.vo.ArticleCategoryListVO;
@@ -80,4 +80,6 @@ public interface LongArticleBaseMapper {
     void updateDatastatScoreCategory(String title, String category);
 
     void updateVideoPoolContentBad(String contentTraceId);
+
+    void batchInsertArticleReMatchRecord(List<ArticleReMatchRecord> list);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/LongArticlesText.java

@@ -1,14 +0,0 @@
-package com.tzld.longarticle.recommend.server.model.dto;
-
-import lombok.Data;
-
-@Data
-public class LongArticlesText {
-    private String contentId;
-    private String articleTitle;
-    private String articleText;
-    private String kimiTitle;
-    private String kimiSummary;
-    private String kimiKeys;
-    private Integer kimiStatus;
-}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleReMatchRecord.java

@@ -0,0 +1,44 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.*;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "article_title_his_cache")
+public class ArticleReMatchRecord {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id")
+    private Integer id;
+
+    @Column(name = "trace_id")
+    private String traceId;
+
+    @Column(name = "content_id")
+    private String contentId;
+
+    @Column(name = "oss_path")
+    private String ossPath;
+
+    @Column(name = "status")
+    private Integer status;
+
+    @Column(name = "old_response")
+    private String oldResponse;
+
+    @Column(name = "new_response")
+    private String newResponse;
+
+    @Column(name = "create_timestamp")
+    private Long createTimestamp;
+
+    @Column(name = "finish_timestamp")
+    private Long finishTimestamp;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/LongArticlesText.java

@@ -0,0 +1,34 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "long_articles_text")
+public class LongArticlesText {
+
+    @Id
+    @Column(name = "content_id")
+    private String contentId;
+    @Column(name = "article_title")
+    private String articleTitle;
+    @Column(name = "article_text")
+    private String articleText;
+    @Column(name = "kimi_title")
+    private String kimiTitle;
+    @Column(name = "kimi_summary")
+    private String kimiSummary;
+    @Column(name = "kimi_keys")
+    private String kimiKeys;
+    @Column(name = "kimi_status")
+    private Integer kimiStatus;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/LongArticlesMatchVideoRepository.java

@@ -12,4 +12,6 @@ public interface LongArticlesMatchVideoRepository extends JpaRepository<LongArti
     List<LongArticlesMatchVideo> getByTraceIdIn(List<String> traceIds);
 
     LongArticlesMatchVideo getByTraceId(String traceId);
+
+    List<LongArticlesMatchVideo> getByContentIdIn(List<String> contentIds);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/LongArticlesTextRepository.java

@@ -0,0 +1,12 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesText;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+@Repository
+public interface LongArticlesTextRepository extends JpaRepository<LongArticlesText, String> {
+    List<LongArticlesText> getByContentIdIn(List<String> contentIds);
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/DataFlushService.java

@@ -11,6 +11,7 @@ import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.model.dto.*;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesRootSourceId;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesText;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleAuditService.java

@@ -4,17 +4,20 @@ import cn.hutool.core.collection.CollectionUtil;
 import com.alibaba.fastjson.JSONArray;
 import com.google.common.collect.Lists;
 import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
+import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishContentStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PushTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.cgi.PQVideoAuditResultEnum;
 import com.tzld.longarticle.recommend.server.common.enums.cgi.PQVideoSensitiveLevelEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleDeleteStatusEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleMatchContentStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
 import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.model.cgi.PQVideoAuditResult;
 import com.tzld.longarticle.recommend.server.model.dto.*;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
+import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishContent;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
 import com.tzld.longarticle.recommend.server.model.param.ArticleDangerFindDeleteParam;
@@ -24,6 +27,7 @@ import com.tzld.longarticle.recommend.server.remote.WxArticleDeleteService;
 import com.tzld.longarticle.recommend.server.remote.pq.PQVideoAuditResultService;
 import com.tzld.longarticle.recommend.server.remote.pq.PQVideoAuditStartProcessService;
 import com.tzld.longarticle.recommend.server.repository.aigc.PublishAccountRepository;
+import com.tzld.longarticle.recommend.server.repository.aigc.PublishContentRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.*;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
@@ -74,6 +78,10 @@ public class ArticleAuditService {
     private PQVideoAuditResultService pqVideoAuditResultService;
     @Autowired
     private LongArticleBaseMapper longArticleBaseMapper;
+    @Autowired
+    private PublishContentRepository publishContentRepository;
+    @Autowired
+    private LongArticlesTextRepository longArticlesTextRepository;
 
 
     @XxlJob("articleVideoAudit")
@@ -201,18 +209,74 @@ public class ArticleAuditService {
         }
     }
 
-    private void saveDeleteRecord(String ossPath) {
+    public void saveDeleteRecord(String ossPath) {
         List<LongArticleCrawlerVideo> crawlerVideoList = longArticleCrawlerVideoRepository.getByVideoOssPath(ossPath);
-        List<String> traceIds = crawlerVideoList.stream().map(LongArticleCrawlerVideo::getTraceId).collect(Collectors.toList());
-        List<PublishContentMiniprogramDTO> publishContentList = aigcBaseMapper.getPublishContentByTraceIdIn(traceIds);
-        List<String> publishContentIds = publishContentList.stream().map(PublishContentMiniprogramDTO::getPublishContentId).collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(crawlerVideoList)) {
+            return;
+        }
+        for (LongArticleCrawlerVideo longArticleCrawlerVideo : crawlerVideoList) {
+            longArticleCrawlerVideo.setIsIllegal(1);
+            longArticleCrawlerVideoRepository.save(longArticleCrawlerVideo);
+        }
+        List<String> contentIds = crawlerVideoList.stream().map(LongArticleCrawlerVideo::getContentId).collect(Collectors.toList());
+        List<LongArticlesMatchVideo> matchVideoList = longArticlesMatchVideoRepository.getByContentIdIn(contentIds);
+        Map<String, LongArticlesMatchVideo> matchVideoMap = matchVideoList.stream()
+               .collect(Collectors.toMap(LongArticlesMatchVideo::getTraceId, Function.identity()));
+        List<String> traceIds = matchVideoList.stream().map(LongArticlesMatchVideo::getTraceId).collect(Collectors.toList());
+        List<PublishContentMiniprogramDTO> publishContentMiniprogramList = aigcBaseMapper.getPublishContentByTraceIdIn(traceIds);
+        List<String> publishContentIds = publishContentMiniprogramList.stream()
+                .map(PublishContentMiniprogramDTO::getPublishContentId).collect(Collectors.toList());
+        Map<String, String> PublishTraceIdMap = publishContentMiniprogramList.stream()
+               .collect(Collectors.toMap(PublishContentMiniprogramDTO::getPublishContentId, PublishContentMiniprogramDTO::getTraceId));
+        // 过滤状态非已发布内容
+        List<PublishContent> publishContentList = publishContentRepository.getByIdIn(publishContentIds);
+        List<String> publishedIds = publishContentList.stream()
+                .filter(o -> Objects.equals(o.getStatus(), PublishContentStatusEnum.published.getVal()))
+                .map(PublishContent::getId).collect(Collectors.toList());
+        publishContentIds = publishContentIds.stream().filter(publishedIds::contains).collect(Collectors.toList());
+        List<String> deleteTraceIds = publishContentIds.stream().map(PublishTraceIdMap::get).collect(Collectors.toList());
+        // 重新匹配小程序
+        List<String> reMatchTraceIds = traceIds.stream().filter(o -> !deleteTraceIds.contains(o)).collect(Collectors.toList());
+        buildReMatchRecord(reMatchTraceIds, ossPath, matchVideoMap);
+        // 文章删除
         buildArticleAuditDelete(publishContentIds);
     }
 
+    private void buildReMatchRecord(List<String> reMatchTraceIds, String ossPath,
+                                    Map<String, LongArticlesMatchVideo> matchVideoMap) {
+        if (CollectionUtils.isEmpty(reMatchTraceIds)) {
+            return;
+        }
+        Long now = System.currentTimeMillis() / 1000;
+        List<ArticleReMatchRecord> reMatchRecordList = new ArrayList<>();
+        for (String traceId : reMatchTraceIds) {
+            LongArticlesMatchVideo matchVideo = matchVideoMap.get(traceId);
+            if (Objects.isNull(matchVideo)
+                    || ArticleMatchContentStatusEnum.ETL_COMPLETE_STATUS.getCode().equals(matchVideo.getContentStatus())
+                    || ArticleMatchContentStatusEnum.MISMATCH_STATUS.getCode().equals(matchVideo.getContentStatus())
+                    || ArticleMatchContentStatusEnum.EXIT_STATUS.getCode().equals(matchVideo.getContentStatus())
+                    || ArticleMatchContentStatusEnum.FAIL_STATUS.getCode().equals(matchVideo.getContentStatus())) {
+                continue;
+            }
+            ArticleReMatchRecord record = new ArticleReMatchRecord();
+            record.setTraceId(traceId);
+            record.setContentId(matchVideo.getContentId());
+            record.setOssPath(ossPath);
+            record.setStatus(0);
+            if (StringUtils.hasText(matchVideo.getResponse())) {
+                record.setOldResponse(matchVideo.getResponse());
+            }
+            record.setCreateTimestamp(now);
+            reMatchRecordList.add(record);
+        }
+        longArticleBaseMapper.batchInsertArticleReMatchRecord(reMatchRecordList);
+    }
+
     private void buildArticleAuditDelete(List<String> publishContentIds) {
         if (CollectionUtils.isEmpty(publishContentIds)) {
             return;
         }
+        // 过滤已删除数据
         List<LongArticleAuditDelete> existList = longArticleAuditDeleteRepository.getByPublishContentIdIn(publishContentIds);
         List<String> existContentIds = existList.stream().map(LongArticleAuditDelete::getPublishContentId).collect(Collectors.toList());
         publishContentIds = publishContentIds.stream().filter(id -> !existContentIds.contains(id)).collect(Collectors.toList());
@@ -244,7 +308,7 @@ public class ArticleAuditService {
                 .collect(Collectors.groupingBy(PublishGzhPushContentRelDTO::getPushId));
         List<PublishAccountTypeDTO> accountTypeList = aigcBaseMapper.getAccountTypeList();
         Map<String, PublishAccountTypeDTO> accountTypeMap = accountTypeList.stream()
-               .collect(Collectors.toMap(PublishAccountTypeDTO::getGhId, Function.identity()));
+                .collect(Collectors.toMap(PublishAccountTypeDTO::getGhId, Function.identity()));
         // 删除文章
         for (String publishContentId : publishContentIds) {
             String pushId = publishPushIdMap.get(publishContentId);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/recommend/ArticleAuditController.java

@@ -27,6 +27,12 @@ public class ArticleAuditController {
         return CommonResponse.success();
     }
 
+    @GetMapping("/saveDeleteRecord")
+    public CommonResponse<Void> saveDeleteRecord(String ossPath) {
+        service.saveDeleteRecord(ossPath);
+        return CommonResponse.success();
+    }
+
     @PostMapping("/titleDangerFindDelete")
     public CommonResponse<Void> titleDangerFindDelete(@RequestBody ArticleDangerFindDeleteParam param) {
         service.titleDangerFindDelete(param);

long-article-recommend-service/src/main/resources/mapper/crawler/CrawlerBaseMapper.xml

@@ -35,7 +35,7 @@
             resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesVideoDTO">
         select * from long_articles_video where content_status = 2 and success = 0 and id > #{id} order by id limit #{pageSize}
     </select>
-    <select id="getLongArticlesText" resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesText">
+    <select id="getLongArticlesText" resultType="com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesText">
         select content_id, kimi_title, kimi_summary, kimi_keys from long_articles_video where kimi_summary is not null group by content_id
     </select>
 

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -168,7 +168,7 @@
     </select>
 
     <select id="getNeedUpdateRecords"
-            resultType="com.tzld.longarticle.recommend.server.model.dto.LongArticlesText">
+            resultType="com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticlesText">
         select * from long_articles_text where kimi_title is null
     </select>
 
@@ -296,4 +296,14 @@
         update publish_single_video_source set bad_status = 1 where content_trace_id = #{contentTraceId};
     </update>
 
+    <insert id="batchInsertArticleReMatchRecord">
+        insert into article_re_match_record
+        (trace_id, content_id, oss_path, status, old_response, create_timestamp)
+        values
+        <foreach collection="list" item="item" separator=",">
+            (#{item.traceId}, #{item.contentId}, #{item.ossPath}, #{item.status}, #{item.oldResponse},
+             #{item.createTimestamp})
+        </foreach>
+    </insert>
+
 </mapper>