Merge branch 'wyp/0611-syncPublishContent' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/PublishContentMapper.java

@@ -71,4 +71,6 @@ public interface PublishContentMapper {
     List<PublishPlanMiniprogramTask> getAccountMiniprogramTaskList(String id);
 
     List<PublishAccountRemarkDTO> getPublishAccountRemarkList(List<String> accountIds);
+
+    List<PublishPlanAccountDTO> getGroupPublishPlanAccounts();
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -107,4 +107,6 @@ public interface LongArticleBaseMapper {
     void batchInsertPublishContentGzhWaiting(List<PublishContentGzhWaiting> list);
 
     void updatePublishContentGzhWaitingStatus(String planId, String accountId, List<String> contentIds, Integer status, Long updateTimestamp);
+
+    void updatePublishContentGzhWaitingStatusByIds(List<String> contentIds, Integer status);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/aigc/PublishPlanAccountDTO.java

@@ -6,4 +6,5 @@ import lombok.Data;
 public class PublishPlanAccountDTO {
     private String planId;
     private String accountId;
+    private Integer cnt;
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/aigc/AIGCWaitingPublishContentService.java

@@ -47,10 +47,10 @@ public class AIGCWaitingPublishContentService {
     private static final String aigcGetAllContentUrl = "http://aigc-api.cybertogether.net/aigc/publish/content/gzhWaitingPublishContent";
 
 
-    public List<Content> getAllContent(String planId, String accountId) {
+    public List<Content> getAllContent(String planId, String accountId, List<String> sourceIds) {
         int retryTimes = 2;
         while (retryTimes > 0) {
-            List<Content> result = post(planId, accountId);
+            List<Content> result = post(planId, accountId, sourceIds);
             if (Objects.nonNull(result)) {
                 return result;
             }
@@ -59,12 +59,13 @@ public class AIGCWaitingPublishContentService {
         return new ArrayList<>();
     }
 
-    public List<Content> post(String planId, String accountId) {
+    public List<Content> post(String planId, String accountId, List<String> sourceIds) {
         List<Content> result = null;
         JSONObject bodyParam = new JSONObject();
         JSONObject bodyParamParams = new JSONObject();
         bodyParamParams.put("accountId", accountId);
         bodyParamParams.put("planId", planId);
+        bodyParamParams.put("sourceIds", sourceIds);
         bodyParam.put("params", bodyParamParams);
         try {
             MediaType mediaType = MediaType.parse("application/json");
@@ -92,7 +93,7 @@ public class AIGCWaitingPublishContentService {
     public List<Content> getAllContentByDB(String planId, String accountId) {
         List<PublishContentGzhWaiting> list = publishContentGzhWaitingRepository.getByPlanIdAndPublishAccountIdAndStatus(planId, accountId, 1);
         if (CollectionUtil.isEmpty(list)) {
-            List<Content> contentList = getAllContent(planId, accountId);
+            List<Content> contentList = getAllContent(planId, accountId, null);
             if (CollectionUtil.isNotEmpty(contentList)) {
                 Long now = System.currentTimeMillis();
                 CompletableFuture.runAsync(() -> xxlJobService.savePublishContentCache(contentList, planId, accountId, now));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/XxlJobService.java

@@ -13,6 +13,7 @@ import com.tzld.longarticle.recommend.server.common.enums.FieshuTableColumnDataT
 import com.tzld.longarticle.recommend.server.common.enums.aigc.ChannelEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.MiniprogramUseTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.ProduceContentAuditStatusEnum;
+import com.tzld.longarticle.recommend.server.common.enums.longArticle.ArticleVideoAuditStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.AccountBusinessTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ContentPoolEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.FeishuRobotIdEnum;
@@ -128,6 +129,8 @@ public class XxlJobService {
     private AIGCWaitingPublishContentService aigcWaitingPublishContentService;
     @Autowired
     private PublishContentGzhWaitingRepository publishContentGzhWaitingRepository;
+    @Autowired
+    private LongArticleTitleAuditRepository titleAuditRepository;
 
     ExecutorService thread = new CommonThreadPoolExecutor(
             5,
@@ -988,6 +991,15 @@ public class XxlJobService {
         List<PublishPlanAccountDTO> accountList = publishContentMapper.getPublishPlanAccounts(planIds);
         Collections.shuffle(accountList);
         CountDownLatch cdl = new CountDownLatch(accountList.size());
+        // 获取最近3天审核通过内容
+        List<LongArticleTitleAudit> newAuditList = titleAuditRepository.getByStatusAndAuditTimestampBetween(
+                ArticleVideoAuditStatusEnum.PASS.getCode(), now - (86400000 * 3), now);
+        List<String> newAuditIds = newAuditList.stream().map(LongArticleTitleAudit::getContentId).collect(Collectors.toList());
+        // 获取已存在数
+        List<PublishPlanAccountDTO> existAccountList = publishContentMapper.getGroupPublishPlanAccounts();
+        Map<String, Map<String, Integer>> existAccountMap = existAccountList.stream().collect(
+                Collectors.groupingBy(PublishPlanAccountDTO::getPlanId,
+                        Collectors.toMap(PublishPlanAccountDTO::getAccountId, PublishPlanAccountDTO::getCnt)));
 
         ExecutorService thread = new CommonThreadPoolExecutor(
                 syncPublishContentThreadPoolSize, syncPublishContentThreadPoolSize, 0L, TimeUnit.SECONDS,
@@ -997,7 +1009,13 @@ public class XxlJobService {
         for (PublishPlanAccountDTO account : accountList) {
             thread.submit(() -> {
                 try {
-                    List<Content> contentList = aigcWaitingPublishContentService.getAllContent(account.getPlanId(), account.getAccountId());
+                    // 首次同步所有待发布内容，后续增量同步最近三天审核通过内容
+                    Integer cnt = 0;
+                    if (existAccountMap.containsKey(account.getPlanId())) {
+                        cnt = existAccountMap.get(account.getPlanId()).getOrDefault(account.getAccountId(), 0);
+                    }
+                    List<Content> contentList = aigcWaitingPublishContentService.getAllContent(account.getPlanId(),account.getAccountId()
+                            , cnt > 0 ? newAuditIds : null);
                     savePublishContentCache(contentList, account.getPlanId(), account.getAccountId(), now);
                 } finally {
                     cdl.countDown();
@@ -1037,7 +1055,7 @@ public class XxlJobService {
         }
         List<String> contentIds = contentList.stream().map(Content::getId).collect(Collectors.toList());
         // 不存在状态置0
-        longArticleBaseMapper.updatePublishContentGzhWaitingStatus(planId, accountId, contentIds, 0, now);
+//        longArticleBaseMapper.updatePublishContentGzhWaitingStatus(planId, accountId, contentIds, 0, now);
         List<PublishContentGzhWaiting> existList = publishContentGzhWaitingRepository.getByIdIn(contentIds);
         List<String> existContentIds = existList.stream().map(PublishContentGzhWaiting::getId).collect(Collectors.toList());
         Map<String, PublishContentGzhWaiting> existMap = existList.stream().collect(

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/RecommendService.java

@@ -8,6 +8,7 @@ import com.tzld.longarticle.recommend.server.common.enums.aigc.PushTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.mapper.crawler.ext.ArticleUserGroupMapperExt;
+import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.dto.UserGroupCountDTO;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishPlan;
@@ -43,6 +44,7 @@ import org.springframework.util.StringUtils;
 
 import java.time.LocalDate;
 import java.util.*;
+import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 
 import static com.tzld.longarticle.recommend.server.common.constant.SceneConstants.FWH_COLD_START;
@@ -70,6 +72,8 @@ public class RecommendService {
     private ArticleRepository articleRepository;
     @Autowired
     private PublishPlanRepository publishPlanRepository;
+    @Autowired
+    private LongArticleBaseMapper longArticleBaseMapper;
 
     @ApolloJsonValue("${accountStrategyConfig:{}}")
     private Map<String, String> accountStrategyConfigMap;
@@ -236,6 +240,9 @@ public class RecommendService {
         data.setRank_list(rankList.subList(0, Math.min(rankList.size(), publishNum)));
         data.setFilter_list(filterContentList);
 
+        // 异步保存过滤内容状态
+        CompletableFuture.runAsync(() -> updateWaitingContentFilter(filterContentList));
+
         RecommendResponse response = new RecommendResponse();
         response.setCode(0);
         response.setMsg("success");
@@ -243,6 +250,14 @@ public class RecommendService {
         return response;
     }
 
+    private void updateWaitingContentFilter(List<ArticleSortResponseDataItem> filterContentList) {
+        if (CollectionUtils.isEmpty(filterContentList)) {
+            return;
+        }
+        List<String> contentIds = filterContentList.stream().map(ArticleSortResponseDataItem::getId).collect(Collectors.toList());
+        longArticleBaseMapper.updatePublishContentGzhWaitingStatusByIds(contentIds, 0);
+    }
+
     public RecommendParam genRecommendParam(RecommendRequest request, String abCode) {
         RecommendParam param = new RecommendParam();
         BeanUtils.copyProperties(request, param);

long-article-recommend-service/src/main/resources/mapper/aigc/PublishContentMapper.xml

@@ -323,7 +323,13 @@
         <foreach collection="accountIds" item="item" open="(" close=")" separator=",">
             #{item}
         </foreach>
+    </select>
 
+    <select id="getGroupPublishPlanAccounts"
+            resultType="com.tzld.longarticle.recommend.server.model.dto.aigc.PublishPlanAccountDTO">
+        select plan_id, publish_account_id, count(1) as cnt
+        from publish_content_gzh_waiting
+        GROUP BY plan_id, publish_account_id
     </select>
 
 </mapper>

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -448,4 +448,14 @@
         </foreach>
     </update>
 
+    <update id="updatePublishContentGzhWaitingStatusByIds">
+        update publish_content_gzh_waiting
+        set status = #{status},
+            update_timestamp = #{updateTimestamp}
+        where id in
+        <foreach collection="contentIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+    </update>
+
 </mapper>

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/FilterStrategyTest.java

@@ -62,7 +62,7 @@ public class FilterStrategyTest {
         param.setAccountName("票圈最新消息");
         param.setAccountId("20231212144543068928298");
         param.setPlanId("20240724092832149199870");
-        List<Content> content = aigcWaitingPublishContentService.getAllContent(param.getPlanId(), param.getAccountId());
+        List<Content> content = aigcWaitingPublishContentService.getAllContent(param.getPlanId(), param.getAccountId(), null);
         FilterParam filterParam = new FilterParam();
         filterParam.setAccountName(param.getAccountName());
         filterParam.setContents(content);