Merge branch 'wyp/1024-export' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -10,6 +10,8 @@ public interface LongArticleBaseMapper {
 
     void deleteByDateStrGreaterThanEqual(String dateStr);
 
+    void deleteByDateStrBetween(String dateStrBegin, String dateStrEnd);
+
     void batchInsertDatastatSortStrategy(List<DatastatSortStrategy> list);
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/ArticleRepository.java

@@ -18,6 +18,8 @@ public interface ArticleRepository extends JpaRepository<Article, String> {
 
     List<Article> getByGhIdInAndAppMsgIdInAndItemIndexAndTypeEqualsAndStatusEquals(Set<String> ghIds, Set<String> appMsgIds, Integer itemIndex, String type, Integer status);
 
+    List<Article> getByGhIdInAndUpdateTimeBetweenAndTypeEquals(List<String> ghIds, Long updateTimeStart, Long updateTimeEnd, String type);
+
     List<Article> getByGhIdInAndUpdateTimeGreaterThanAndTypeEquals(List<String> ghIds, Long updateTime, String type);
 
     List<Article> getByGhIdInAndUpdateTimeLessThanAndTypeEquals(List<String> ghIds, Long updateTIme, String type);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/PublishSortLogRepository.java

@@ -17,6 +17,8 @@ public interface PublishSortLogRepository extends JpaRepository<PublishSortLog,
 
     List<PublishSortLog> findByCrawlerChannelContentIdIn(List<String> crawlerChannelContentIds);
 
+    List<PublishSortLog> findByGhIdInAndDateStrBetween(List<String> ghIds, String dateStrStart, String dateStrEnd);
+
     List<PublishSortLog> findByGhIdInAndDateStrGreaterThanEqual(List<String> ghIds, String dateStr);
 
     List<PublishSortLog> findByStrategyInAndDateStrGreaterThanEqual(List<String> strategies, String dateStr);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/DataDashboardService.java

@@ -80,14 +80,14 @@ public class DataDashboardService {
     private static List<String> ghIdList;
     private static final String sheetToken = "M0pLs3uF6hfL0htn2dMcB9eFn8e";
 
-    public void export(String dateStr) {
-        List<String> dateStrList = DateUtils.getBeforeDays(dateStr, 5);
+    public void export(String beginDate, String endDate) {
+        List<String> dateStrList = DateUtils.getBeforeDays(beginDate, endDate, 5);
         exportFeishuNewSortStrategy(dateStrList, sheetToken, "7d4e12");
     }
 
     @XxlJob("scheduledExport")
     public ReturnT<String> scheduledExport(String param) {
-        List<String> dateStrList = DateUtils.getBeforeDays(null, 5);
+        List<String> dateStrList = DateUtils.getBeforeDays(null, null, 5);
         exportFeishuNewSortStrategy(dateStrList, sheetToken, "7d4e12");
         return ReturnT.SUCCESS;
     }
@@ -95,7 +95,8 @@ public class DataDashboardService {
 
     private void exportFeishuNewSortStrategy(List<String> dateStrList, String sheetToken, String sheetId) {
         String minDate = dateStrList.stream().min(String::compareTo).orElse("");
-        List<NewSortStrategyExport> newContentsYesData = newSortStrategyData(minDate);
+        String maxDate = dateStrList.stream().max(String::compareTo).orElse("");
+        List<NewSortStrategyExport> newContentsYesData = newSortStrategyData(minDate, maxDate);
         if (CollectionUtil.isEmpty(newContentsYesData)) {
             return;
         }
@@ -138,24 +139,28 @@ public class DataDashboardService {
         doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles);
     }
 
-    private List<NewSortStrategyExport> newSortStrategyData(String dateStr) {
-        long timestamp = DateUtils.dateStrToTimestamp(dateStr, "yyyyMMdd");
-        String dateStrS = DateUtils.timestampToYMDStr(timestamp, "yyyy-MM-dd");
+    private List<NewSortStrategyExport> newSortStrategyData(String beginDate, String endDate) {
+        long beginTimestamp = DateUtils.dateStrToTimestamp(beginDate, "yyyyMMdd");
+        long endTimestamp = DateUtils.dateStrToTimestamp(endDate, "yyyyMMdd") + 86400;
         List<AccountAvgInfo> accountAvgInfoList = accountAvgInfoRepository.findAll();
+        log.info("newSortStrategyData accountAvgInfoList finish");
         List<String> ghIds = accountAvgInfoList.stream().map(AccountAvgInfo::getGhId).distinct().collect(Collectors.toList());
 
-        List<Article> articleList = articleRepository.getByGhIdInAndUpdateTimeGreaterThanAndTypeEquals(ghIds, timestamp, ArticleTypeEnum.QUNFA.getVal());
+        List<Article> articleList = articleRepository.getByGhIdInAndUpdateTimeBetweenAndTypeEquals(ghIds,
+                beginTimestamp, endTimestamp, ArticleTypeEnum.QUNFA.getVal());
         Map<String, Map<String, Map<Integer, Article>>> articleMap = articleList.stream().collect(Collectors.groupingBy(Article::getGhId,
                 Collectors.groupingBy(Article::getAppMsgId, Collectors.toMap(Article::getItemIndex, o -> o))));
-
+        log.info("newSortStrategyData articleList finish");
         Set<String> snList = articleList.stream().map(Article::getWxSn).collect(Collectors.toSet());
         List<ArticleDetailInfo> articleDetailInfoList = new ArrayList<>();
         for (List<String> partitions : Lists.partition(new ArrayList<>(snList), 1000)) {
             articleDetailInfoList.addAll(articleDetailInfoRepository.getAllByWxSnIn(partitions));
         }
+        log.info("newSortStrategyData articleDetailInfoList finish");
         Map<String, List<ArticleDetailInfo>> articleDetailInfoMap = articleDetailInfoList.stream()
                 .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
-        List<PublishSortLog> sortLogList = publishSortLogRepository.findByGhIdInAndDateStrGreaterThanEqual(ghIds, dateStr);
+        List<PublishSortLog> sortLogList = publishSortLogRepository.findByGhIdInAndDateStrBetween(ghIds, beginDate, endDate);
+        log.info("newSortStrategyData sortLogList finish");
         Map<String, Map<String, String>> sortStrategyMap = sortLogList.stream()
                 .collect(Collectors.groupingBy(PublishSortLog::getGhId,
                         Collectors.toMap(PublishSortLog::getDateStr, PublishSortLog::getStrategy, (existing, replacement) -> replacement)));
@@ -165,6 +170,7 @@ public class DataDashboardService {
                         Collectors.toMap(AccountAvgInfo::getPosition, o -> o))));
         // 获取发布账号
         List<PublishAccount> publishAccountList = publishAccountRepository.getAllByGhIdIn(ghIds);
+        log.info("newSortStrategyData publishAccountList finish");
         Map<String, PublishAccount> publishAccountMap = publishAccountList.stream().collect(Collectors.toMap(PublishAccount::getGhId, o -> o));
         // 获取发布内容
         List<PublishContentParam> publishContentParamList = articleList.stream().map(article -> {
@@ -178,9 +184,10 @@ public class DataDashboardService {
             return null;
         }).filter(Objects::nonNull).collect(Collectors.toList());
         List<PublishContent> publishContents = new ArrayList<>();
-        for (List<PublishContentParam> partitions : Lists.partition(publishContentParamList, 1000)) {
+        for (List<PublishContentParam> partitions : Lists.partition(publishContentParamList, 100)) {
             publishContents.addAll(aigcBaseMapper.getPublishContentByTitle(partitions));
         }
+        log.info("newSortStrategyData publishContents finish");
         Map<String, Map<String, Map<Long, PublishContent>>> publishContentMap = publishContents.stream()
                 .sorted(Comparator.comparingLong(PublishContent::getPublishTimestamp)).collect(
                         Collectors.groupingBy(PublishContent::getPublishAccountId,
@@ -193,6 +200,7 @@ public class DataDashboardService {
         for (List<String> partitions : Lists.partition(publishContentIds, 1000)) {
             publishContentLayoutList.addAll(publishContentLayOutRepository.findByPublishContentIdIn(partitions));
         }
+        log.info("newSortStrategyData publishContentLayoutList finish");
         Map<String, PublishContentLayout> publishContentLayoutMap = publishContentLayoutList.stream()
                 .collect(Collectors.toMap(PublishContentLayout::getPublishContentId, o -> o,
                         (existing, replacement) -> replacement));
@@ -200,6 +208,7 @@ public class DataDashboardService {
         List<String> publishPlanIds = publishContents.stream().map(PublishContent::getPlanId).distinct()
                 .collect(Collectors.toList());
         List<PublishPlan> publishPlanList = publishPlanRepository.findByIdIn(publishPlanIds);
+        log.info("newSortStrategyData publishPlanList finish");
         Map<String, PublishPlan> publishPlanMap = publishPlanList.stream()
                 .collect(Collectors.toMap(PublishPlan::getId, o -> o));
         // 获取生成记录
@@ -210,28 +219,33 @@ public class DataDashboardService {
         for (List<String> partitions : Lists.partition(contentSourceIds, 1000)) {
             planExeRecordList.addAll(producePlanExeRecordRepository.findByPlanExeIdIn(partitions));
         }
+        log.info("newSortStrategyData planExeRecordList finish");
         Map<String, ProducePlanExeRecord> planExeRecordMap = planExeRecordList.stream()
                 .collect(Collectors.toMap(ProducePlanExeRecord::getPlanExeId, o -> o));
         // 获取生成计划
         List<String> producePlanIds = planExeRecordList.stream().map(ProducePlanExeRecord::getPlanId).distinct()
                 .collect(Collectors.toList());
         List<ProducePlan> producePlanList = producePlanRepository.findByIdIn(producePlanIds);
+        log.info("newSortStrategyData producePlanList finish");
         Map<String, ProducePlan> producePlanMap = producePlanList.stream()
                 .collect(Collectors.toMap(ProducePlan::getId, o -> o));
         // 获取生成计划输入
         List<ProducePlanInputSource> inputSourceList = producePlanInputSourceRepository.findByPlanIdIn(producePlanIds);
+        log.info("newSortStrategyData inputSourceList finish");
         Map<String, List<ProducePlanInputSource>> inputSourceMap = inputSourceList.stream()
                 .collect(Collectors.groupingBy(ProducePlanInputSource::getPlanId));
         // 获取抓取内容关联
         List<String> crawlerChannelContentIds = publishContents.stream().map(PublishContent::getCrawlerChannelContentId)
                 .distinct().collect(Collectors.toList());
         List<CrawlerPlanResultRel> resultRelList = aigcBaseMapper.getCrawlerPlanRelByChannelContentIds(crawlerChannelContentIds);
+        log.info("newSortStrategyData resultRelList finish");
         Map<String, List<CrawlerPlanResultRel>> resultRelMap = resultRelList.stream()
                 .collect(Collectors.groupingBy(CrawlerPlanResultRel::getChannelSourceId));
         // 获取抓取计划
         List<String> crawlerPlanIds = resultRelList.stream().map(CrawlerPlanResultRel::getPlanId).distinct()
                 .collect(Collectors.toList());
         List<CrawlerPlan> crawlerPlanList = aigcBaseMapper.getCrawlerPlanByPlanIds(crawlerPlanIds);
+        log.info("newSortStrategyData crawlerPlanList finish");
         Map<String, CrawlerPlan> crawlerPlanMap = crawlerPlanList.stream()
                 .collect(Collectors.toMap(CrawlerPlan::getId, o -> o));
         // 获取小程序任务
@@ -253,11 +267,13 @@ public class DataDashboardService {
         for (List<MiniprogramTaskParam> partitions : Lists.partition(miniprogramTaskParamList, 1000)) {
             miniprogramTaskList.addAll(aigcBaseMapper.getMiniProgramTask(partitions));
         }
+        log.info("newSortStrategyData miniprogramTaskList finish");
         Map<String, List<PublishPlanMiniprogramTask>> miniprogramTaskMap = miniprogramTaskList.stream()
                 .collect(Collectors.groupingBy(PublishPlanMiniprogramTask::getPlanId));
         // 源生成计划
         List<String> titleList = articleList.stream().map(Article::getTitle).distinct().collect(Collectors.toList());
         Map<String, ProducePlan> sourceTitlePlanMap = getTitleSourceProducePlanMap(titleList);
+        log.info("newSortStrategyData sourceTitlePlan finish");
         // 历史发布情况
         List<String> titleMd5List = articleList.stream().map(Article::getTitleMd5).distinct().collect(Collectors.toList());
         List<Article> hisArticleList = new ArrayList<>();
@@ -265,6 +281,7 @@ public class DataDashboardService {
         for (List<String> titleMd5s : titleMd5Partition) {
             hisArticleList.addAll(articleRepository.getByTitleMd5InAndTypeEqualsAndStatusEquals(titleMd5s, ArticleTypeEnum.QUNFA.getVal(), 1));
         }
+        log.info("newSortStrategyData hisArticleList finish");
         Map<String, List<Article>> hisArticleMap = hisArticleList.stream().collect(Collectors.groupingBy(Article::getTitle));
         Set<String> hisWxSnList = hisArticleList.stream().map(Article::getWxSn).collect(Collectors.toSet());
         List<ArticleDetailInfo> hisArticleDetailInfoList = new ArrayList<>();
@@ -272,6 +289,7 @@ public class DataDashboardService {
         for (List<String> sns : hisSnPartition) {
             hisArticleDetailInfoList.addAll(articleDetailInfoRepository.getAllByWxSnIn(sns));
         }
+        log.info("newSortStrategyData hisArticleDetailInfoList finish");
         Map<String, List<ArticleDetailInfo>> hisArticleDetailInfoMap = hisArticleDetailInfoList.stream()
                 .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
 
@@ -377,10 +395,11 @@ public class DataDashboardService {
                 obj.setSourceProducePlanName(sourceProducePlan.getName());
             }
         }
+        log.info("newSortStrategyData buildData finish");
         result.sort(Comparator.comparing(NewSortStrategyExport::getDateStr).reversed()
                 .thenComparing(NewSortStrategyExport::getGhId).thenComparing(NewSortStrategyExport::getPosition));
         if (CollectionUtils.isNotEmpty(result)) {
-            longArticleBaseMapper.deleteByDateStrGreaterThanEqual(dateStr);
+            longArticleBaseMapper.deleteByDateStrBetween(beginDate, endDate);
             List<DatastatSortStrategy> saveList = new ArrayList<>();
             for (NewSortStrategyExport newSortStrategyExport : result) {
                 DatastatSortStrategy item = new DatastatSortStrategy();
@@ -391,6 +410,7 @@ public class DataDashboardService {
                 longArticleBaseMapper.batchInsertDatastatSortStrategy(saveListPartition);
             }
         }
+        log.info("newSortStrategyData finish");
         return result;
     }
 
@@ -697,15 +717,15 @@ public class DataDashboardService {
 
     @XxlJob("scheduleExportIntermediateIndicators")
     public ReturnT<String> scheduleIntermediateIndicatorsExport(String param) {
-        List<String> dateStrList = DateUtils.getBeforeDays(null, 3);
+        List<String> dateStrList = DateUtils.getBeforeDays(null, null, 3);
         for (String date : dateStrList) {
             exportFeishuIntermediateIndicators(date, dateStrList, sheetToken, "OuaLWV");
         }
         return ReturnT.SUCCESS;
     }
 
-    public void intermediateIndicatorsExport(String dateStr) {
-        List<String> dateStrList = DateUtils.getBeforeDays(dateStr, 3);
+    public void intermediateIndicatorsExport(String beginDate, String endDate) {
+        List<String> dateStrList = DateUtils.getBeforeDays(beginDate, endDate, 3);
         for (String date : dateStrList) {
             exportFeishuIntermediateIndicators(date, dateStrList, sheetToken, "OuaLWV");
         }
@@ -1265,13 +1285,13 @@ public class DataDashboardService {
 
     @XxlJob("scheduleExportFirstContentScore")
     public ReturnT<String> scheduleExportFirstContentScore(String param) {
-        List<String> dateStrList = DateUtils.getBeforeDays(null, 1);
+        List<String> dateStrList = DateUtils.getBeforeDays(null, null, 1);
         exportFeishuFirstContentScore(dateStrList, sheetToken, "XBFd16");
         return ReturnT.SUCCESS;
     }
 
-    public void firstContentScoreExport(String dateStr) {
-        List<String> dateStrList = DateUtils.getBeforeDays(dateStr, 1);
+    public void firstContentScoreExport(String beginDate, String endDate) {
+        List<String> dateStrList = DateUtils.getBeforeDays(beginDate, endDate, 1);
         exportFeishuFirstContentScore(dateStrList, sheetToken, "XBFd16");
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recall/RecallService.java

@@ -145,8 +145,8 @@ public class RecallService implements ApplicationContextAware {
         if (CollectionUtils.isEmpty(content)) {
             FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
                     "内容召回失败\n"
-                    + "ghId: " + param.getGhId() + "\n"
-                    + "账号名称: " + param.getAccountName());
+                            + "ghId: " + param.getGhId() + "\n"
+                            + "账号名称: " + param.getAccountName());
             return content;
         }
         // 标题历史均值
@@ -303,10 +303,10 @@ public class RecallService implements ApplicationContextAware {
                     } else {
                         FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
                                 "历史表现阅读均值获取失败\n"
-                                + "ghId: " + hisArticle.getGhId() + "\n"
-                                + "账号名称: " + hisArticle.getAccountName() + "\n"
-                                + "日期: " + hisPublishDate + "\n"
-                                + "位置: " + hisArticle.getItemIndex());
+                                        + "ghId: " + hisArticle.getGhId() + "\n"
+                                        + "账号名称: " + hisArticle.getAccountName() + "\n"
+                                        + "日期: " + hisPublishDate + "\n"
+                                        + "位置: " + hisArticle.getItemIndex());
                     }
                 }
                 article.setAvgViewCount(avgViewCount);
@@ -362,12 +362,12 @@ public class RecallService implements ApplicationContextAware {
                 if (article.getUpdateTime() > 1720713600 && contentHisFeishuEnable) {
                     FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
                             "历史表现裂变特征获取失败\n"
-                            + "ghId: " + article.getGhId() + "\n"
-                            + "账号名称: " + article.getAccountName() + "\n"
-                            + "位置: " + article.getItemIndex() + "\n"
-                            + "标题: " + article.getTitle() + "\n"
-                            + "发布时间: " + DateUtils.timestampToYMDStr(article.getUpdateTime(), "yyyyMMdd") + "\n"
-                            + "wxsn: " + article.getWxSn());
+                                    + "ghId: " + article.getGhId() + "\n"
+                                    + "账号名称: " + article.getAccountName() + "\n"
+                                    + "位置: " + article.getItemIndex() + "\n"
+                                    + "标题: " + article.getTitle() + "\n"
+                                    + "发布时间: " + DateUtils.timestampToYMDStr(article.getUpdateTime(), "yyyyMMdd") + "\n"
+                                    + "wxsn: " + article.getWxSn());
                 }
                 continue;
             }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/util/DateUtils.java

@@ -114,21 +114,25 @@ public final class DateUtils {
         return nearestDateStr;  // 返回最近的日期字符串
     }
 
-    public static List<String> getBeforeDays(String dateStr, int days) {
+    public static List<String> getBeforeDays(String beginDate, String endDate, int days) {
         DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
         // 获取今天的日期
         LocalDate today = LocalDate.now();
-        if (!StringUtils.hasText(dateStr)) {
-            dateStr = today.format(formatter);
+        if (!StringUtils.hasText(beginDate)) {
+            beginDate = today.format(formatter);
+        }
+        if (!StringUtils.hasText(endDate)) {
+            endDate = today.format(formatter);
         }
         // 解析输入的日期字符串为LocalDate对象
-        LocalDate date = LocalDate.parse(dateStr, formatter);
+        LocalDate date = LocalDate.parse(beginDate, formatter);
+        LocalDate end = LocalDate.parse(endDate, formatter);
         // 获取从输入日期前三天的日期
         LocalDate startDate = date.minusDays(days);
         // 存储所有日期的列表
         List<String> datesList = new ArrayList<>();
         // 从startDate到today遍历日期
-        while (!startDate.isAfter(today)) {
+        while (!startDate.isAfter(today) && !startDate.isAfter(end)) {
             // 将当前日期格式化为"yyyyMMdd"并添加到列表中
             datesList.add(startDate.format(formatter));
             // 日期加1天

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/DataDashboardController.java

@@ -16,23 +16,23 @@ public class DataDashboardController {
     private DataDashboardService service;
 
     @GetMapping("/export/newSortStrategy")
-    public void newSortStrategyExport(String dateStr) {
+    public void newSortStrategyExport(String beginDate, String endDate) {
         new Thread(() -> {
-            service.export(dateStr);
+            service.export(beginDate, endDate);
         }).start();
     }
 
     @GetMapping("/export/intermediateIndicators")
-    public void intermediateIndicatorsExport(String dateStr) {
+    public void intermediateIndicatorsExport(String beginDate, String endDate) {
         new Thread(() -> {
-            service.intermediateIndicatorsExport(dateStr);
+            service.intermediateIndicatorsExport(beginDate, endDate);
         }).start();
     }
 
     @GetMapping("/export/firstContentScore")
-    public void firstContentScoreExport(String dateStr) {
+    public void firstContentScoreExport(String beginDate, String endDate) {
         new Thread(() -> {
-            service.firstContentScoreExport(dateStr);
+            service.firstContentScoreExport(beginDate, endDate);
         }).start();
     }
 

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -6,6 +6,10 @@
         delete from datastat_sort_strategy where date_str >= #{dateStr}
     </delete>
 
+    <delete id="deleteByDateStrBetween">
+        delete from datastat_sort_strategy where date_str between #{dateStrBegin} and #{dateStrEnd}
+    </delete>
+
     <insert id="batchInsertDatastatSortStrategy">
         INSERT INTO datastat_sort_strategy
         (date_str, publish_time, account_mode, account_source, account_type, account_status, bussiness_type,