Merge branch 'master' into 20250507-luojunhui-rankV18

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/PublishDailyReport.java

@@ -0,0 +1,105 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.*;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "publish_daily_report")
+public class PublishDailyReport {
+
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @Column(name = "date_str")
+    private String dateStr;
+
+    @Column(name = "publish_time")
+    private String publishTime;
+
+    @Column(name = "account_source")
+    private String accountSource;
+
+    @Column(name = "account_status")
+    private String accountStatus;
+
+    @Column(name = "business_type")
+    private String businessType;
+
+    @Column(name = "account_name")
+    private String accountName;
+
+    @Column(name = "strategy")
+    private String strategy;
+
+    @Column(name = "fans")
+    private Integer fans;
+
+    @Column(name = "avg_view_count")
+    private Double avgViewCount;
+
+    @Column(name = "read_rate")
+    private Double readRate = 0.0;
+
+    @Column(name = "read_fans_rate")
+    private Double readFansRate = 0.0;
+
+    @Column(name = "first_read_rate")
+    private Double firstReadRate = 0.0;
+
+    @Column(name = "fission0_first_rate")
+    private Double fission0FirstRate = 0.0;
+
+    @Column(name = "fission0_head_first_rate")
+    private Double fission0HeadFirstRate = 0.0;
+
+    @Column(name = "fission0_recommend_first_rate")
+    private Double fission0RecommendFirstRate = 0.0;
+
+    @Column(name = "fission1_fission0_rate")
+    private Double fission1Fission0Rate = 0.0;
+
+    @Column(name = "fission0_read_avg_rate")
+    private Double fission0ReadAvgRate = 0.0;
+
+    @Column(name = "his_read_rate")
+    private Double hisReadRate = 0.0;
+
+    @Column(name = "his_first_read_rate")
+    private Double hisFirstReadRate = 0.0;
+
+    @Column(name = "his_fission0_first_rate")
+    private Double hisFission0FirstRate = 0.0;
+
+    @Column(name = "position")
+    private Integer position;
+
+    @Column(name = "gh_id")
+    private String ghId;
+
+    @Column(name = "title")
+    private String title;
+
+    @Column(name = "link")
+    private String link;
+
+    @Column(name = "crawler_plan_name")
+    private String crawlerPlanName;
+
+    @Column(name = "produce_plan_name")
+    private String producePlanName;
+
+    @Column(name = "publish_plan_name")
+    private String publishPlanName;
+
+    @Column(name = "source_produce_plan_name")
+    private String sourceProducePlanName;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/PublishDailyReportRepository.java

@@ -0,0 +1,10 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.PublishDailyReport;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface PublishDailyReportRepository extends JpaRepository<PublishDailyReport, Long> {
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticlePromotionService.java

@@ -95,6 +95,8 @@ public class ArticlePromotionService {
     private Map<String, Map<String, Map<String, String>>> produceConfig;
     @Value("${topProducePlanId:}")
     private String topProducePlanId;
+    @Value("${readOpenFissionRateThreshold:}")
+    private Double readOpenFissionRateThreshold;
 
     private final List<String> contentPoolType = Arrays.asList("autoArticlePoolLevel1", "autoArticlePoolLevel3", "autoArticlePoolLevel4");
 
@@ -132,22 +134,57 @@ public class ArticlePromotionService {
         return ciLow > 0;
     }
 
+    private Double nullSafe(Double value) {
+        return value != null ? value : 0.0;
+    }
+
+    private Integer nullSafe(Integer value) {
+        return value != null ? value : 0;
+    }
+
+    public List<DatastatSortStrategy> promotionWithReadOpenFissionRate(double threshold, List<DatastatSortStrategy> list) {
+        if (CollectionUtils.isEmpty(list)) {
+            throw new IllegalArgumentException("List cannot be empty");
+        }
+
+        return list.stream().filter(o -> {
+            double denominator =
+                    nullSafe(o.getFirstLevel()) +
+                            nullSafe(o.getSecondFirstLevel()) +
+                            nullSafe(o.getThirdFirstLevel());
+            if (denominator == 0) return false;
+
+            double numerator =
+                    nullSafe(o.getFission0()) + nullSafe(o.getFission1()) + nullSafe(o.getFission2()) +
+                            nullSafe(o.getSecondFission0()) + nullSafe(o.getSecondFission1()) + nullSafe(o.getSecondFission2()) +
+                            nullSafe(o.getThirdFission0()) + nullSafe(o.getThirdFission1()) + nullSafe(o.getThirdFission2());
+
+            double readRate = nullSafe(o.getReadRate());
+            double firstReadRate = nullSafe(o.getFirstReadRate());
+            // 计算 log10（阅读量)
+            double logViewCount = (nullSafe(o.getViewCount()) > 0) ? Math.log10(nullSafe(o.getViewCount())) : 0;
+            // 计算完整公式  score = 阅读均值倍数 * 阅读率 * T2裂变率 * log10（阅读量）
+            double result = readRate * firstReadRate * (numerator / denominator) * logViewCount;
+            return result > threshold;
+        }).collect(Collectors.toList());
+    }
+
     public void articlePromotion(String pos, String way, String accountNickName, String tag,
                                  Integer viewCountFilter, Double viewCountRateFilter, List<Integer> positionFilter) {
         String today = DateUtils.getCurrentDateStr("yyyyMMdd");
         String dateStrFilter = DateUtils.getBeforeDaysDateStr("yyyyMMdd", 10);
         // 获取内部表现
         List<DatastatSortStrategy> list;
+        list = longArticleBaseMapper.getArticlePromotionCandidates(10000, dateStrFilter, positionFilter);
+        // 使用阅读均值倍数+阅读量晋级
+        List<DatastatSortStrategy> listStrategyV1 = list.stream()
+                .filter(o -> o.getReadRate() >= viewCountRateFilter && o.getViewCount() >= viewCountFilter)
+                .collect(Collectors.toList());
+        List<String> StrategyV1WxSn = listStrategyV1.stream().map(DatastatSortStrategy::getWxSn).collect(Collectors.toList());
+        List<DatastatSortStrategy> listStrategyV2;
         if (pos.equals("【2】")) {
-            list = longArticleBaseMapper.getArticlePromotionCandidates(10000, dateStrFilter, positionFilter);
-
-            // 使用阅读均值倍数+阅读量晋级
-            List<DatastatSortStrategy> listStrategy1 = list.stream()
-                    .filter(o -> o.getReadRate() >= 1.33 && o.getViewCount() >= 100)
-                    .collect(Collectors.toList());
-
             // 使用显著性检验晋级
-            List<DatastatSortStrategy> listStrategy2 = list.stream()
+            listStrategyV2 = list.stream()
                     .filter(o -> {
                         try {
                             return isExperimentGroupStatisticallySuperior(o.getAvgViewCount() * 1.1 * 30, o.getFans() * 30,
@@ -159,26 +196,26 @@ public class ArticlePromotionService {
                         }
                     })
                     .collect(Collectors.toList());
-
-            list = Stream.concat(listStrategy1.stream(), listStrategy2.stream())
-                    .collect(Collectors.collectingAndThen(
-                            Collectors.toMap(
-                                    DatastatSortStrategy::getWxSn,
-                                    Function.identity(),
-                                    (existing, replacement) -> existing
-                            ),
-                            map -> new ArrayList<>(map.values())
-                    ));
         } else {
-            list = longArticleBaseMapper.getArticlePromotion(viewCountFilter, viewCountRateFilter,
-                    10000, dateStrFilter, positionFilter);
+            // 使用新规则过滤
+            listStrategyV2 = promotionWithReadOpenFissionRate(readOpenFissionRateThreshold, list);
         }
+        listStrategyV2 = listStrategyV2.stream().filter(o -> !StrategyV1WxSn.contains(o.getWxSn())).collect(Collectors.toList());
+
+        //策略 1 晋级
+        filterAndAdd2CrawlerPlan(listStrategyV1, "策略V1", accountNickName, tag, pos, way, today);
+
+        //策略 2 晋级
+        filterAndAdd2CrawlerPlan(listStrategyV2, "策略V2", accountNickName, tag, pos, way, today);
+    }
+
+    private void filterAndAdd2CrawlerPlan(List<DatastatSortStrategy> list, String promotionStrategy, String accountNickName, String tag, String pos, String way, String today) {
         list = filterEarlyContent(list, true);
         log.info("优质{}文章数量: {}", accountNickName, list.size());
         List<DatastatSortStrategy> distinct = filterSameTitle(list);
         distinct.sort(Comparator.comparing(DatastatSortStrategy::getDateStr, Comparator.reverseOrder()));
         log.info("优质{}文章数量(去重后): {}", accountNickName, distinct.size());
-        addUrlListToAccount(accountNickName, distinct, pos, way, today, tag);
+        addUrlListToAccount(accountNickName, distinct, pos, way, today, tag, promotionStrategy);
     }
 
     private List<DatastatSortStrategy> filterEarlyContent(List<DatastatSortStrategy> list, Boolean filterVideoPool) {
@@ -217,7 +254,7 @@ public class ArticlePromotionService {
     }
 
     private void addUrlListToAccount(String accountNickName, List<DatastatSortStrategy> list, String pos, String way,
-                                     String today, String tag) {
+                                     String today, String tag, String promotionStrategy) {
         List<String> urlList = list.stream().map(DatastatSortStrategy::getLink).collect(Collectors.toList());
         if (!produceConfig.containsKey(accountNickName)) {
             log.info("account_nickname not in produceConfig: " + accountNickName);
@@ -264,7 +301,7 @@ public class ArticlePromotionService {
                 }
             }
             if (CollectionUtils.isNotEmpty(filterUrlList)) {
-                String planName = String.format("%d_%s_%s_%s【%s】_%s", filterUrlList.size(), today, accountNickName, pos, way, today);
+                String planName = String.format("%d_%s_%s_%s【%s】_%s_%s", filterUrlList.size(), today, accountNickName, pos, way, today, promotionStrategy);
                 log.info("url_len: " + list.size() + ", " + filterUrlList.size());
                 IdNameVO<String> planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, filterUrlList, tag, CrawlerModeEnum.ContentIDs.getVal());
                 if (StringUtils.hasText(produceId)) {
@@ -274,14 +311,14 @@ public class ArticlePromotionService {
                 log.info("{}, {}, produce plan not exist: {}, {}, {}", planInfo.getName(), planInfo.getId(), accountNickName, pos, way);
             }
             if (CollectionUtils.isNotEmpty(publishContentIds)) {
-                String planName = String.format("%d_%s_%s_%s【%s】_%s", publishContentIds.size(), today, accountNickName, pos, way, today);
+                String planName = String.format("%d_%s_%s_%s【%s】_%s_%s", publishContentIds.size(), today, accountNickName, pos, way, today, promotionStrategy);
                 IdNameVO<String> planInfo = aigcCrawlerPlanSaveService.createArticleUrlPlan(planName, publishContentIds, tag, CrawlerModeEnum.PublishContentIds.getVal());
                 if (StringUtils.hasText(produceId)) {
                     String inputSourceLabel = String.format("原始帖子-长文-微信公众号-内容添加计划-%s", planInfo.getName());
                     articleAddDependPlan(produceId, planInfo.getId(), inputSourceLabel, ProducePlanInputSourceTypeEnum.contentPlan.getVal());
                 }
             }
-            sendFeishuJobFinishMessage(accountNickName, filterUrlList.size(), publishContentIds.size());
+            sendFeishuJobFinishMessage(accountNickName, filterUrlList.size(), publishContentIds.size(), promotionStrategy);
         } catch (Exception e) {
             log.error("articlePromotion error: ", e);
             FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.DAILY.getRobotId(),
@@ -292,13 +329,14 @@ public class ArticlePromotionService {
         }
     }
 
-    private void sendFeishuJobFinishMessage(String accountNickName, Integer urlListSize, Integer contentListSize) {
+    private void sendFeishuJobFinishMessage(String accountNickName, Integer urlListSize, Integer contentListSize, String promotionStrategy) {
         log.info("articlePromotion finish: 晋级任务:{}, id晋级数量：{}, url晋级数量：{}", accountNickName, contentListSize, urlListSize);
         FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.DAILY.getRobotId(),
                 "【文章晋级job完成】\n" +
                         "晋级任务：" + accountNickName + "\n" +
                         "id晋级数量：" + contentListSize + "\n" +
-                        "url晋级数量：" + urlListSize + "\n");
+                        "url晋级数量：" + urlListSize + "\n" +
+                        "晋级策略: " + promotionStrategy + "\n");
     }
 
     private List<ProduceContentListItemVO> getProduceContentList(String accountNickName, String pos, String way) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/DataDashboardService.java

@@ -51,6 +51,7 @@ import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.util.Pair;
@@ -120,6 +121,8 @@ public class DataDashboardService {
     private LongArticlesPublishMiniprogramRepository longArticlesPublishMiniprogramRepository;
     @Autowired
     private LongArticlesNewVideoCoverRepository longArticlesNewVideoCoverRepository;
+    @Autowired
+    private PublishDailyReportRepository publishDailyReportRepository;
 
     @ApolloJsonValue("${export.account.ghId:[]}")
     private static List<String> ghIdList;
@@ -149,6 +152,28 @@ public class DataDashboardService {
         return ReturnT.SUCCESS;
     }
 
+    @XxlJob("scheduledExportMysql")
+    public ReturnT<String> scheduledExportMysql(String param) {
+        if (!StringUtils.hasText(param)) {
+            return ReturnT.FAIL;
+        }
+        JSONObject jsonObject = JSONObject.parseObject(param);
+        String minDate = jsonObject.getString("minDate");
+        String maxDate = jsonObject.getString("maxDate");
+        List<NewSortStrategyExport> newContentsYesData = newSortStrategyData(minDate, maxDate,
+                ArticleTypeEnum.QUNFA.getVal(), StatusEnum.ZERO.getCode());
+        if (CollectionUtils.isNotEmpty(newContentsYesData)) {
+            List<PublishDailyReport> reports = new ArrayList<>();
+            for (NewSortStrategyExport newSortStrategyExport : newContentsYesData) {
+                PublishDailyReport publishDailyReport = new PublishDailyReport();
+                BeanUtils.copyProperties(newSortStrategyExport, publishDailyReport);
+                reports.add(publishDailyReport);
+            }
+            publishDailyReportRepository.saveAll(reports);
+        }
+        return ReturnT.SUCCESS;
+    }
+
     public void exportWuXianLiu(String beginDate, String endDate) {
         List<String> dateStrList = DateUtils.getBeforeDays(beginDate, endDate, 5);
         exportFeishuNewSortStrategy(dateStrList, ArticleTypeEnum.WUXIANLIU.getVal(), StatusEnum.ONE.getCode(),
@@ -537,8 +562,8 @@ public class DataDashboardService {
                 ProducePlan producePlan = producePlanMap.get(record.getPlanId());
                 obj.setProducePlanName(producePlan.getName());
                 obj.setProducePlanTag(producePlan.getPlanTag());
+                producePlanInputSourceList = inputSourceMap.get(record.getPlanId());
             }
-            producePlanInputSourceList = inputSourceMap.get(record.getPlanId());
         }
         if (publishContent.getSourceType().equals(PublishPlanInputSourceTypesEnum.longArticleVideoPoolSource.getVal())) {
             PublishSingleVideoSource videoPoolSource = videoPoolSourceMap.get(publishContent.getSourceId());
@@ -554,18 +579,25 @@ public class DataDashboardService {
             }
         }
         List<CrawlerPlanResultRel> crawlerPlanRelList = resultRelMap.get(publishContent.getCrawlerChannelContentId());
-        if (CollectionUtil.isNotEmpty(crawlerPlanRelList) && CollectionUtil.isNotEmpty(producePlanInputSourceList)) {
+        if (CollectionUtils.isNotEmpty(crawlerPlanRelList) && CollectionUtils.isNotEmpty(producePlanInputSourceList)) {
             List<String> inputSourceValues = producePlanInputSourceList.stream()
                     .map(ProducePlanInputSource::getInputSourceValue).collect(Collectors.toList());
-            List<CrawlerPlan> crawlerPlanItemList = new ArrayList<>();
-            for (CrawlerPlanResultRel crawlerPlanResultRel : crawlerPlanRelList) {
-                crawlerPlanItemList.add(crawlerPlanMap.get(crawlerPlanResultRel.getPlanId()));
-            }
-            for (CrawlerPlan crawlerPlan : crawlerPlanItemList) {
-                if (inputSourceValues.contains(crawlerPlan.getId())) {
-                    obj.setCrawlerPlanName(crawlerPlan.getName());
-                    obj.setCrawlerPlanTag(crawlerPlan.getPlanTag());
-                    break;
+            if (CollectionUtils.isNotEmpty(inputSourceValues)) {
+                List<CrawlerPlan> crawlerPlanItemList = new ArrayList<>();
+                for (CrawlerPlanResultRel crawlerPlanResultRel : crawlerPlanRelList) {
+                    CrawlerPlan crawlerPlan = crawlerPlanMap.get(crawlerPlanResultRel.getPlanId());
+                    if (Objects.nonNull(crawlerPlan)) {
+                        crawlerPlanItemList.add(crawlerPlan);
+                    }
+                }
+                if (CollectionUtils.isNotEmpty(crawlerPlanItemList)) {
+                    for (CrawlerPlan crawlerPlan : crawlerPlanItemList) {
+                        if (inputSourceValues.contains(crawlerPlan.getId())) {
+                            obj.setCrawlerPlanName(crawlerPlan.getName());
+                            obj.setCrawlerPlanTag(crawlerPlan.getPlanTag());
+                            break;
+                        }
+                    }
                 }
             }
         }