Merge branch 'wyp/0125-contentFunnel' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -38,4 +38,16 @@ public interface AigcBaseMapper {
     List<ProduceTaskAtom> getProduceScoreByContentId(List<String> planExeIds);
 
     List<ContentPoolTypeDTO> getContentPoolType(List<String> contentIds);
+
+    List<String> getProducePlanId();
+
+    List<String> getCrawlerPlanByProducePlanIds(List<String> producePlanIds);
+
+    Long getCrawlerContentCountByCrawlerPlanIds(List<String> crawlerPlanIds, Long start, Long end);
+
+    List<String> getCrawlerSuccessPlanByCrawlerPlanIds(List<String> crawlerPlanIds, Long start, Long end);
+
+    Long getProduceContentCountByProducePlanIds(List<String> producePlanIds, Long start, Long end);
+
+    Long getProduceAuditPassCountByProducePlanIds(List<String> producePlanIds, Long start, Long end);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -7,6 +7,7 @@ import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
 import com.tzld.longarticle.recommend.server.model.param.ArticleVideoPoolSourceParam;
 import org.apache.ibatis.annotations.Mapper;
 
+import java.util.Date;
 import java.util.List;
 
 @Mapper
@@ -74,4 +75,6 @@ public interface LongArticleBaseMapper {
     void batchInsertArticleReMatchRecord(List<ArticleReMatchRecord> list);
 
     List<String> getDisAuditDeleteGhid();
+
+    Long countMatchSuccessCount(Date start, Date end);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/ContentFunnelExport.java

@@ -0,0 +1,16 @@
+package com.tzld.longarticle.recommend.server.model.vo;
+
+import lombok.Data;
+
+@Data
+public class ContentFunnelExport {
+
+    private String dateStr;
+    private Long planCrawlerCount = 0L;
+    private Long crawlerCount = 0L;
+    private Long produceCount = 0L;
+    private Long produceAuditPassCount = 0L;
+    private Long matchSuccessCount = 0L;
+    private Long videoAuditPassCount = 0L;
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/DataDashboardService.java

@@ -96,6 +96,8 @@ public class DataDashboardService {
     private LongArticleTitleAuditRepository titleAuditRepository;
     @Autowired
     private LongArticleCrawlerVideoRepository crawlerVideoRepository;
+    @Autowired
+    private LongArticleTitleAuditRepository longArticleTitleAuditRepository;
 
     @ApolloJsonValue("${export.account.ghId:[]}")
     private static List<String> ghIdList;
@@ -2010,7 +2012,7 @@ public class DataDashboardService {
         List<LongArticleTitleAudit> auditList = titleAuditRepository.getByAuditTimestampBetween(timestamp, timestamp + 86400000);
         List<LongArticleCrawlerVideo> crawlerVideoList = crawlerVideoRepository.getByAuditTimestampBetween(timestamp, timestamp + 86400000);
         Map<String, List<LongArticleCrawlerVideo>> crawlerVideoMap = crawlerVideoList.stream()
-               .collect(Collectors.groupingBy(LongArticleCrawlerVideo::getContentId));
+                .collect(Collectors.groupingBy(LongArticleCrawlerVideo::getContentId));
         List<String> contentIds = auditList.stream().map(LongArticleTitleAudit::getContentId).collect(Collectors.toList());
         List<ContentPoolTypeDTO> poolTypeDTOS = aigcBaseMapper.getContentPoolType(contentIds);
         Map<String, ContentPoolTypeDTO> poolTypeMap = poolTypeDTOS.stream().collect(Collectors.toMap(ContentPoolTypeDTO::getContentId, o -> o));
@@ -2034,7 +2036,7 @@ public class DataDashboardService {
             }
             videoAuditExport.setVideoAuditCount(videoAuditExport.getVideoAuditCount() + videoList.size());
             long videoAuditPassCount = videoList.stream()
-                   .filter(o -> o.getStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
+                    .filter(o -> o.getStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
             videoAuditExport.setVideoAuditPassCount(videoAuditExport.getVideoAuditPassCount() + videoAuditPassCount);
         }
         for (Map.Entry<String, VideoAuditExport> entry : map.entrySet()) {
@@ -2061,4 +2063,93 @@ public class DataDashboardService {
         return result;
     }
 
+    @XxlJob("contentFunnelExport")
+    public ReturnT<String> contentFunnelExportJob(String param) {
+        List<String> dateStrList = DateUtils.getBeforeDays(null, null, 1);
+        contentFunnelExport(dateStrList);
+        return ReturnT.SUCCESS;
+    }
+
+    public void contentFunnelExport(String dateStr) {
+        if (!StringUtils.hasText(dateStr)) {
+            dateStr = DateUtils.getBeforeDaysDateStr("yyyyMMdd", 1);
+        }
+        contentFunnelExport(Collections.singletonList(dateStr));
+    }
+
+    public void contentFunnelExport(List<String> dateStrList) {
+        List<ContentFunnelExport> exportList = new ArrayList<>();
+        dateStrList = Lists.reverse(dateStrList);
+        for (String dateStr : dateStrList) {
+            exportList.add(buildContentFunnelExport(dateStr));
+        }
+        if (CollectionUtil.isEmpty(exportList)) {
+            return;
+        }
+        int rowNum = exportList.size();
+        List<List<Object>> rows = new ArrayList<>();
+        Field[] fields = ContentFunnelExport.class.getDeclaredFields();
+        for (ContentFunnelExport datum : exportList) {
+            List<Object> rowDatas = new ArrayList<>();
+            rows.add(rowDatas);
+
+            for (Field field : fields) {
+                field.setAccessible(true);
+                try {
+                    rowDatas.add(field.get(datum));
+                } catch (IllegalAccessException e) {
+                    log.error("获取值出错：{}", field.getName());
+                } catch (Exception e) {
+                    throw new RuntimeException(e.getMessage());
+                }
+            }
+        }
+
+        doSendFeishuSheet(dateStrList, dailyDetailSheetToken, "qEipyL", rowNum, rows,
+                2, null, null);
+    }
+
+    private ContentFunnelExport buildContentFunnelExport(String dateStr) {
+        ContentFunnelExport result = new ContentFunnelExport();
+        Long start = DateUtils.getStartOfDay(dateStr, "yyyyMMdd") * 1000;
+        Long end = start + 86400000;
+        List<String> producePlanIds = aigcBaseMapper.getProducePlanId();
+        List<String> crawlerPlanIds = aigcBaseMapper.getCrawlerPlanByProducePlanIds(producePlanIds);
+        Long crawlerCount = aigcBaseMapper.getCrawlerContentCountByCrawlerPlanIds(crawlerPlanIds, start, end);
+        Long planCrawlerCount = 0L;
+        List<String> crawlerSuccessPlanIds = aigcBaseMapper.getCrawlerSuccessPlanByCrawlerPlanIds(crawlerPlanIds, start, end);
+        List<CrawlerPlan> crawlerPlanList = aigcBaseMapper.getCrawlerPlanByPlanIds(crawlerSuccessPlanIds);
+        for (CrawlerPlan crawlerPlan : crawlerPlanList) {
+            JSONObject crawlerModeValue = JSONObject.parseObject(crawlerPlan.getCrawlerModeValue());
+            if (crawlerModeValue == null) {
+                continue;
+            }
+            JSONObject sourceModeValues = crawlerModeValue.getJSONObject("sourceModeValues");
+            if (sourceModeValues == null) {
+                continue;
+            }
+            JSONArray inputModeValues = sourceModeValues.getJSONArray("inputModeValues");
+            if (inputModeValues == null) {
+                continue;
+            }
+            planCrawlerCount += inputModeValues.size();
+        }
+        Long produceCount = aigcBaseMapper.getProduceContentCountByProducePlanIds(producePlanIds, start, end);
+        Long produceAuditPassCount = aigcBaseMapper.getProduceAuditPassCountByProducePlanIds(producePlanIds, start, end);
+        Date dateStart = DateUtils.getStartDateOfDay(start / 1000);
+        Date dateEnd = DateUtils.getStartDateOfDay(end / 1000);
+        Long matchSuccessCount = longArticleBaseMapper.countMatchSuccessCount(dateStart, dateEnd);
+        Long videoAuditPassCount = longArticleTitleAuditRepository.countByStatusAndAuditTimestampBetween(
+                1, start, end);
+
+        result.setDateStr(dateStr);
+        result.setPlanCrawlerCount(planCrawlerCount);
+        result.setCrawlerCount(crawlerCount);
+        result.setProduceCount(produceCount);
+        result.setProduceAuditPassCount(produceAuditPassCount);
+        result.setMatchSuccessCount(matchSuccessCount);
+        result.setVideoAuditPassCount(videoAuditPassCount);
+        return result;
+    }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/recommend/DataDashboardController.java

@@ -57,6 +57,13 @@ public class DataDashboardController {
         }).start();
     }
 
+    @GetMapping("/export/contentFunnel")
+    public void contentFunnelExport(String dateStr) {
+        new Thread(() -> {
+            service.contentFunnelExport(dateStr);
+        }).start();
+    }
+
     @GetMapping("/export/dailySafeScore")
     public void dailySafeScore(String dateStr) {
         new Thread(() -> {

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -139,4 +139,62 @@
         </foreach>
     </select>
 
+    <select id="getProducePlanId" resultType="java.lang.String">
+        select input.input_source_value
+        from publish_plan plan
+         join publish_plan_input_source input on plan.id = input.plan_id
+        where plan.plan_status = 1 and plan.channel = 5
+    </select>
+
+    <select id="getCrawlerPlanByProducePlanIds" resultType="java.lang.String">
+        select input.input_source_value
+        from produce_plan plan
+         join produce_plan_input_source input on plan.id = input.plan_id
+        where plan.id in
+        <foreach collection="producePlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+    </select>
+
+    <select id="getCrawlerContentCountByCrawlerPlanIds" resultType="java.lang.Long">
+        select count(1)
+        from crawler_plan_result_rel
+        where plan_id in
+        <foreach collection="crawlerPlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and create_timestamp between #{start} and #{end}
+    </select>
+
+    <select id="getCrawlerSuccessPlanByCrawlerPlanIds" resultType="java.lang.String">
+        select distinct plan_id
+        from crawler_plan_result_rel
+        where plan_id in
+        <foreach collection="crawlerPlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and create_timestamp between #{start} and #{end}
+    </select>
+
+    <select id="getProduceContentCountByProducePlanIds" resultType="java.lang.Long">
+        select count(1)
+        from produce_plan_exe_record
+        where plan_id in
+        <foreach collection="producePlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and produce_timestamp between #{start} and #{end}
+    </select>
+
+    <select id="getProduceAuditPassCountByProducePlanIds" resultType="java.lang.Long">
+        select count(1)
+        from produce_plan_exe_record
+        where plan_id in
+        <foreach collection="producePlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and audit_status = 1
+        and audit_timestamp between #{start} and #{end}
+    </select>
+
 </mapper>

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -271,4 +271,10 @@
         select distinct gh_id from long_article_audit_delete where create_timestamp > 1736157600000 and status = 2 and fail_reason like 'reach max api daily quota%'
     </select>
 
+    <select id="countMatchSuccessCount" resultType="java.lang.Long">
+        select count(distinct content_id)
+        from long_articles_crawler_videos
+        where crawler_time between #{start} and #{end}
+    </select>
+
 </mapper>