video pool dataDashboard

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -1,11 +1,9 @@
 package com.tzld.longarticle.recommend.server.mapper.aigc;
 
-import com.tzld.longarticle.recommend.server.model.dto.ContentPoolTypeDTO;
-import com.tzld.longarticle.recommend.server.model.dto.CrawlerContent;
-import com.tzld.longarticle.recommend.server.model.dto.ProduceContentDTO;
-import com.tzld.longarticle.recommend.server.model.dto.ProducePlanAuditCheckDTO;
+import com.tzld.longarticle.recommend.server.model.dto.*;
 import com.tzld.longarticle.recommend.server.model.dto.aigc.BadCrawlerAccountDTO;
 import com.tzld.longarticle.recommend.server.model.dto.aigc.IdChannelDTO;
+import com.tzld.longarticle.recommend.server.model.dto.aigc.IdPlatformDTO;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.*;
 import com.tzld.longarticle.recommend.server.model.vo.IdNameVO;
 import org.apache.ibatis.annotations.Mapper;
@@ -78,4 +76,12 @@ public interface AigcBaseMapper {
     List<IdChannelDTO> getIdChannelByContentId(List<String> idList);
 
     List<String> getTodaySpecialAdAccountNames(String dateStr);
+
+    Long getVideoPoolPlatformCrawlerCount(Long start, Long end, String platform);
+
+    Long getVideoPoolPlatformProduceCount(List<String> producePlanIds, Long start, Long end, String platform);
+
+    Long getVideoPoolPlatformProduceAuditPassCount(List<String> producePlanIds, List<Integer> auditStatus, Long start, Long end, String platform);
+
+    List<IdPlatformDTO> getIdPlatformByContentId(List<String> idList);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/aigc/IdPlatformDTO.java

@@ -0,0 +1,9 @@
+package com.tzld.longarticle.recommend.server.model.dto.aigc;
+
+import lombok.Data;
+
+@Data
+public class IdPlatformDTO {
+    private String id;
+    private String platform;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/VideoAuditExport.java

@@ -13,8 +13,5 @@ public class VideoAuditExport {
     private Long videoAuditCount = 0L;
     private Long videoAuditPassCount = 0L;
     private Double articleAuditPassRate = 0.0;
-    private Long videoPoolAuditCount = 0L;
-    private Long videoPoolAuditPassCount = 0L;
-    private Double videoPoolAuditPassRate = 0.0;
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/DataDashboardService.java

@@ -25,6 +25,7 @@ import com.tzld.longarticle.recommend.server.model.dto.ProduceContentDTO;
 import com.tzld.longarticle.recommend.server.model.dto.ProducePlanAuditCheckDTO;
 import com.tzld.longarticle.recommend.server.model.dto.PublishContentDTO;
 import com.tzld.longarticle.recommend.server.model.dto.aigc.IdChannelDTO;
+import com.tzld.longarticle.recommend.server.model.dto.aigc.IdPlatformDTO;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.*;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
@@ -50,7 +51,6 @@ import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
-import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.util.Pair;
 import org.springframework.http.*;
@@ -2160,10 +2160,6 @@ public class DataDashboardService {
             videoAuditExport.setPoolType(contentPool.getContentPool());
             map.put(contentPool.getContentPool(), videoAuditExport);
         }
-        List<PublishSingleVideoSource> videoPoolAuditList = videoPoolRepository.getByVideoPoolAuditTimestampBetween(
-                timestamp, timestamp + 86400000);
-        Map<Integer, List<PublishSingleVideoSource>> videoPoolAuditMap = videoPoolAuditList.stream()
-                .collect(Collectors.groupingBy(PublishSingleVideoSource::getFlowPoolLevel));
         for (LongArticleTitleAudit titleAudit : auditList) {
             ContentPoolTypeDTO poolTypeDTO = poolTypeMap.get(titleAudit.getContentId());
             if (!map.containsKey(poolTypeDTO.getPoolType())) {
@@ -2198,17 +2194,6 @@ public class DataDashboardService {
             if (CollectionUtils.isNotEmpty(watingAuditList)) {
                 videoAuditExport.setWaitingAuditCount(watingAuditList.size());
             }
-            List<PublishSingleVideoSource> videoPoolList = videoPoolAuditMap.get(poolEnum.getValue());
-            if (CollectionUtils.isNotEmpty(videoPoolList)) {
-                long videoPoolAuditCount = videoPoolList.size();
-                long videoPoolAuditPassCount = videoPoolList.stream()
-                        .filter(o -> o.getVideoPoolAuditStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
-                videoAuditExport.setVideoPoolAuditCount(videoPoolAuditCount);
-                videoAuditExport.setVideoPoolAuditPassCount(videoPoolAuditPassCount);
-                if (videoPoolAuditCount > 0) {
-                    videoAuditExport.setVideoPoolAuditPassRate(videoPoolAuditPassCount * 1.0 / videoPoolAuditCount);
-                }
-            }
             result.add(videoAuditExport);
         }
         // sum
@@ -2218,9 +2203,6 @@ public class DataDashboardService {
         long videoAuditCount = crawlerVideoList.size();
         long videoAuditPassCount = crawlerVideoList.stream()
                 .filter(o -> o.getStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
-        long videoPoolAuditCount = videoPoolAuditList.size();
-        long videoPoolAuditPassCount = videoPoolAuditList.stream()
-                .filter(o -> o.getVideoPoolAuditStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
         VideoAuditExport sum = new VideoAuditExport();
         sum.setDateStr(dateStr);
         sum.setPoolType("SUM");
@@ -2232,11 +2214,6 @@ public class DataDashboardService {
         if (sum.getVideoAuditCount() > 0) {
             sum.setArticleAuditPassRate(sum.getArticleAuditPassCount() * 1.0 / sum.getArticleAuditCount());
         }
-        sum.setVideoPoolAuditCount(videoPoolAuditCount);
-        sum.setVideoPoolAuditPassCount(videoPoolAuditPassCount);
-        if (sum.getVideoPoolAuditCount() > 0) {
-            sum.setVideoPoolAuditPassRate(sum.getVideoPoolAuditPassCount() * 1.0 / sum.getVideoPoolAuditCount());
-        }
         result.add(sum);
         result.sort(Comparator.comparing(VideoAuditExport::getPoolType));
         return result;
@@ -2459,11 +2436,20 @@ public class DataDashboardService {
         Long end = start + 86400000;
         Date dateStart = DateUtils.getStartDateOfDay(start / 1000);
         Date dateEnd = DateUtils.getStartDateOfDay(end / 1000);
+        List<String> producePlanIds = aigcBaseMapper.getProducePlanId();
+        List<String> crawlerPlanIds = aigcBaseMapper.getCrawlerPlanByProducePlanIds(producePlanIds);
+        // 匹配
+        List<String> matchSuccessIdList = longArticleBaseMapper.getMatchSuccessContentId(dateStart, dateEnd);
+        List<String> matchIdList = longArticleBaseMapper.getMatchContentId(dateStart, dateEnd);
+        // 视频审核
+        List<LongArticleTitleAudit> videoAuditPassList = longArticleTitleAuditRepository.getByStatusAndAuditTimestampBetween(
+                1, start, end);
+        List<LongArticleTitleAudit> videoAuditList = longArticleTitleAuditRepository.getByAuditTimestampBetween(start, end);
+        List<String> videoAuditPassIdList = videoAuditPassList.stream().map(LongArticleTitleAudit::getContentId).collect(Collectors.toList());
+        List<String> videoAuditIdList = videoAuditList.stream().map(LongArticleTitleAudit::getContentId).collect(Collectors.toList());
         // 文章
         List<ChannelEnum> channelList = Arrays.asList(ChannelEnum.wx, ChannelEnum.toutiao);
         for (ChannelEnum channel : channelList) {
-            List<String> producePlanIds = aigcBaseMapper.getProducePlanId();
-            List<String> crawlerPlanIds = aigcBaseMapper.getCrawlerPlanByProducePlanIds(producePlanIds);
             Long crawlerCount = aigcBaseMapper.getCrawlerContentCountByCrawlerPlanIdsAndChannel(crawlerPlanIds, start, end, channel.getVal());
             Long produceCount = aigcBaseMapper.getProduceContentCountByProducePlanIdsAndChannel(producePlanIds, start, end, channel.getVal());
             List<Integer> auditStatus = new ArrayList<>();
@@ -2471,30 +2457,22 @@ public class DataDashboardService {
             Long produceAuditPassCount = aigcBaseMapper.getProduceAuditPassCountByProducePlanIdsAndChannel(producePlanIds, auditStatus, start, end, channel.getVal());
             auditStatus.add(2);
             Long produceAuditCount = aigcBaseMapper.getProduceAuditPassCountByProducePlanIdsAndChannel(producePlanIds, auditStatus, start, end, channel.getVal());
-            List<String> matchSuccessIdList = longArticleBaseMapper.getMatchSuccessContentId(dateStart, dateEnd);
             List<IdChannelDTO> matchSuccessList = new ArrayList<>();
             if (CollectionUtils.isNotEmpty(matchSuccessIdList)) {
                 matchSuccessList = aigcBaseMapper.getIdChannelByContentId(matchSuccessIdList);
             }
-            List<String> matchIdList = longArticleBaseMapper.getMatchContentId(dateStart, dateEnd);
             List<IdChannelDTO> matchList = new ArrayList<>();
             if (CollectionUtils.isNotEmpty(matchIdList)) {
                 matchList = aigcBaseMapper.getIdChannelByContentId(matchIdList);
             }
-            List<LongArticleTitleAudit> videoAuditPassList = longArticleTitleAuditRepository.getByStatusAndAuditTimestampBetween(
-                    1, start, end);
-            List<String> videoAuditPassIdList = videoAuditPassList.stream().map(LongArticleTitleAudit::getContentId).collect(Collectors.toList());
             List<IdChannelDTO> videoAuditPassIdChannelList = new ArrayList<>();
             if (CollectionUtils.isNotEmpty(videoAuditPassIdList)) {
                 videoAuditPassIdChannelList = aigcBaseMapper.getIdChannelByContentId(videoAuditPassIdList);
             }
-            List<LongArticleTitleAudit> videoAuditList = longArticleTitleAuditRepository.getByAuditTimestampBetween(start, end);
-            List<String> videoAuditIdList = videoAuditList.stream().map(LongArticleTitleAudit::getContentId).collect(Collectors.toList());
             List<IdChannelDTO> videoAuditIdChannelList = new ArrayList<>();
             if (CollectionUtils.isNotEmpty(videoAuditIdList)) {
                 videoAuditIdChannelList = aigcBaseMapper.getIdChannelByContentId(videoAuditIdList);
             }
-
             ContentGroupFunnelExport item = ContentGroupFunnelExport.getDefault();
             item.setDateStr(dateStr);
             item.setType("文章");
@@ -2525,17 +2503,42 @@ public class DataDashboardService {
         List<VideoPoolPlatformEnum> videoPoolType = Arrays.asList(VideoPoolPlatformEnum.GZH, VideoPoolPlatformEnum.HkSP,
                 VideoPoolPlatformEnum.SPH, VideoPoolPlatformEnum.TOUTIAO);
         for (VideoPoolPlatformEnum videoType : videoPoolType) {
-            List<PublishSingleVideoSource> videoPoolCrawlerCount = videoPoolRepository.getByCrawlerTimestampBetweenAndPlatformAndBadStatus(
-                    start / 1000, end / 1000, videoType.getPlatform(), ArticleVideoBadStatusEnum.TITLE_DEFAULT_STATUS.getCode());
-            List<PublishSingleVideoSource> videoPoolPQAuditCount = videoPoolRepository.getByAuditTimestampBetweenAndPlatform(
-                    start / 1000, end / 1000, videoType.getPlatform());
-            List<PublishSingleVideoSource> videoPoolAuditList = videoPoolRepository.getByVideoPoolAuditTimestampBetweenAndPlatform(
-                    start, end, videoType.getPlatform());
             ContentGroupFunnelExport item = ContentGroupFunnelExport.getDefault();
             item.setDateStr(dateStr);
             item.setType("视频");
             item.setSource(videoType.getDescription());
-            item.setCrawlerCount((long) videoPoolCrawlerCount.size());
+            long videoPoolCrawlerCount = aigcBaseMapper.getVideoPoolPlatformCrawlerCount(start, end, videoType.getPlatform());
+            item.setCrawlerCount(videoPoolCrawlerCount);
+            Long produceCount = aigcBaseMapper.getVideoPoolPlatformProduceCount(producePlanIds, start, end, videoType.getPlatform());
+            item.setProduceCount(produceCount);
+            if (item.getCrawlerCount() > 0) {
+                item.setProduceRate(item.getProduceCount() * 1.0 / item.getCrawlerCount());
+            }
+            List<Integer> auditStatus = new ArrayList<>();
+            auditStatus.add(1);
+            Long produceAuditPassCount = aigcBaseMapper.getVideoPoolPlatformProduceAuditPassCount(producePlanIds, auditStatus, start, end, videoType.getPlatform());
+            auditStatus.add(2);
+            Long produceAuditCount = aigcBaseMapper.getVideoPoolPlatformProduceAuditPassCount(producePlanIds, auditStatus, start, end, videoType.getPlatform());
+            item.setProduceAuditCount(produceAuditCount);
+            item.setProduceAuditPassCount(produceAuditPassCount);
+            if (produceAuditCount > 0) {
+                item.setProduceAuditPassRate(produceAuditPassCount * 1.0 / produceAuditCount);
+            }
+            List<IdPlatformDTO> matchSuccessList = new ArrayList<>();
+            if (CollectionUtils.isNotEmpty(matchSuccessIdList)) {
+                matchSuccessList = aigcBaseMapper.getIdPlatformByContentId(matchSuccessIdList);
+            }
+            List<IdPlatformDTO> matchList = new ArrayList<>();
+            if (CollectionUtils.isNotEmpty(matchIdList)) {
+                matchList = aigcBaseMapper.getIdPlatformByContentId(matchIdList);
+            }
+            item.setMatchCount(matchList.stream().filter(o -> o.getPlatform().equals(videoType.getPlatform())).count());
+            item.setMatchSuccessCount(matchSuccessList.stream().filter(o -> o.getPlatform().equals(videoType.getPlatform())).count());
+            if (item.getMatchCount() > 0) {
+                item.setMatchSuccessRate(item.getMatchSuccessCount() * 1.0 / item.getMatchCount());
+            }
+            List<PublishSingleVideoSource> videoPoolPQAuditCount = videoPoolRepository.getByAuditTimestampBetweenAndPlatform(
+                    start / 1000, end / 1000, videoType.getPlatform());
             item.setVideoPoolPQAuditCount((long) videoPoolPQAuditCount.size());
             long videoPoolPQAuditPassCount = videoPoolPQAuditCount.stream()
                     .filter(o -> o.getAuditStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
@@ -2543,32 +2546,59 @@ public class DataDashboardService {
             if (item.getVideoPoolPQAuditCount() > 0) {
                 item.setVideoPoolPQAuditPassRate(videoPoolPQAuditPassCount * 1.0 / item.getVideoPoolPQAuditCount());
             }
-            item.setVideoAuditCount((long) videoPoolAuditList.size());
-            long videoPoolAuditPassCount = videoPoolAuditList.stream()
-                    .filter(o -> o.getVideoPoolAuditStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
-            item.setVideoAuditPassCount(videoPoolAuditPassCount);
+            List<IdPlatformDTO> videoAuditPassIdPlatformList = new ArrayList<>();
+            if (CollectionUtils.isNotEmpty(videoAuditPassIdList)) {
+                videoAuditPassIdPlatformList = aigcBaseMapper.getIdPlatformByContentId(videoAuditPassIdList);
+            }
+            List<IdPlatformDTO> videoAuditIdPlatformList = new ArrayList<>();
+            if (CollectionUtils.isNotEmpty(videoAuditIdList)) {
+                videoAuditIdPlatformList = aigcBaseMapper.getIdPlatformByContentId(videoAuditIdList);
+            }
+            item.setVideoAuditCount(videoAuditIdPlatformList.stream().filter(o -> o.getPlatform().equals(videoType.getPlatform())).count());
+            item.setVideoAuditPassCount(videoAuditPassIdPlatformList.stream().filter(o -> o.getPlatform().equals(videoType.getPlatform())).count());
             if (item.getVideoAuditCount() > 0) {
-                item.setVideoAuditPassRate(videoPoolAuditPassCount * 1.0 / item.getVideoAuditCount());
+                item.setVideoAuditPassRate(item.getVideoAuditPassCount() * 1.0 / item.getVideoAuditCount());
             }
             result.add(item);
         }
         // sum
-        ContentFunnelExport sum = buildContentFunnelExport(dateStr);
-        ContentGroupFunnelExport groupSum = new ContentGroupFunnelExport();
-        BeanUtils.copyProperties(sum, groupSum);
-        groupSum.setType("SUM");
-        groupSum.setSource("SUM");
-        if (sum.getCrawlerCount() > 0) {
-            groupSum.setProduceRate(sum.getProduceCount() * 1.0 / sum.getCrawlerCount());
-        }
-        groupSum.setCrawlerCount(sum.getCrawlerCount() + sum.getVideoPoolCrawlerCount());
-        groupSum.setVideoAuditCount(sum.getVideoAuditCount() + sum.getVideoPoolAuditCount());
-        groupSum.setVideoAuditPassCount(sum.getVideoAuditPassCount() + sum.getVideoPoolAuditPassCount());
-        if (groupSum.getVideoAuditCount() > 0) {
-            groupSum.setVideoAuditPassRate(groupSum.getVideoAuditPassCount() * 1.0 / groupSum.getVideoAuditCount());
-        }
-        result.add(groupSum);
+        result.add(buildContentFunnelGroupSum(dateStr, result));
         return result;
     }
 
+    private ContentGroupFunnelExport buildContentFunnelGroupSum(String dateStr, List<ContentGroupFunnelExport> result) {
+        ContentGroupFunnelExport sum = ContentGroupFunnelExport.getDefault();
+        sum.setType("SUM");
+        sum.setSource("SUM");
+        sum.setDateStr(dateStr);
+        for (ContentGroupFunnelExport contentGroupFunnelExport : result) {
+            sum.setCrawlerCount(sum.getCrawlerCount() + contentGroupFunnelExport.getCrawlerCount());
+            sum.setProduceCount(sum.getProduceCount() + contentGroupFunnelExport.getProduceCount());
+            if (sum.getCrawlerCount() > 0) {
+                sum.setProduceRate(sum.getProduceCount() * 1.0 / sum.getCrawlerCount());
+            }
+            sum.setProduceAuditCount(sum.getProduceAuditCount() + contentGroupFunnelExport.getProduceAuditCount());
+            sum.setProduceAuditPassCount(sum.getProduceAuditPassCount() + contentGroupFunnelExport.getProduceAuditPassCount());
+            if (sum.getProduceAuditCount() > 0) {
+                sum.setProduceAuditPassRate(sum.getProduceAuditPassCount() * 1.0 / sum.getProduceAuditCount());
+            }
+            sum.setMatchCount(sum.getMatchCount() + contentGroupFunnelExport.getMatchCount());
+            sum.setMatchSuccessCount(sum.getMatchSuccessCount() + contentGroupFunnelExport.getMatchSuccessCount());
+            if (sum.getMatchCount() > 0) {
+                sum.setMatchSuccessRate(sum.getMatchSuccessCount() * 1.0 / sum.getMatchCount());
+            }
+            sum.setVideoAuditCount(sum.getVideoAuditCount() + contentGroupFunnelExport.getVideoAuditCount());
+            sum.setVideoAuditPassCount(sum.getVideoAuditPassCount() + contentGroupFunnelExport.getVideoAuditPassCount());
+            if (sum.getVideoAuditCount() > 0) {
+                sum.setVideoAuditPassRate(sum.getVideoAuditPassCount() * 1.0 / sum.getVideoAuditCount());
+            }
+            sum.setVideoPoolPQAuditCount(sum.getVideoPoolPQAuditCount() + contentGroupFunnelExport.getVideoPoolPQAuditCount());
+            sum.setVideoPoolPQAuditPassCount(sum.getVideoPoolPQAuditPassCount() + contentGroupFunnelExport.getVideoPoolPQAuditPassCount());
+            if (sum.getVideoPoolPQAuditCount() > 0) {
+                sum.setVideoPoolPQAuditPassRate(sum.getVideoPoolPQAuditPassCount() * 1.0 / sum.getVideoPoolPQAuditCount());
+            }
+        }
+        return sum;
+    }
+
 }

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -361,4 +361,54 @@
         where pgss.effective_date = #{dateStr}
     </select>
 
+    <select id="getVideoPoolPlatformCrawlerCount" resultType="java.lang.Long">
+        select count(1)
+        from crawler_content cc
+        join crawler_content_video_pool_relation ccvpr on cc.channel_content_id = ccvpr.channel_content_id
+        where ccvpr.platform = #{platform}
+        and cc.crawler_timestamp between #{start} and #{end}
+    </select>
+
+    <select id="getVideoPoolPlatformProduceCount" resultType="java.lang.Long">
+        select count(1)
+        from produce_plan_exe_record pper
+        join crawler_content cc on pper.channel_content_id = cc.channel_content_id
+        join crawler_content_video_pool_relation ccvpr on cc.channel_content_id = ccvpr.channel_content_id
+        where pper.plan_id in
+        <foreach collection="producePlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and pper.produce_timestamp between #{start} and #{end}
+        and ccvpr.platform = #{platform}
+    </select>
+
+    <select id="getVideoPoolPlatformProduceAuditPassCount" resultType="java.lang.Long">
+        select count(1)
+        from produce_plan_exe_record pper
+        join crawler_content cc on pper.channel_content_id = cc.channel_content_id
+        join crawler_content_video_pool_relation ccvpr on cc.channel_content_id = ccvpr.channel_content_id
+        where pper.plan_id in
+        <foreach collection="producePlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and pper.audit_status in
+        <foreach collection="auditStatus" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and pper.audit_timestamp between #{start} and #{end}
+        and ccvpr.platform = #{platform}
+    </select>
+
+    <select id="getIdPlatformByContentId"
+            resultType="com.tzld.longarticle.recommend.server.model.dto.aigc.IdPlatformDTO">
+        select pper.plan_exe_id as id, ccvpr.platform
+        from produce_plan_exe_record pper
+        join crawler_content cc on pper.channel_content_id = cc.channel_content_id
+        join crawler_content_video_pool_relation ccvpr on cc.channel_content_id = ccvpr.channel_content_id
+        where pper.plan_exe_id in
+        <foreach collection="idList" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+    </select>
+
 </mapper>