Merge branch 'wyp/0303-contentGroupFunnel' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/ContentGroupFunnelExport.java

@@ -7,56 +7,41 @@ public class ContentGroupFunnelExport {
 
     private String dateStr;
     private String type;
-    private Long planCrawlerCount;
+    private String source;
     private Long crawlerCount;
     private Long produceCount;
-    private Long produceAuditPassCount;
+    private Double produceRate;
     private Long produceAuditCount;
+    private Long produceAuditPassCount;
     private Double produceAuditPassRate;
-    private Long matchSuccessCount;
     private Long matchCount;
+    private Long matchSuccessCount;
     private Double matchSuccessRate;
-    private Long videoAuditPassCount;
-    private Long videoAuditCount;
-    private Double videoAuditPassRate;
-    private Double articleAuditPassRate;
-    private Long videoPoolCrawlerCount;
     private Long videoPoolPQAuditCount;
     private Long videoPoolPQAuditPassCount;
     private Double videoPoolPQAuditPassRate;
-    private Long videoPoolAuditCount;
-    private Long videoPoolAuditPassCount;
-    private Double videoPoolAuditPassRate;
-    private Double videoPoolCrawlerAuditPassRate;
+    private Long videoAuditPassCount;
+    private Long videoAuditCount;
+    private Double videoAuditPassRate;
 
-    public static ContentGroupFunnelExport getArticleDefault() {
+    public static ContentGroupFunnelExport getDefault() {
         ContentGroupFunnelExport export = new ContentGroupFunnelExport();
-        export.setProduceAuditCount(0L);
         export.setCrawlerCount(0L);
         export.setProduceCount(0L);
+        export.setProduceRate(0.0);
         export.setProduceAuditPassCount(0L);
         export.setProduceAuditCount(0L);
         export.setProduceAuditPassRate(0.0);
-        export.setMatchSuccessCount(0L);
         export.setMatchCount(0L);
+        export.setMatchSuccessCount(0L);
         export.setMatchSuccessRate(0.0);
+        export.setVideoPoolPQAuditCount(0L);
+        export.setVideoPoolPQAuditPassCount(0L);
+        export.setVideoPoolPQAuditPassRate(0.0);
         export.setVideoAuditPassCount(0L);
         export.setVideoAuditCount(0L);
         export.setVideoAuditPassRate(0.0);
-        export.setArticleAuditPassRate(0.0);
         return export;
     }
 
-    public static ContentGroupFunnelExport getVideoDefault() {
-        ContentGroupFunnelExport export = new ContentGroupFunnelExport();
-        export.setVideoPoolCrawlerCount(0L);
-        export.setVideoPoolPQAuditCount(0L);
-        export.setVideoPoolPQAuditPassCount(0L);
-        export.setVideoPoolPQAuditPassRate(0.0);
-        export.setVideoPoolAuditCount(0L);
-        export.setVideoPoolAuditPassCount(0L);
-        export.setVideoPoolAuditPassRate(0.0);
-        export.setVideoPoolCrawlerAuditPassRate(0.0);
-        return export;
-    }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/DataDashboardService.java

@@ -187,7 +187,7 @@ public class DataDashboardService {
                         Pair.of("Q", "0.00%"),
                         Pair.of("R", "0.00%")
                 );
-        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles, delDateStrList);
+        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles, delDateStrList, null);
     }
 
     public List<NewSortStrategyExport> newSortStrategyData(String beginDate, String endDate,
@@ -742,7 +742,8 @@ public class DataDashboardService {
     private static void doSendFeishuSheet(List<String> dateStrList, String sheetToken, String sheetId,
                                           int rowNum, List<List<Object>> rows, Integer startRowIndex,
                                           List<Pair<String, String>> styles,
-                                          List<String> delDateStrList) {
+                                          List<String> delDateStrList,
+                                          List<Pair<String, List<Pair<String, String>>>> thanks) {
         Pair<String, Integer> token = FeiShu.requestAccessToken();
         RestTemplate restTemplate = new RestTemplate();
         HttpHeaders httpHeaders = new HttpHeaders();
@@ -865,6 +866,37 @@ public class DataDashboardService {
                 startRow += 4000;
             } while (startRow < rowNum);
         }
+        if (CollectionUtil.isNotEmpty(thanks)) {
+            startRow = startRowIndex;
+            do {
+                for (Pair<String, List<Pair<String, String>>> thank : thanks) {
+                    List<String> keyList = thank.getSecond().stream().map(Pair::getFirst).collect(Collectors.toList());
+                    List<String> colorList = thank.getSecond().stream().map(Pair::getSecond).collect(Collectors.toList());
+                    HttpEntity<Map<Object, Object>> styleEntity = new HttpEntity<>(MapBuilder
+                            .builder()
+                            .put("range", String.format("%s!%s" + startRow + ":%s", sheetId,
+                                    thank.getFirst(), thank.getFirst())
+                                    + (Math.min(startRow + 4000, rowNum + startRowIndex) - 1))
+                            .put("dataValidationType", "list")
+                            .put("dataValidation", MapBuilder.builder()
+                                    .put("conditionValues", keyList)
+                                    .put("options", MapBuilder.builder()
+                                            .put("highlightValidData", true)
+                                            .put("colors", colorList)
+                                            .build()
+                                    ).build())
+                            .build(), httpHeaders);
+                    restTemplate.exchange(
+                            String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/dataValidation",
+                                    sheetToken),
+                            HttpMethod.POST,
+                            styleEntity,
+                            String.class
+                    );
+                }
+                startRow += 4000;
+            } while (startRow < rowNum);
+        }
     }
 
     public static void delFeishuSheet(HttpHeaders httpHeaders, String sheetId, Integer startRowIndex,
@@ -998,7 +1030,7 @@ public class DataDashboardService {
                         Pair.of("BP", "0.00%")
                 );
 
-        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 3, styles, null);
+        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 3, styles, null, null);
     }
 
     public List<IntermediateIndicatorsExport> intermediateIndicatorsData(String dateStr) {
@@ -1628,7 +1660,7 @@ public class DataDashboardService {
                         Pair.of("W", "#,##0.00")
                 );
 
-        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles, null);
+        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles, null, null);
     }
 
     private List<FirstContentScoreExport> firstContentScoreData(List<String> dateStrList) {
@@ -1819,7 +1851,7 @@ public class DataDashboardService {
                 );
 
         doSendFeishuSheet(dateStrList, dailyDetailSheetToken, "28RgAZ", rowNum, rows,
-                2, styles, null);
+                2, styles, null, null);
     }
 
     private Map<String, ProducePlanAuditExport> getOldProducePlanAuditExport(String dateStr, String sheetToken, String sheetId) {
@@ -1973,7 +2005,7 @@ public class DataDashboardService {
         }
 
         doSendFeishuSheet(dateStrList, dailySafeSheetToken, "9fc2e8", rowNum, rows,
-                2, null, null);
+                2, null, null, null);
     }
 
     private List<ProduceSafeScoreExport> getProduceSafeScoreExport(String dateStr) {
@@ -2048,7 +2080,7 @@ public class DataDashboardService {
                 );
 
         doSendFeishuSheet(dateStrList, dailyDetailSheetToken, "vddANt", rowNum, rows,
-                2, styles, null);
+                2, styles, null, null);
     }
 
     private List<VideoAuditExport> buildVideoAuditExport(String dateStr) {
@@ -2209,7 +2241,7 @@ public class DataDashboardService {
                         Pair.of("V", "0.00%")
                 );
         doSendFeishuSheet(dateStrList, dailyDetailSheetToken, "qEipyL", rowNum, rows,
-                4, styles, null);
+                4, styles, null, null);
     }
 
     private ContentFunnelExport buildContentFunnelExport(String dateStr) {
@@ -2342,16 +2374,24 @@ public class DataDashboardService {
 
         List<Pair<String, String>> styles = Arrays
                 .asList(
-                        Pair.of("H", "0.00%"),
-                        Pair.of("K", "0.00%"),
-                        Pair.of("N", "0.00%"),
+                        Pair.of("F", "0.00%"),
+                        Pair.of("I", "0.00%"),
+                        Pair.of("L", "0.00%"),
                         Pair.of("O", "0.00%"),
-                        Pair.of("S", "0.00%"),
-                        Pair.of("V", "0.00%"),
-                        Pair.of("W", "0.00%")
+                        Pair.of("R", "0.00%")
+                );
+        List<Pair<String, List<Pair<String, String>>>> thank = Arrays
+                .asList(
+                        Pair.of("B", Arrays.asList(Pair.of("文章", "#FED4A4"), Pair.of("视频", "#BACEFD"))),
+                        Pair.of("C", Arrays.asList(Pair.of("SUM", "#BACEFD"),
+                                Pair.of("公众号文章", "#FED4A4"),
+                                Pair.of("头条文章", "#B1E8FC"),
+                                Pair.of("公众号视频", "#F8E6AB"),
+                                Pair.of("好看视频", "#A9EFE6"),
+                                Pair.of("视频号视频", "#FDE2E2")))
                 );
-        doSendFeishuSheet(dateStrList, dailyDetailSheetToken, "tP33UK", rowNum, rows,
-                2, styles, null);
+        doSendFeishuSheet(dateStrList, dailyDetailSheetToken, "6aW60b", rowNum, rows,
+                3, styles, null, thank);
     }
 
     private List<ContentGroupFunnelExport> buildContentGroupFunnelExport(String dateStr) {
@@ -2364,7 +2404,16 @@ public class DataDashboardService {
         ContentFunnelExport sum = buildContentFunnelExport(dateStr);
         ContentGroupFunnelExport groupSum = new ContentGroupFunnelExport();
         BeanUtils.copyProperties(sum, groupSum);
-        groupSum.setType("SUM");
+        groupSum.setSource("SUM");
+        if (sum.getCrawlerCount() > 0) {
+            groupSum.setProduceRate(sum.getProduceCount() * 1.0 / sum.getCrawlerCount());
+        }
+        groupSum.setCrawlerCount(sum.getCrawlerCount() + sum.getVideoPoolCrawlerCount());
+        groupSum.setVideoAuditCount(sum.getVideoAuditCount() + sum.getVideoPoolAuditCount());
+        groupSum.setVideoAuditPassCount(sum.getVideoAuditPassCount() + sum.getVideoPoolAuditPassCount());
+        if (groupSum.getVideoAuditCount() > 0) {
+            groupSum.setVideoAuditPassRate(groupSum.getVideoAuditPassCount() * 1.0 / groupSum.getVideoAuditCount());
+        }
         result.add(groupSum);
         // 文章
         List<ChannelEnum> channelList = Arrays.asList(ChannelEnum.wx, ChannelEnum.toutiao);
@@ -2372,26 +2421,6 @@ public class DataDashboardService {
             List<String> producePlanIds = aigcBaseMapper.getProducePlanId();
             List<String> crawlerPlanIds = aigcBaseMapper.getCrawlerPlanByProducePlanIds(producePlanIds);
             Long crawlerCount = aigcBaseMapper.getCrawlerContentCountByCrawlerPlanIdsAndChannel(crawlerPlanIds, start, end, channel.getVal());
-            Long planCrawlerCount = 0L;
-            List<String> crawlerSuccessPlanIds = aigcBaseMapper.getCrawlerSuccessPlanByCrawlerPlanIdsAndChannel(crawlerPlanIds, start, end, channel.getVal());
-            if (CollectionUtils.isNotEmpty(crawlerSuccessPlanIds)) {
-                List<CrawlerPlan> crawlerPlanList = aigcBaseMapper.getCrawlerPlanByPlanIds(crawlerSuccessPlanIds);
-                for (CrawlerPlan crawlerPlan : crawlerPlanList) {
-                    JSONObject crawlerModeValue = JSONObject.parseObject(crawlerPlan.getCrawlerModeValue());
-                    if (crawlerModeValue == null) {
-                        continue;
-                    }
-                    JSONObject sourceModeValues = crawlerModeValue.getJSONObject("sourceModeValues");
-                    if (sourceModeValues == null) {
-                        continue;
-                    }
-                    JSONArray inputModeValues = sourceModeValues.getJSONArray("inputModeValues");
-                    if (inputModeValues == null) {
-                        continue;
-                    }
-                    planCrawlerCount += inputModeValues.size();
-                }
-            }
             Long produceCount = aigcBaseMapper.getProduceContentCountByProducePlanIdsAndChannel(producePlanIds, start, end, channel.getVal());
             List<Integer> auditStatus = new ArrayList<>();
             auditStatus.add(1);
@@ -2422,30 +2451,30 @@ public class DataDashboardService {
                 videoAuditIdChannelList = aigcBaseMapper.getIdChannelByContentId(videoAuditIdList);
             }
 
-            ContentGroupFunnelExport item = ContentGroupFunnelExport.getArticleDefault();
+            ContentGroupFunnelExport item = ContentGroupFunnelExport.getDefault();
             item.setDateStr(dateStr);
-            item.setType(channel.getDescription() + "文章");
-            item.setPlanCrawlerCount(planCrawlerCount);
+            item.setType("文章");
+            item.setSource(channel.getDescription() + "文章");
             item.setCrawlerCount(crawlerCount);
             item.setProduceCount(produceCount);
-            item.setProduceAuditPassCount(produceAuditPassCount);
+            if (item.getCrawlerCount() > 0) {
+                item.setProduceRate(item.getProduceCount() * 1.0 / item.getCrawlerCount());
+            }
             item.setProduceAuditCount(produceAuditCount);
+            item.setProduceAuditPassCount(produceAuditPassCount);
             if (produceAuditCount > 0) {
                 item.setProduceAuditPassRate(produceAuditPassCount * 1.0 / produceAuditCount);
             }
-            item.setMatchSuccessCount(matchSuccessList.stream().filter(o -> o.getChannel().equals(channel.getVal())).count());
             item.setMatchCount(matchList.stream().filter(o -> o.getChannel().equals(channel.getVal())).count());
+            item.setMatchSuccessCount(matchSuccessList.stream().filter(o -> o.getChannel().equals(channel.getVal())).count());
             if (item.getMatchCount() > 0) {
                 item.setMatchSuccessRate(item.getMatchSuccessCount() * 1.0 / item.getMatchCount());
             }
-            item.setVideoAuditPassCount(videoAuditPassIdChannelList.stream().filter(o -> o.getChannel().equals(channel.getVal())).count());
             item.setVideoAuditCount(videoAuditIdChannelList.stream().filter(o -> o.getChannel().equals(channel.getVal())).count());
+            item.setVideoAuditPassCount(videoAuditPassIdChannelList.stream().filter(o -> o.getChannel().equals(channel.getVal())).count());
             if (item.getVideoAuditCount() > 0) {
                 item.setVideoAuditPassRate(item.getVideoAuditPassCount() * 1.0 / item.getVideoAuditCount());
             }
-            if (item.getProduceCount() > 0) {
-                item.setArticleAuditPassRate(item.getVideoAuditPassCount() * 1.0 / item.getProduceCount());
-            }
             result.add(item);
         }
         // 视频
@@ -2457,11 +2486,11 @@ public class DataDashboardService {
                     start / 1000, end / 1000, videoType.getPlatform());
             List<PublishSingleVideoSource> videoPoolAuditList = videoPoolRepository.getByVideoPoolAuditTimestampBetweenAndPlatform(
                     start, end, videoType.getPlatform());
-
-            ContentGroupFunnelExport item = ContentGroupFunnelExport.getVideoDefault();
+            ContentGroupFunnelExport item = ContentGroupFunnelExport.getDefault();
             item.setDateStr(dateStr);
-            item.setType(videoType.getDescription());
-            item.setVideoPoolCrawlerCount((long) videoPoolCrawlerCount.size());
+            item.setType("视频");
+            item.setSource(videoType.getDescription());
+            item.setCrawlerCount((long) videoPoolCrawlerCount.size());
             item.setVideoPoolPQAuditCount((long) videoPoolPQAuditCount.size());
             long videoPoolPQAuditPassCount = videoPoolPQAuditCount.stream()
                     .filter(o -> o.getAuditStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
@@ -2469,15 +2498,12 @@ public class DataDashboardService {
             if (item.getVideoPoolPQAuditCount() > 0) {
                 item.setVideoPoolPQAuditPassRate(videoPoolPQAuditPassCount * 1.0 / item.getVideoPoolPQAuditCount());
             }
-            item.setVideoPoolAuditCount((long) videoPoolAuditList.size());
+            item.setVideoAuditCount((long) videoPoolAuditList.size());
             long videoPoolAuditPassCount = videoPoolAuditList.stream()
                     .filter(o -> o.getVideoPoolAuditStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
-            item.setVideoPoolAuditPassCount(videoPoolAuditPassCount);
-            if (item.getVideoPoolAuditCount() > 0) {
-                item.setVideoPoolAuditPassRate(videoPoolAuditPassCount * 1.0 / item.getVideoPoolAuditCount());
-            }
-            if (item.getVideoPoolCrawlerCount() > 0) {
-                item.setVideoPoolCrawlerAuditPassRate(videoPoolAuditPassCount * 1.0 / item.getVideoPoolCrawlerCount());
+            item.setVideoAuditPassCount(videoPoolAuditPassCount);
+            if (item.getVideoAuditCount() > 0) {
+                item.setVideoAuditPassRate(videoPoolAuditPassCount * 1.0 / item.getVideoAuditCount());
             }
             result.add(item);
         }