|
@@ -110,6 +110,10 @@ public class DataDashboardService {
|
|
|
private PublishSingleVideoSourceRepository videoPoolRepository;
|
|
|
@Autowired
|
|
|
private CrawlerBaseMapper crawlerBaseMapper;
|
|
|
+ @Autowired
|
|
|
+ private LongArticlesRootSourceIdRepository rootSourceIdRepository;
|
|
|
+ @Autowired
|
|
|
+ private DatastatSortStrategyRepository datastatSortStrategyRepository;
|
|
|
|
|
|
@ApolloJsonValue("${export.account.ghId:[]}")
|
|
|
private static List<String> ghIdList;
|
|
@@ -212,7 +216,7 @@ public class DataDashboardService {
|
|
|
Set<String> snList = articleList.stream().map(Article::getWxSn).collect(Collectors.toSet());
|
|
|
List<ArticleDetailInfo> articleDetailInfoList = new ArrayList<>();
|
|
|
for (List<String> partitions : Lists.partition(new ArrayList<>(snList), 1000)) {
|
|
|
- articleDetailInfoList.addAll(articleDetailInfoRepository.getAllByWxSnIn(partitions));
|
|
|
+ articleDetailInfoList.addAll(articleDetailInfoRepository.getByWxSnIn(partitions));
|
|
|
}
|
|
|
log.info("newSortStrategyData articleDetailInfoList finish");
|
|
|
Map<String, List<ArticleDetailInfo>> articleDetailInfoMap = articleDetailInfoList.stream()
|
|
@@ -357,7 +361,7 @@ public class DataDashboardService {
|
|
|
List<ArticleDetailInfo> hisArticleDetailInfoList = new ArrayList<>();
|
|
|
List<List<String>> hisSnPartition = Lists.partition(new ArrayList<>(hisWxSnList), 1000);
|
|
|
for (List<String> sns : hisSnPartition) {
|
|
|
- hisArticleDetailInfoList.addAll(articleDetailInfoRepository.getAllByWxSnIn(sns));
|
|
|
+ hisArticleDetailInfoList.addAll(articleDetailInfoRepository.getByWxSnIn(sns));
|
|
|
}
|
|
|
log.info("newSortStrategyData hisArticleDetailInfoList finish");
|
|
|
Map<String, List<ArticleDetailInfo>> hisArticleDetailInfoMap = hisArticleDetailInfoList.stream()
|
|
@@ -2607,4 +2611,316 @@ public class DataDashboardService {
|
|
|
return sum;
|
|
|
}
|
|
|
|
|
|
+
|
|
|
+ @XxlJob("contentEffectGroupBySourceExport")
|
|
|
+ public ReturnT<String> contentEffectGroupBySourceJob(String param) {
|
|
|
+ List<String> dateStrList = DateUtils.getBeforeDays(null, null, 1);
|
|
|
+ contentEffectGroupBySource(dateStrList);
|
|
|
+ return ReturnT.SUCCESS;
|
|
|
+ }
|
|
|
+
|
|
|
+ public void contentEffectGroupBySource(String dateStr) {
|
|
|
+ if (!StringUtils.hasText(dateStr)) {
|
|
|
+ dateStr = DateUtils.getBeforeDaysDateStr("yyyyMMdd", 1);
|
|
|
+ }
|
|
|
+ contentEffectGroupBySource(Collections.singletonList(dateStr));
|
|
|
+ }
|
|
|
+
|
|
|
+ public void contentEffectGroupBySource(List<String> dateStrList) {
|
|
|
+ List<ContentEffectGroupBySourceExport> exportList = new ArrayList<>();
|
|
|
+ dateStrList = Lists.reverse(dateStrList);
|
|
|
+ for (String dateStr : dateStrList) {
|
|
|
+ exportList.addAll(buildContentEffectGroupBySourceExport(dateStr));
|
|
|
+ }
|
|
|
+ if (CollectionUtil.isEmpty(exportList)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ int rowNum = exportList.size();
|
|
|
+ List<List<Object>> rows = new ArrayList<>();
|
|
|
+ Field[] fields = ContentEffectGroupBySourceExport.class.getDeclaredFields();
|
|
|
+ for (ContentEffectGroupBySourceExport datum : exportList) {
|
|
|
+ List<Object> rowDatas = new ArrayList<>();
|
|
|
+ rows.add(rowDatas);
|
|
|
+
|
|
|
+ for (Field field : fields) {
|
|
|
+ field.setAccessible(true);
|
|
|
+ try {
|
|
|
+ rowDatas.add(field.get(datum));
|
|
|
+ } catch (IllegalAccessException e) {
|
|
|
+ log.error("获取值出错:{}", field.getName());
|
|
|
+ } catch (Exception e) {
|
|
|
+ throw new RuntimeException(e.getMessage());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ List<Pair<String, String>> styles = Arrays
|
|
|
+ .asList(
|
|
|
+ Pair.of("H", "0.00%"),
|
|
|
+ Pair.of("L", "0.00%"),
|
|
|
+ Pair.of("M", "0.00%"),
|
|
|
+ Pair.of("N", "0.00%"),
|
|
|
+ Pair.of("R", "0.00%"),
|
|
|
+ Pair.of("S", "0.00%"),
|
|
|
+ Pair.of("T", "0.00%")
|
|
|
+ );
|
|
|
+ List<Pair<String, List<Pair<String, String>>>> thank = Arrays
|
|
|
+ .asList(
|
|
|
+ Pair.of("B", Arrays.asList(
|
|
|
+ Pair.of("SUM", "#BACEFD"),
|
|
|
+ Pair.of("文章", "#FED4A4"),
|
|
|
+ Pair.of("视频", "#B1E8FC"),
|
|
|
+ Pair.of("文章SUM", "#F8E6AB"),
|
|
|
+ Pair.of("视频SUM", "#A9EFE6"))),
|
|
|
+ Pair.of("C", Arrays.asList(
|
|
|
+ Pair.of("SUM", "#BACEFD"),
|
|
|
+ Pair.of("公众号文章", "#FED4A4"),
|
|
|
+ Pair.of("头条文章", "#B1E8FC"),
|
|
|
+ Pair.of("公众号视频", "#F8E6AB"),
|
|
|
+ Pair.of("好看视频", "#A9EFE6"),
|
|
|
+ Pair.of("视频号视频", "#FDE2E2"),
|
|
|
+ Pair.of("头条视频", "#ECE2FE"),
|
|
|
+ Pair.of("文章SUM", "#D9F5D6"),
|
|
|
+ Pair.of("视频SUM", "#F8DEF8"))),
|
|
|
+ Pair.of("D", Arrays.asList(
|
|
|
+ Pair.of("SUM", "#BACEFD"),
|
|
|
+ Pair.of("相同", "#FED4A4"),
|
|
|
+ Pair.of("不同", "#B1E8FC")))
|
|
|
+ );
|
|
|
+ doSendFeishuSheet(dateStrList, dailyDetailSheetToken, "qvxJsD", rowNum, rows,
|
|
|
+ 2, styles, null, thank);
|
|
|
+ }
|
|
|
+
|
|
|
+ private List<ContentEffectGroupBySourceExport> buildContentEffectGroupBySourceExport(String dateStr) {
|
|
|
+ List<ContentEffectGroupBySourceExport> result = buildContentEffectGroupBySource(dateStr);
|
|
|
+ Long start = DateUtils.getStartOfDay(dateStr, "yyyyMMdd");
|
|
|
+ Long end = start + 86400;
|
|
|
+ List<Article> articleList = articleRepository.getByPublishTimestampBetweenAndTypeEquals(start, end, ArticleTypeEnum.QUNFA.getVal());
|
|
|
+ List<String> wxSnList = articleList.stream().map(Article::getWxSn).collect(Collectors.toList());
|
|
|
+ List<ArticleDetailInfo> articleDetailInfoList = new ArrayList<>();
|
|
|
+ for (List<String> partitions : Lists.partition(wxSnList, 1000)) {
|
|
|
+ articleDetailInfoList.addAll(articleDetailInfoRepository.getByWxSnIn(partitions));
|
|
|
+ }
|
|
|
+ List<DatastatSortStrategy> datastatSortStrategyList = datastatSortStrategyRepository.getByWxSnIn(wxSnList);
|
|
|
+ Map<String, DatastatSortStrategy> datastatSortStrategyMap = datastatSortStrategyList.stream()
|
|
|
+ .collect(Collectors.toMap(DatastatSortStrategy::getWxSn, Function.identity()));
|
|
|
+ Map<String, List<ArticleDetailInfo>> articleDetailInfoMap = articleDetailInfoList.stream()
|
|
|
+ .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
|
|
|
+ // 获取rootSourceId
|
|
|
+ Map<String, List<String>> rootSourceIdMap = new HashMap<>();
|
|
|
+ List<String> rootSourceIdList = new ArrayList<>();
|
|
|
+ for (Article article : articleList) {
|
|
|
+ List<String> rootSourceIds = JSONArray.parseArray(article.getRootSourceIdList(), String.class);
|
|
|
+ rootSourceIdMap.put(article.getWxSn(), rootSourceIds);
|
|
|
+ rootSourceIdList.addAll(rootSourceIds);
|
|
|
+ }
|
|
|
+ List<LongArticlesRootSourceId> longArticlesRootSourceIdList = rootSourceIdRepository.getByRootSourceIdIn(rootSourceIdList);
|
|
|
+ Map<String, LongArticlesRootSourceId> longArticlesRootSourceIdMap = longArticlesRootSourceIdList.stream()
|
|
|
+ .collect(Collectors.toMap(LongArticlesRootSourceId::getRootSourceId, Function.identity()));
|
|
|
+ // 获取视频
|
|
|
+ List<String> contentIdList = longArticlesRootSourceIdList.stream().map(LongArticlesRootSourceId::getContentId).distinct().collect(Collectors.toList());
|
|
|
+ List<LongArticleCrawlerVideo> longArticleCrawlerVideoList = crawlerVideoRepository.getByContentIdInAndIsIllegal(contentIdList, 0);
|
|
|
+ Map<String, List<LongArticleCrawlerVideo>> longArticleCrawlerVideoMap = longArticleCrawlerVideoList.stream()
|
|
|
+ .collect(Collectors.groupingBy(LongArticleCrawlerVideo::getContentId));
|
|
|
+ for (Article article : articleList) {
|
|
|
+ boolean isVideo = false;
|
|
|
+ boolean isSameMiniProgram = false;
|
|
|
+ String source = "";
|
|
|
+ List<String> rootSourceIds = rootSourceIdMap.get(article.getWxSn());
|
|
|
+ if (CollectionUtil.isNotEmpty(rootSourceIds)) {
|
|
|
+ List<LongArticlesRootSourceId> rootSourceIdItemList = new ArrayList<>();
|
|
|
+ for (String rootSourceId : rootSourceIds) {
|
|
|
+ LongArticlesRootSourceId longArticlesRootSourceId = longArticlesRootSourceIdMap.get(rootSourceId);
|
|
|
+ if (Objects.isNull(longArticlesRootSourceId)) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ rootSourceIdItemList.add(longArticlesRootSourceId);
|
|
|
+ }
|
|
|
+ String contentId = rootSourceIdItemList.get(0).getContentId();
|
|
|
+ List<LongArticleCrawlerVideo> crawlerVideoList = longArticleCrawlerVideoMap.get(contentId);
|
|
|
+ List<String> videoOssPaths = crawlerVideoList.stream().map(LongArticleCrawlerVideo::getVideoOssPath).collect(Collectors.toList());
|
|
|
+ List<PublishSingleVideoSource> singleVideoSources = videoPoolRepository.getByVideoOssPathIn(videoOssPaths);
|
|
|
+ if (CollectionUtil.isNotEmpty(singleVideoSources)) {
|
|
|
+ isVideo = true;
|
|
|
+ source = VideoPoolPlatformEnum.from(singleVideoSources.get(0).getPlatform()).getDescription();
|
|
|
+ }
|
|
|
+ if (CollectionUtil.isNotEmpty(crawlerVideoList) && crawlerVideoList.size() == 1) {
|
|
|
+ isSameMiniProgram = true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ List<ArticleDetailInfo> detailInfoList = articleDetailInfoMap.get(article.getWxSn());
|
|
|
+ DatastatSortStrategy datastatSortStrategy = datastatSortStrategyMap.get(article.getWxSn());
|
|
|
+ if (isVideo) {
|
|
|
+ addContentEffectGroupBySourceRate(article, "视频", source, isSameMiniProgram, result, detailInfoList,
|
|
|
+ datastatSortStrategy);
|
|
|
+ } else {
|
|
|
+ addContentEffectGroupBySourceRate(article, "文章", source, isSameMiniProgram, result, detailInfoList,
|
|
|
+ datastatSortStrategy);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for (ContentEffectGroupBySourceExport export : result) {
|
|
|
+ setContentEffectGroupBySourceRate(export);
|
|
|
+ }
|
|
|
+
|
|
|
+ result.add(buildContentEffectGroupBySourceSum(dateStr, result));
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ private void addContentEffectGroupBySourceRate(Article article, String type, String source,
|
|
|
+ boolean isSameMiniProgram,
|
|
|
+ List<ContentEffectGroupBySourceExport> result,
|
|
|
+ List<ArticleDetailInfo> detailInfoList,
|
|
|
+ DatastatSortStrategy datastatSortStrategy) {
|
|
|
+ if (Objects.isNull(datastatSortStrategy)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ if ("文章".equals(type) && "".equals(source)) {
|
|
|
+ source = getArticleSource(datastatSortStrategy);
|
|
|
+ }
|
|
|
+ for (ContentEffectGroupBySourceExport export : result) {
|
|
|
+ if ((export.getType().contains("SUM") && export.getType().contains(type)) || export.getType().equals(type)) {
|
|
|
+ if ((export.getIsSameMiniprogram().equals("相同") && isSameMiniProgram)
|
|
|
+ || (export.getIsSameMiniprogram().equals("不同") && !isSameMiniProgram)
|
|
|
+ || export.getIsSameMiniprogram().equals("SUM")) {
|
|
|
+ if (export.getSource().contains("SUM") || export.getSource().equals(source)) {
|
|
|
+ if (Objects.nonNull(datastatSortStrategy.getFans())) {
|
|
|
+ export.setFansCount(export.getFansCount() + datastatSortStrategy.getFans());
|
|
|
+ }
|
|
|
+ export.setPublishContentCount(export.getPublishContentCount() + 1);
|
|
|
+ export.setReadCount(export.getReadCount() + article.getShowViewCount());
|
|
|
+ if (CollectionUtil.isNotEmpty(detailInfoList)) {
|
|
|
+ int totalFirstLevel = 0;
|
|
|
+ int totalT0Fission = 0;
|
|
|
+ int card1FirstLevel = 0;
|
|
|
+ int card1T0Fission = 0;
|
|
|
+ int card2FirstLevel = 0;
|
|
|
+ int card2T0Fission = 0;
|
|
|
+ for (ArticleDetailInfo articleDetailInfo : detailInfoList) {
|
|
|
+ if (Objects.isNull(articleDetailInfo.getFirstLevel())) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (articleDetailInfo.getVideoIndex() == 1) {
|
|
|
+ card1FirstLevel += articleDetailInfo.getFirstLevel();
|
|
|
+ if (Objects.nonNull(articleDetailInfo.getFission0())) {
|
|
|
+ card1T0Fission += articleDetailInfo.getFission0();
|
|
|
+ }
|
|
|
+ } else if (articleDetailInfo.getVideoIndex() == 2) {
|
|
|
+ card2FirstLevel += articleDetailInfo.getFirstLevel();
|
|
|
+ if (Objects.nonNull(articleDetailInfo.getFission0())) {
|
|
|
+ card2T0Fission += articleDetailInfo.getFission0();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ totalFirstLevel += articleDetailInfo.getFirstLevel();
|
|
|
+ if (Objects.nonNull(articleDetailInfo.getFission0())) {
|
|
|
+ totalT0Fission += articleDetailInfo.getFission0();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ export.setFirstLevel(export.getFirstLevel() + totalFirstLevel);
|
|
|
+ export.setMiniprogram1FirstLevel(export.getMiniprogram1FirstLevel() + card1FirstLevel);
|
|
|
+ export.setMiniprogram2FirstLevel(export.getMiniprogram2FirstLevel() + card2FirstLevel);
|
|
|
+ export.setT0Fission(export.getT0Fission() + totalT0Fission);
|
|
|
+ export.setMiniprogram1T0Fission(export.getMiniprogram1T0Fission() + card1T0Fission);
|
|
|
+ export.setMiniprogram2T0Fission(export.getMiniprogram2T0Fission() + card2T0Fission);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private String getArticleSource(DatastatSortStrategy datastatSortStrategy) {
|
|
|
+ if (StringUtils.hasText(datastatSortStrategy.getProducePlanName())
|
|
|
+ && datastatSortStrategy.getProducePlanName().contains("头条")) {
|
|
|
+ return "头条文章";
|
|
|
+ } else {
|
|
|
+ return "公众号文章";
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ private List<ContentEffectGroupBySourceExport> buildContentEffectGroupBySource(String dateStr) {
|
|
|
+ List<ContentEffectGroupBySourceExport> result = new ArrayList<>();
|
|
|
+ List<String> isSameMiniprogram = Arrays.asList("相同", "不同", "SUM");
|
|
|
+ List<String> articleSource = Arrays.asList("公众号文章", "头条文章");
|
|
|
+ List<String> videoSource = Arrays.asList("公众号视频", "好看视频", "视频号视频", "头条视频");
|
|
|
+ for (String source : articleSource) {
|
|
|
+ for (String same : isSameMiniprogram) {
|
|
|
+ ContentEffectGroupBySourceExport item = new ContentEffectGroupBySourceExport();
|
|
|
+ item.setDateStr(dateStr);
|
|
|
+ item.setType("文章");
|
|
|
+ item.setSource(source);
|
|
|
+ item.setIsSameMiniprogram(same);
|
|
|
+ result.add(item);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for (String same : isSameMiniprogram) {
|
|
|
+ ContentEffectGroupBySourceExport item = new ContentEffectGroupBySourceExport();
|
|
|
+ item.setDateStr(dateStr);
|
|
|
+ item.setType("文章SUM");
|
|
|
+ item.setSource("文章SUM");
|
|
|
+ item.setIsSameMiniprogram(same);
|
|
|
+ result.add(item);
|
|
|
+ }
|
|
|
+ for (String source : videoSource) {
|
|
|
+ ContentEffectGroupBySourceExport item = new ContentEffectGroupBySourceExport();
|
|
|
+ item.setDateStr(dateStr);
|
|
|
+ item.setType("视频");
|
|
|
+ item.setSource(source);
|
|
|
+ item.setIsSameMiniprogram("SUM");
|
|
|
+ result.add(item);
|
|
|
+ }
|
|
|
+ ContentEffectGroupBySourceExport item = new ContentEffectGroupBySourceExport();
|
|
|
+ item.setDateStr(dateStr);
|
|
|
+ item.setType("视频SUM");
|
|
|
+ item.setSource("视频SUM");
|
|
|
+ item.setIsSameMiniprogram("SUM");
|
|
|
+ result.add(item);
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ private ContentEffectGroupBySourceExport buildContentEffectGroupBySourceSum(String dateStr, List<ContentEffectGroupBySourceExport> resultList) {
|
|
|
+ ContentEffectGroupBySourceExport sum = new ContentEffectGroupBySourceExport();
|
|
|
+ sum.setDateStr(dateStr);
|
|
|
+ sum.setType("SUM");
|
|
|
+ sum.setSource("SUM");
|
|
|
+ sum.setIsSameMiniprogram("SUM");
|
|
|
+ for (ContentEffectGroupBySourceExport export : resultList) {
|
|
|
+ if (export.getType().contains("SUM")
|
|
|
+ || (export.getType().equals("文章") && export.getIsSameMiniprogram().equals("SUM"))) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ sum.setFansCount(sum.getFansCount() + export.getFansCount());
|
|
|
+ sum.setPublishContentCount(sum.getPublishContentCount() + export.getPublishContentCount());
|
|
|
+ sum.setReadCount(sum.getReadCount() + export.getReadCount());
|
|
|
+ sum.setFirstLevel(sum.getFirstLevel() + export.getFirstLevel());
|
|
|
+ sum.setMiniprogram1FirstLevel(sum.getMiniprogram1FirstLevel() + export.getMiniprogram1FirstLevel());
|
|
|
+ sum.setMiniprogram2FirstLevel(sum.getMiniprogram2FirstLevel() + export.getMiniprogram2FirstLevel());
|
|
|
+ sum.setT0Fission(sum.getT0Fission() + export.getT0Fission());
|
|
|
+ sum.setMiniprogram1T0Fission(sum.getMiniprogram1T0Fission() + export.getMiniprogram1T0Fission());
|
|
|
+ sum.setMiniprogram2T0Fission(sum.getMiniprogram2T0Fission() + export.getMiniprogram2T0Fission());
|
|
|
+ setContentEffectGroupBySourceRate(sum);
|
|
|
+ }
|
|
|
+ return sum;
|
|
|
+ }
|
|
|
+
|
|
|
+ private void setContentEffectGroupBySourceRate(ContentEffectGroupBySourceExport export) {
|
|
|
+ if (Objects.nonNull(export.getFansCount()) && export.getFansCount() > 0) {
|
|
|
+ export.setReadRate(export.getReadCount() * 1.0 / export.getFansCount());
|
|
|
+ }
|
|
|
+ if (Objects.nonNull(export.getReadCount()) && export.getReadCount() > 0) {
|
|
|
+ export.setOpenRate(export.getFirstLevel() * 1.0 / export.getReadCount());
|
|
|
+ export.setMiniprogram1OpenRate(export.getMiniprogram1FirstLevel() * 1.0 / export.getReadCount());
|
|
|
+ export.setMiniprogram2OpenRate(export.getMiniprogram2FirstLevel() * 1.0 / export.getReadCount());
|
|
|
+ }
|
|
|
+ if (Objects.nonNull(export.getFirstLevel()) && export.getFirstLevel() > 0) {
|
|
|
+ export.setT0FissionRate(export.getT0Fission() * 1.0 / export.getFirstLevel());
|
|
|
+ }
|
|
|
+ if (Objects.nonNull(export.getMiniprogram1FirstLevel()) && export.getMiniprogram1FirstLevel() > 0) {
|
|
|
+ export.setMiniprogram1T0FissionRate(export.getMiniprogram1T0Fission() * 1.0 / export.getMiniprogram1FirstLevel());
|
|
|
+ }
|
|
|
+ if (Objects.nonNull(export.getMiniprogram2FirstLevel()) && export.getMiniprogram2FirstLevel() > 0) {
|
|
|
+ export.setMiniprogram2T0FissionRate(export.getMiniprogram2T0Fission() * 1.0 / export.getMiniprogram2FirstLevel());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
}
|