|
@@ -7,6 +7,7 @@ import com.alibaba.fastjson.JSONObject;
|
|
|
import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
import com.google.common.collect.Lists;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
|
|
|
+import com.tzld.longarticle.recommend.server.common.enums.aigc.ChannelEnum;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.aigc.ProduceContentAuditStatusEnum;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
|
|
|
import com.tzld.longarticle.recommend.server.common.enums.longArticle.ArticleVideoAuditStatusEnum;
|
|
@@ -22,6 +23,7 @@ import com.tzld.longarticle.recommend.server.model.dto.ContentPoolTypeDTO;
|
|
|
import com.tzld.longarticle.recommend.server.model.dto.ProduceContentDTO;
|
|
|
import com.tzld.longarticle.recommend.server.model.dto.ProducePlanAuditCheckDTO;
|
|
|
import com.tzld.longarticle.recommend.server.model.dto.PublishContentDTO;
|
|
|
+import com.tzld.longarticle.recommend.server.model.dto.aigc.IdChannelDTO;
|
|
|
import com.tzld.longarticle.recommend.server.model.entity.aigc.*;
|
|
|
import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
|
|
|
import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
|
|
@@ -47,6 +49,7 @@ import com.xxl.job.core.handler.annotation.XxlJob;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.apache.commons.collections4.CollectionUtils;
|
|
|
import org.apache.commons.collections4.MapUtils;
|
|
|
+import org.springframework.beans.BeanUtils;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.data.util.Pair;
|
|
|
import org.springframework.http.*;
|
|
@@ -2295,4 +2298,190 @@ public class DataDashboardService {
|
|
|
return result;
|
|
|
}
|
|
|
|
|
|
+ @XxlJob("contentGroupFunnelExport")
|
|
|
+ public ReturnT<String> contentGroupFunnelExportJob(String param) {
|
|
|
+ List<String> dateStrList = DateUtils.getBeforeDays(null, null, 1);
|
|
|
+ contentGroupFunnelExport(dateStrList);
|
|
|
+ return ReturnT.SUCCESS;
|
|
|
+ }
|
|
|
+
|
|
|
+ public void contentGroupFunnelExport(String dateStr) {
|
|
|
+ if (!StringUtils.hasText(dateStr)) {
|
|
|
+ dateStr = DateUtils.getBeforeDaysDateStr("yyyyMMdd", 1);
|
|
|
+ }
|
|
|
+ contentGroupFunnelExport(Collections.singletonList(dateStr));
|
|
|
+ }
|
|
|
+
|
|
|
+ public void contentGroupFunnelExport(List<String> dateStrList) {
|
|
|
+ List<ContentGroupFunnelExport> exportList = new ArrayList<>();
|
|
|
+ dateStrList = Lists.reverse(dateStrList);
|
|
|
+ for (String dateStr : dateStrList) {
|
|
|
+ exportList.addAll(buildContentGroupFunnelExport(dateStr));
|
|
|
+ }
|
|
|
+ if (CollectionUtil.isEmpty(exportList)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ int rowNum = exportList.size();
|
|
|
+ List<List<Object>> rows = new ArrayList<>();
|
|
|
+ Field[] fields = ContentGroupFunnelExport.class.getDeclaredFields();
|
|
|
+ for (ContentGroupFunnelExport datum : exportList) {
|
|
|
+ List<Object> rowDatas = new ArrayList<>();
|
|
|
+ rows.add(rowDatas);
|
|
|
+
|
|
|
+ for (Field field : fields) {
|
|
|
+ field.setAccessible(true);
|
|
|
+ try {
|
|
|
+ rowDatas.add(field.get(datum));
|
|
|
+ } catch (IllegalAccessException e) {
|
|
|
+ log.error("获取值出错:{}", field.getName());
|
|
|
+ } catch (Exception e) {
|
|
|
+ throw new RuntimeException(e.getMessage());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ List<Pair<String, String>> styles = Arrays
|
|
|
+ .asList(
|
|
|
+ Pair.of("H", "0.00%"),
|
|
|
+ Pair.of("K", "0.00%"),
|
|
|
+ Pair.of("N", "0.00%"),
|
|
|
+ Pair.of("O", "0.00%"),
|
|
|
+ Pair.of("S", "0.00%"),
|
|
|
+ Pair.of("V", "0.00%"),
|
|
|
+ Pair.of("W", "0.00%")
|
|
|
+ );
|
|
|
+ doSendFeishuSheet(dateStrList, dailyDetailSheetToken, "tP33UK", rowNum, rows,
|
|
|
+ 2, styles, null);
|
|
|
+ }
|
|
|
+
|
|
|
+ private List<ContentGroupFunnelExport> buildContentGroupFunnelExport(String dateStr) {
|
|
|
+ List<ContentGroupFunnelExport> result = new ArrayList<>();
|
|
|
+ Long start = DateUtils.getStartOfDay(dateStr, "yyyyMMdd") * 1000;
|
|
|
+ Long end = start + 86400000;
|
|
|
+ Date dateStart = DateUtils.getStartDateOfDay(start / 1000);
|
|
|
+ Date dateEnd = DateUtils.getStartDateOfDay(end / 1000);
|
|
|
+ // sum
|
|
|
+ ContentFunnelExport sum = buildContentFunnelExport(dateStr);
|
|
|
+ ContentGroupFunnelExport groupSum = new ContentGroupFunnelExport();
|
|
|
+ BeanUtils.copyProperties(sum, groupSum);
|
|
|
+ groupSum.setType("SUM");
|
|
|
+ result.add(groupSum);
|
|
|
+ // 文章
|
|
|
+ List<ChannelEnum> channelList = Arrays.asList(ChannelEnum.wx, ChannelEnum.toutiao);
|
|
|
+ for (ChannelEnum channel : channelList) {
|
|
|
+ List<String> producePlanIds = aigcBaseMapper.getProducePlanId();
|
|
|
+ List<String> crawlerPlanIds = aigcBaseMapper.getCrawlerPlanByProducePlanIds(producePlanIds);
|
|
|
+ Long crawlerCount = aigcBaseMapper.getCrawlerContentCountByCrawlerPlanIdsAndChannel(crawlerPlanIds, start, end, channel.getVal());
|
|
|
+ Long planCrawlerCount = 0L;
|
|
|
+ List<String> crawlerSuccessPlanIds = aigcBaseMapper.getCrawlerSuccessPlanByCrawlerPlanIdsAndChannel(crawlerPlanIds, start, end, channel.getVal());
|
|
|
+ if (CollectionUtils.isNotEmpty(crawlerSuccessPlanIds)) {
|
|
|
+ List<CrawlerPlan> crawlerPlanList = aigcBaseMapper.getCrawlerPlanByPlanIds(crawlerSuccessPlanIds);
|
|
|
+ for (CrawlerPlan crawlerPlan : crawlerPlanList) {
|
|
|
+ JSONObject crawlerModeValue = JSONObject.parseObject(crawlerPlan.getCrawlerModeValue());
|
|
|
+ if (crawlerModeValue == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ JSONObject sourceModeValues = crawlerModeValue.getJSONObject("sourceModeValues");
|
|
|
+ if (sourceModeValues == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ JSONArray inputModeValues = sourceModeValues.getJSONArray("inputModeValues");
|
|
|
+ if (inputModeValues == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ planCrawlerCount += inputModeValues.size();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ Long produceCount = aigcBaseMapper.getProduceContentCountByProducePlanIdsAndChannel(producePlanIds, start, end, channel.getVal());
|
|
|
+ List<Integer> auditStatus = new ArrayList<>();
|
|
|
+ auditStatus.add(1);
|
|
|
+ Long produceAuditPassCount = aigcBaseMapper.getProduceAuditPassCountByProducePlanIdsAndChannel(producePlanIds, auditStatus, start, end, channel.getVal());
|
|
|
+ auditStatus.add(2);
|
|
|
+ Long produceAuditCount = aigcBaseMapper.getProduceAuditPassCountByProducePlanIdsAndChannel(producePlanIds, auditStatus, start, end, channel.getVal());
|
|
|
+ List<String> matchSuccessIdList = longArticleBaseMapper.getMatchSuccessContentId(dateStart, dateEnd);
|
|
|
+ List<IdChannelDTO> matchSuccessList = new ArrayList<>();
|
|
|
+ if (CollectionUtils.isNotEmpty(matchSuccessIdList)) {
|
|
|
+ matchSuccessList = aigcBaseMapper.getIdChannelByContentId(matchSuccessIdList);
|
|
|
+ }
|
|
|
+ List<String> matchIdList = longArticleBaseMapper.getMatchContentId(dateStart, dateEnd);
|
|
|
+ List<IdChannelDTO> matchList = new ArrayList<>();
|
|
|
+ if (CollectionUtils.isNotEmpty(matchIdList)) {
|
|
|
+ matchList = aigcBaseMapper.getIdChannelByContentId(matchIdList);
|
|
|
+ }
|
|
|
+ List<LongArticleTitleAudit> videoAuditPassList = longArticleTitleAuditRepository.getByStatusAndAuditTimestampBetween(
|
|
|
+ 1, start, end);
|
|
|
+ List<String> videoAuditPassIdList = videoAuditPassList.stream().map(LongArticleTitleAudit::getContentId).collect(Collectors.toList());
|
|
|
+ List<IdChannelDTO> videoAuditPassIdChannelList = new ArrayList<>();
|
|
|
+ if (CollectionUtils.isNotEmpty(videoAuditPassIdList)) {
|
|
|
+ videoAuditPassIdChannelList = aigcBaseMapper.getIdChannelByContentId(videoAuditPassIdList);
|
|
|
+ }
|
|
|
+ List<LongArticleTitleAudit> videoAuditList = longArticleTitleAuditRepository.getByAuditTimestampBetween(start, end);
|
|
|
+ List<String> videoAuditIdList = videoAuditList.stream().map(LongArticleTitleAudit::getContentId).collect(Collectors.toList());
|
|
|
+ List<IdChannelDTO> videoAuditIdChannelList = new ArrayList<>();
|
|
|
+ if (CollectionUtils.isNotEmpty(videoAuditIdList)) {
|
|
|
+ videoAuditIdChannelList = aigcBaseMapper.getIdChannelByContentId(videoAuditIdList);
|
|
|
+ }
|
|
|
+
|
|
|
+ ContentGroupFunnelExport item = ContentGroupFunnelExport.getArticleDefault();
|
|
|
+ item.setDateStr(dateStr);
|
|
|
+ item.setType(channel.getDescription() + "文章");
|
|
|
+ item.setPlanCrawlerCount(planCrawlerCount);
|
|
|
+ item.setCrawlerCount(crawlerCount);
|
|
|
+ item.setProduceCount(produceCount);
|
|
|
+ item.setProduceAuditPassCount(produceAuditPassCount);
|
|
|
+ item.setProduceAuditCount(produceAuditCount);
|
|
|
+ if (produceAuditCount > 0) {
|
|
|
+ item.setProduceAuditPassRate(produceAuditPassCount * 1.0 / produceAuditCount);
|
|
|
+ }
|
|
|
+ item.setMatchSuccessCount(matchSuccessList.stream().filter(o -> o.getChannel().equals(channel.getVal())).count());
|
|
|
+ item.setMatchCount(matchList.stream().filter(o -> o.getChannel().equals(channel.getVal())).count());
|
|
|
+ if (item.getMatchCount() > 0) {
|
|
|
+ item.setMatchSuccessRate(item.getMatchSuccessCount() * 1.0 / item.getMatchCount());
|
|
|
+ }
|
|
|
+ item.setVideoAuditPassCount(videoAuditPassIdChannelList.stream().filter(o -> o.getChannel().equals(channel.getVal())).count());
|
|
|
+ item.setVideoAuditCount(videoAuditIdChannelList.stream().filter(o -> o.getChannel().equals(channel.getVal())).count());
|
|
|
+ if (item.getVideoAuditCount() > 0) {
|
|
|
+ item.setVideoAuditPassRate(item.getVideoAuditPassCount() * 1.0 / item.getVideoAuditCount());
|
|
|
+ }
|
|
|
+ if (item.getProduceCount() > 0) {
|
|
|
+ item.setArticleAuditPassRate(item.getVideoAuditPassCount() * 1.0 / item.getProduceCount());
|
|
|
+ }
|
|
|
+ result.add(item);
|
|
|
+ }
|
|
|
+ // 视频
|
|
|
+ List<VideoPoolPlatformEnum> videoPoolType = Arrays.asList(VideoPoolPlatformEnum.GZH, VideoPoolPlatformEnum.HkSP, VideoPoolPlatformEnum.SPH);
|
|
|
+ for (VideoPoolPlatformEnum videoType : videoPoolType) {
|
|
|
+ List<PublishSingleVideoSource> videoPoolCrawlerCount = videoPoolRepository.getByCrawlerTimestampBetweenAndPlatform(
|
|
|
+ start / 1000, end / 1000, videoType.getPlatform());
|
|
|
+ List<PublishSingleVideoSource> videoPoolPQAuditCount = videoPoolRepository.getByAuditTimestampBetweenAndPlatform(
|
|
|
+ start / 1000, end / 1000, videoType.getPlatform());
|
|
|
+ List<PublishSingleVideoSource> videoPoolAuditList = videoPoolRepository.getByVideoPoolAuditTimestampBetweenAndPlatform(
|
|
|
+ start, end, videoType.getPlatform());
|
|
|
+
|
|
|
+ ContentGroupFunnelExport item = ContentGroupFunnelExport.getVideoDefault();
|
|
|
+ item.setDateStr(dateStr);
|
|
|
+ item.setType(videoType.getDescription());
|
|
|
+ item.setVideoPoolCrawlerCount((long) videoPoolCrawlerCount.size());
|
|
|
+ item.setVideoPoolPQAuditCount((long) videoPoolPQAuditCount.size());
|
|
|
+ long videoPoolPQAuditPassCount = videoPoolPQAuditCount.stream()
|
|
|
+ .filter(o -> o.getAuditStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
|
|
|
+ item.setVideoPoolPQAuditPassCount(videoPoolPQAuditPassCount);
|
|
|
+ if (item.getVideoPoolPQAuditCount() > 0) {
|
|
|
+ item.setVideoPoolPQAuditPassRate(videoPoolPQAuditPassCount * 1.0 / item.getVideoPoolPQAuditCount());
|
|
|
+ }
|
|
|
+ item.setVideoPoolAuditCount((long) videoPoolAuditList.size());
|
|
|
+ long videoPoolAuditPassCount = videoPoolAuditList.stream()
|
|
|
+ .filter(o -> o.getVideoPoolAuditStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
|
|
|
+ item.setVideoPoolAuditPassCount(videoPoolAuditPassCount);
|
|
|
+ if (item.getVideoPoolAuditCount() > 0) {
|
|
|
+ item.setVideoPoolAuditPassRate(videoPoolAuditPassCount * 1.0 / item.getVideoPoolAuditCount());
|
|
|
+ }
|
|
|
+ if (item.getVideoPoolCrawlerCount() > 0) {
|
|
|
+ item.setVideoPoolCrawlerAuditPassRate(videoPoolAuditPassCount * 1.0 / item.getVideoPoolCrawlerCount());
|
|
|
+ }
|
|
|
+ result.add(item);
|
|
|
+ }
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
}
|