|
@@ -5,12 +5,19 @@ import com.alibaba.fastjson.JSON;
|
|
|
import com.alibaba.fastjson.JSONArray;
|
|
|
import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
import com.google.common.collect.Lists;
|
|
|
+import com.tzld.longarticle.recommend.server.common.enums.AccountBusinessTypeEnum;
|
|
|
+import com.tzld.longarticle.recommend.server.model.MiniprogramTaskParam;
|
|
|
import com.tzld.longarticle.recommend.server.model.NewSortStrategyExport;
|
|
|
+import com.tzld.longarticle.recommend.server.model.ProduceAuditLayoutContentObjVO;
|
|
|
+import com.tzld.longarticle.recommend.server.model.PublishContentParam;
|
|
|
+import com.tzld.longarticle.recommend.server.repository.aigc.*;
|
|
|
import com.tzld.longarticle.recommend.server.repository.crawler.AccountAvgInfoRepository;
|
|
|
import com.tzld.longarticle.recommend.server.repository.crawler.ArticleDetailInfoRepository;
|
|
|
import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
|
|
|
import com.tzld.longarticle.recommend.server.repository.crawler.PublishSortLogRepository;
|
|
|
+import com.tzld.longarticle.recommend.server.repository.entity.aigc.*;
|
|
|
import com.tzld.longarticle.recommend.server.repository.entity.crawler.*;
|
|
|
+import com.tzld.longarticle.recommend.server.repository.mapper.aigc.AigcBaseMapper;
|
|
|
import com.tzld.longarticle.recommend.server.repository.mapper.crawler.CrawlerBaseMapper;
|
|
|
import com.tzld.longarticle.recommend.server.util.DateUtils;
|
|
|
import com.tzld.longarticle.recommend.server.util.MapBuilder;
|
|
@@ -46,6 +53,20 @@ public class DataDashboardService {
|
|
|
private PublishSortLogRepository publishSortLogRepository;
|
|
|
@Autowired
|
|
|
private CrawlerBaseMapper crawlerBaseMapper;
|
|
|
+ @Autowired
|
|
|
+ private AigcBaseMapper aigcBaseMapper;
|
|
|
+ @Autowired
|
|
|
+ private PublishAccountRepository publishAccountRepository;
|
|
|
+ @Autowired
|
|
|
+ private PublishContentLayOutRepository publishContentLayOutRepository;
|
|
|
+ @Autowired
|
|
|
+ private PublishPlanRepository publishPlanRepository;
|
|
|
+ @Autowired
|
|
|
+ private ProducePlanExeRecordRepository producePlanExeRecordRepository;
|
|
|
+ @Autowired
|
|
|
+ private ProducePlanRepository producePlanRepository;
|
|
|
+ @Autowired
|
|
|
+ private ProducePlanInputSourceRepository producePlanInputSourceRepository;
|
|
|
|
|
|
@ApolloJsonValue("${export.account.ghId:[]}")
|
|
|
private static List<String> ghIdList;
|
|
@@ -56,7 +77,7 @@ public class DataDashboardService {
|
|
|
exportFeishuNewSortStrategy(dateStrList, sheetToken, "7d4e12");
|
|
|
}
|
|
|
|
|
|
- @Scheduled(cron = "0 0 9 * * ?")
|
|
|
+ @Scheduled(cron = "0 0 4 * * ?")
|
|
|
public void scheduledExport() {
|
|
|
List<String> dateStrList = getBeforeThreeDays(null);
|
|
|
exportFeishuNewSortStrategy(dateStrList, sheetToken, "7d4e12");
|
|
@@ -114,15 +135,15 @@ public class DataDashboardService {
|
|
|
|
|
|
List<Pair<String, String>> styles = Arrays
|
|
|
.asList(
|
|
|
- Pair.of("R", "0.00%"),
|
|
|
Pair.of("S", "0.00%"),
|
|
|
Pair.of("T", "0.00%"),
|
|
|
Pair.of("U", "0.00%"),
|
|
|
Pair.of("V", "0.00%"),
|
|
|
Pair.of("W", "0.00%"),
|
|
|
- Pair.of("AC", "0.00%"),
|
|
|
+ Pair.of("X", "0.00%"),
|
|
|
Pair.of("AD", "0.00%"),
|
|
|
- Pair.of("AE", "0.00%")
|
|
|
+ Pair.of("AE", "0.00%"),
|
|
|
+ Pair.of("AF", "0.00%")
|
|
|
);
|
|
|
|
|
|
doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles);
|
|
@@ -130,23 +151,118 @@ public class DataDashboardService {
|
|
|
|
|
|
private List<NewSortStrategyExport> newSortStrategyData(String dateStr) {
|
|
|
long timestamp = DateUtils.dateStrToTimestamp(dateStr, "yyyyMMdd");
|
|
|
- List<AccountAvgInfo> accountAvgInfoList = accountAvgInfoRepository.getAllByStatusEquals(1);
|
|
|
+ String dateStrS = DateUtils.timestampToYMDStr(timestamp, "yyyy-MM-dd");
|
|
|
+ List<AccountAvgInfo> accountAvgInfoList = accountAvgInfoRepository.getAllByUpdateTimeGreaterThanEqual(dateStrS);
|
|
|
Set<String> ghIds = accountAvgInfoList.stream().map(AccountAvgInfo::getGhId).collect(Collectors.toSet());
|
|
|
- List<Article> articleList = articleRepository.getByGhIdInAndUpdateTimeGreaterThan(ghIds, timestamp);
|
|
|
+
|
|
|
+ List<Article> articleList = articleRepository.getByGhIdInAndUpdateTimeGreaterThanAndTypeEquals(ghIds, timestamp, "9");
|
|
|
Map<String, Map<String, Map<Integer, Article>>> articleMap = articleList.stream().collect(Collectors.groupingBy(Article::getGhId,
|
|
|
Collectors.groupingBy(Article::getAppMsgId, Collectors.toMap(Article::getItemIndex, o -> o))));
|
|
|
|
|
|
Set<String> snList = articleList.stream().map(Article::getWxSn).collect(Collectors.toSet());
|
|
|
- List<ArticleDetailInfo> articleDetailInfoList = articleDetailInfoRepository.getAllByWxSnIn(new ArrayList<>(snList));
|
|
|
+ List<ArticleDetailInfo> articleDetailInfoList = new ArrayList<>();
|
|
|
+ for (List<String> partitions : Lists.partition(new ArrayList<>(snList), 1000)) {
|
|
|
+ articleDetailInfoList.addAll(articleDetailInfoRepository.getAllByWxSnIn(partitions));
|
|
|
+ }
|
|
|
Map<String, List<ArticleDetailInfo>> articleDetailInfoMap = articleDetailInfoList.stream()
|
|
|
.collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
|
|
|
List<PublishSortLog> sortLogList = publishSortLogRepository.findByGhIdInAndDateStrGreaterThanEqual(ghIds, dateStr);
|
|
|
Map<String, Map<String, String>> sortStrategyMap = sortLogList.stream()
|
|
|
.collect(Collectors.groupingBy(PublishSortLog::getGhId,
|
|
|
Collectors.toMap(PublishSortLog::getDateStr, PublishSortLog::getStrategy, (existing, replacement) -> replacement)));
|
|
|
-
|
|
|
- Map<String, Map<String, AccountAvgInfo>> accountAvgInfoIndexMap = accountAvgInfoList.stream().collect(
|
|
|
- Collectors.groupingBy(AccountAvgInfo::getGhId, Collectors.toMap(AccountAvgInfo::getPosition, o -> o)));
|
|
|
+ Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap = accountAvgInfoList.stream().collect(
|
|
|
+ Collectors.groupingBy(AccountAvgInfo::getUpdateTime, Collectors.groupingBy(AccountAvgInfo::getGhId,
|
|
|
+ Collectors.toMap(AccountAvgInfo::getPosition, o -> o))));
|
|
|
+ // 获取发布账号
|
|
|
+ List<PublishAccount> publishAccountList = publishAccountRepository.getAllByGhIdIn(ghIds);
|
|
|
+ Map<String, PublishAccount> publishAccountMap = publishAccountList.stream().collect(Collectors.toMap(PublishAccount::getGhId, o -> o));
|
|
|
+ // 获取发布内容
|
|
|
+ List<PublishContentParam> publishContentParamList = articleList.stream().map(article -> {
|
|
|
+ PublishContentParam item = new PublishContentParam();
|
|
|
+ item.setTitle(article.getTitle());
|
|
|
+ PublishAccount account = publishAccountMap.get(article.getGhId());
|
|
|
+ if (Objects.nonNull(account)) {
|
|
|
+ item.setPublishAccountId(account.getId());
|
|
|
+ return item;
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }).filter(Objects::nonNull).collect(Collectors.toList());
|
|
|
+ List<PublishContent> publishContents = new ArrayList<>();
|
|
|
+ for (List<PublishContentParam> partitions : Lists.partition(publishContentParamList, 1000)) {
|
|
|
+ publishContents.addAll(aigcBaseMapper.getPublishContentByTitle(partitions));
|
|
|
+ }
|
|
|
+ Map<String, Map<String, PublishContent>> publishContentMap = publishContents.stream().collect(
|
|
|
+ Collectors.groupingBy(PublishContent::getPublishAccountId,
|
|
|
+ Collectors.toMap(PublishContent::getTitle, o -> o, (existing, replacement) -> existing)));
|
|
|
+ // 获取发布内容排版
|
|
|
+ List<String> publishContentIds = publishContents.stream().map(PublishContent::getId).collect(Collectors.toList());
|
|
|
+ List<PublishContentLayout> publishContentLayoutList = new ArrayList<>();
|
|
|
+ for (List<String> partitions : Lists.partition(publishContentIds, 1000)) {
|
|
|
+ publishContentLayoutList.addAll(publishContentLayOutRepository.findByPublishContentIdIn(partitions));
|
|
|
+ }
|
|
|
+ Map<String, PublishContentLayout> publishContentLayoutMap = publishContentLayoutList.stream()
|
|
|
+ .collect(Collectors.toMap(PublishContentLayout::getPublishContentId, o -> o,
|
|
|
+ (existing, replacement) -> replacement));
|
|
|
+ //获取发布计划
|
|
|
+ List<String> publishPlanIds = publishContents.stream().map(PublishContent::getPlanId).distinct()
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ List<PublishPlan> publishPlanList = publishPlanRepository.findByIdIn(publishPlanIds);
|
|
|
+ Map<String, PublishPlan> publishPlanMap = publishPlanList.stream()
|
|
|
+ .collect(Collectors.toMap(PublishPlan::getId, o -> o));
|
|
|
+ // 获取生成记录
|
|
|
+ List<String> contentSourceIds = publishContents.stream()
|
|
|
+ .filter(o -> Arrays.asList(1, 2).contains(o.getSourceType()))
|
|
|
+ .map(PublishContent::getSourceId).distinct().collect(Collectors.toList());
|
|
|
+ List<ProducePlanExeRecord> planExeRecordList = new ArrayList<>();
|
|
|
+ for (List<String> partitions : Lists.partition(contentSourceIds, 1000)) {
|
|
|
+ planExeRecordList.addAll(producePlanExeRecordRepository.findByPlanExeIdIn(partitions));
|
|
|
+ }
|
|
|
+ Map<String, ProducePlanExeRecord> planExeRecordMap = planExeRecordList.stream()
|
|
|
+ .collect(Collectors.toMap(ProducePlanExeRecord::getPlanExeId, o -> o));
|
|
|
+ // 获取生成计划
|
|
|
+ List<String> producePlanIds = planExeRecordList.stream().map(ProducePlanExeRecord::getPlanId).distinct()
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ List<ProducePlan> producePlanList = producePlanRepository.findByIdIn(producePlanIds);
|
|
|
+ Map<String, ProducePlan> producePlanMap = producePlanList.stream()
|
|
|
+ .collect(Collectors.toMap(ProducePlan::getId, o -> o));
|
|
|
+ // 获取生成计划输入
|
|
|
+ List<ProducePlanInputSource> inputSourceList = producePlanInputSourceRepository.findByPlanIdIn(producePlanIds);
|
|
|
+ Map<String, List<ProducePlanInputSource>> inputSourceMap = inputSourceList.stream()
|
|
|
+ .collect(Collectors.groupingBy(ProducePlanInputSource::getPlanId));
|
|
|
+ // 获取抓取内容关联
|
|
|
+ List<String> crawlerChannelContentIds = publishContents.stream().map(PublishContent::getCrawlerChannelContentId)
|
|
|
+ .distinct().collect(Collectors.toList());
|
|
|
+ List<CrawlerPlanResultRel> resultRelList = aigcBaseMapper.getCrawlerPlanRelByChannelContentIds(crawlerChannelContentIds);
|
|
|
+ Map<String, List<CrawlerPlanResultRel>> resultRelMap = resultRelList.stream()
|
|
|
+ .collect(Collectors.groupingBy(CrawlerPlanResultRel::getChannelSourceId));
|
|
|
+ // 获取抓取计划
|
|
|
+ List<String> crawlerPlanIds = resultRelList.stream().map(CrawlerPlanResultRel::getPlanId).distinct()
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ List<CrawlerPlan> crawlerPlanList = aigcBaseMapper.getCrawlerPlanByPlanIds(crawlerPlanIds);
|
|
|
+ Map<String, CrawlerPlan> crawlerPlanMap = crawlerPlanList.stream()
|
|
|
+ .collect(Collectors.toMap(CrawlerPlan::getId, o -> o));
|
|
|
+ // 获取小程序任务
|
|
|
+ List<MiniprogramTaskParam> miniprogramTaskParamList = new ArrayList<>();
|
|
|
+ Set<String> distinct = new HashSet<>();
|
|
|
+ for (PublishContent publishContent : publishContents) {
|
|
|
+ String key = publishContent.getPlanId() + publishContent.getPublishAccountId();
|
|
|
+ if (distinct.contains(key)) {
|
|
|
+ continue;
|
|
|
+ } else {
|
|
|
+ distinct.add(key);
|
|
|
+ }
|
|
|
+ MiniprogramTaskParam param = new MiniprogramTaskParam();
|
|
|
+ param.setAccountId(publishContent.getPublishAccountId());
|
|
|
+ param.setPlanId(publishContent.getPlanId());
|
|
|
+ miniprogramTaskParamList.add(param);
|
|
|
+ }
|
|
|
+ List<PublishPlanMiniprogramTask> miniprogramTaskList = new ArrayList<>();
|
|
|
+ for (List<MiniprogramTaskParam> partitions : Lists.partition(miniprogramTaskParamList, 1000)) {
|
|
|
+ miniprogramTaskList.addAll(aigcBaseMapper.getMiniProgramTask(partitions));
|
|
|
+ }
|
|
|
+ Map<String, List<PublishPlanMiniprogramTask>> miniprogramTaskMap = miniprogramTaskList.stream()
|
|
|
+ .collect(Collectors.groupingBy(PublishPlanMiniprogramTask::getPlanId));
|
|
|
+ // result
|
|
|
List<NewSortStrategyExport> result = new ArrayList<>();
|
|
|
for (Article article : articleList) {
|
|
|
List<ArticleDetailInfo> articleDetailInfos = articleDetailInfoMap.get(article.getWxSn());
|
|
@@ -168,7 +284,11 @@ public class DataDashboardService {
|
|
|
sumFission2 += Optional.ofNullable(articleDetailInfo.getFission2()).orElse(0);
|
|
|
}
|
|
|
}
|
|
|
- Map<String, AccountAvgInfo> accountAvgInfoMap = accountAvgInfoIndexMap.get(article.getGhId());
|
|
|
+ String hisPublishDate = DateUtils.timestampToYMDStr(article.getUpdateTime(), "yyyy-MM-dd");
|
|
|
+ List<String> avgMapDateList = new ArrayList<>(accountAvgInfoIndexMap.keySet());
|
|
|
+ hisPublishDate = DateUtils.findNearestDate(avgMapDateList, hisPublishDate, "yyyy-MM-dd");
|
|
|
+ Map<String, Map<String, AccountAvgInfo>> dateAvgMap = accountAvgInfoIndexMap.get(hisPublishDate);
|
|
|
+ Map<String, AccountAvgInfo> accountAvgInfoMap = dateAvgMap.get(article.getGhId());
|
|
|
AccountAvgInfo avgInfo = null;
|
|
|
AccountAvgInfo firstAvgInfo = null;
|
|
|
if (Objects.nonNull(accountAvgInfoMap)) {
|
|
@@ -204,6 +324,7 @@ public class DataDashboardService {
|
|
|
obj.setAccountSource(avgInfo.getAccountSource());
|
|
|
obj.setAccountType(avgInfo.getAccountType());
|
|
|
obj.setAccountStatus(avgInfo.getAccountStatus());
|
|
|
+ obj.setBusinessType(AccountBusinessTypeEnum.from(avgInfo.getBusinessType()).getDescription());
|
|
|
obj.setFans(avgInfo.getFans());
|
|
|
obj.setAvgViewCount(avgInfo.getReadAvg());
|
|
|
if (avgInfo.getReadAvg() > 0) {
|
|
@@ -226,6 +347,61 @@ public class DataDashboardService {
|
|
|
if (sumFission0 > 0) {
|
|
|
obj.setFission1Fission0Rate((sumFission1 * 1.0) / sumFission0);
|
|
|
}
|
|
|
+ // aigc 数据
|
|
|
+ PublishAccount publishAccount = publishAccountMap.get(article.getGhId());
|
|
|
+ Map<String, PublishContent> titleContentMap = publishContentMap.get(publishAccount.getId());
|
|
|
+ if (Objects.isNull(titleContentMap)) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ PublishContent publishContent = titleContentMap.get(article.getTitle());
|
|
|
+ if (Objects.isNull(publishContent)) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ PublishContentLayout layout = publishContentLayoutMap.get(publishContent.getId());
|
|
|
+ List<ProduceAuditLayoutContentObjVO> layoutList = JSON.parseArray(layout.getContent(), ProduceAuditLayoutContentObjVO.class);
|
|
|
+ if (CollectionUtil.isNotEmpty(layoutList)) {
|
|
|
+ long miniProgramNum = layoutList.stream().filter(o -> o.getType() == 4).count();
|
|
|
+ obj.setPublishMiniProgramNum((int) miniProgramNum);
|
|
|
+ }
|
|
|
+ PublishPlan publishPlan = publishPlanMap.get(publishContent.getPlanId());
|
|
|
+ obj.setPublishPlanName(publishPlan.getName());
|
|
|
+ // 插入小程序任务
|
|
|
+ List<PublishPlanMiniprogramTask> miniprogramTasks = miniprogramTaskMap.get(publishPlan.getId());
|
|
|
+ if (CollectionUtil.isNotEmpty(miniprogramTasks)) {
|
|
|
+ miniprogramTasks = miniprogramTasks.stream()
|
|
|
+ .filter(o -> o.getAccountIds().contains(publishAccount.getId()))
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ if (CollectionUtil.isNotEmpty(miniprogramTasks)) {
|
|
|
+ obj.setPublishMiniProgramInsertStrategy(miniprogramTasks.get(0).getStrategy());
|
|
|
+ obj.setPublishMiniProgramInsertUseType(miniprogramTasks.get(0).getMiniprogramUseType() == 1 ? "自然流" : "投流");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ List<ProducePlanInputSource> producePlanInputSourceList = new ArrayList<>();
|
|
|
+ if (Arrays.asList(1, 2).contains(publishContent.getSourceType())) {
|
|
|
+ ProducePlanExeRecord record = planExeRecordMap.get(publishContent.getSourceId());
|
|
|
+ if (Objects.nonNull(record)) {
|
|
|
+ ProducePlan producePlan = producePlanMap.get(record.getPlanId());
|
|
|
+ obj.setProducePlanName(producePlan.getName());
|
|
|
+ obj.setProducePlanTag(producePlan.getPlanTag());
|
|
|
+ }
|
|
|
+ producePlanInputSourceList = inputSourceMap.get(record.getPlanId());
|
|
|
+ }
|
|
|
+ List<CrawlerPlanResultRel> crawlerPlanRelList = resultRelMap.get(publishContent.getCrawlerChannelContentId());
|
|
|
+ if (CollectionUtil.isNotEmpty(crawlerPlanRelList) && CollectionUtil.isNotEmpty(producePlanInputSourceList)) {
|
|
|
+ List<String> inputSourceValues = producePlanInputSourceList.stream()
|
|
|
+ .map(ProducePlanInputSource::getInputSourceValue).collect(Collectors.toList());
|
|
|
+ List<CrawlerPlan> crawlerPlanItemList = new ArrayList<>();
|
|
|
+ for (CrawlerPlanResultRel crawlerPlanResultRel : crawlerPlanRelList) {
|
|
|
+ crawlerPlanItemList.add(crawlerPlanMap.get(crawlerPlanResultRel.getPlanId()));
|
|
|
+ }
|
|
|
+ for (CrawlerPlan crawlerPlan : crawlerPlanItemList) {
|
|
|
+ if (inputSourceValues.contains(crawlerPlan.getId())) {
|
|
|
+ obj.setCrawlerPlanName(crawlerPlan.getName());
|
|
|
+ obj.setCrawlerPlanTag(crawlerPlan.getPlanTag());
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
result.add(obj);
|
|
|
}
|
|
|
result.sort(Comparator.comparing(NewSortStrategyExport::getDateStr).reversed()
|
|
@@ -257,8 +433,8 @@ public class DataDashboardService {
|
|
|
// 先删除掉已存在的dateStr数据
|
|
|
HttpEntity<Object> queryEntity = new HttpEntity<>(httpHeaders);
|
|
|
ResponseEntity<String> queryResponseEntity = restTemplate.exchange(
|
|
|
- String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/values/%s!A" + startRowIndex + ":A" + (startRowIndex + (rowNum * 2)),
|
|
|
- sheetToken, sheetId),
|
|
|
+ String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/values/%s!A"
|
|
|
+ + startRowIndex + ":A" + (startRowIndex + (rowNum * 2)), sheetToken, sheetId),
|
|
|
HttpMethod.GET, queryEntity, String.class);
|
|
|
JSONArray values = JSON.parseObject(queryResponseEntity.getBody())
|
|
|
.getJSONObject("data")
|
|
@@ -327,6 +503,11 @@ public class DataDashboardService {
|
|
|
// HttpMethod.POST, mergeEntity, String.class);
|
|
|
// }
|
|
|
// 此处先简单处理,调整单元格为”百分比小数点“
|
|
|
+ try {
|
|
|
+ Thread.sleep(1000);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("export sleep error");
|
|
|
+ }
|
|
|
if (CollectionUtil.isNotEmpty(styles)) {
|
|
|
startRow = startRowIndex;
|
|
|
do {
|