fix qw datastat

api-module/src/main/java/com/tzld/piaoquan/api/job/contentplatform/ContentPlatformDatastatJob.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.api.job.contentplatform;
 import com.alibaba.fastjson.JSONObject;
 import com.aliyun.odps.data.Record;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.collect.Lists;
 import com.tzld.piaoquan.api.common.enums.contentplatform.BussinessTypeEnum;
 import com.tzld.piaoquan.api.component.AigcApiService;
 import com.tzld.piaoquan.api.dao.mapper.contentplatform.*;
@@ -585,17 +586,22 @@ public class ContentPlatformDatastatJob {
         }
         Map<Long, ContentPlatformQwPlan> planMap = qwPlanList.stream()
                 .collect(Collectors.toMap(ContentPlatformQwPlan::getId, plan -> plan));
-        List<ContentPlatformQwDataStat> existList = getQwDatastatCount(dt);
-        List<String> existRootSourceIds = existList.stream().map(ContentPlatformQwDataStat::getRootSourceId)
-                .collect(Collectors.toList());
         Map<String, Long> rootSourceIdMap = qwPlanList.stream()
                 .collect(Collectors.toMap(ContentPlatformQwPlan::getRootSourceId, ContentPlatformQwPlan::getId));
         List<Long> planIds = qwPlanList.stream().map(ContentPlatformQwPlan::getId).collect(Collectors.toList());
-        List<ContentPlatformQwPlanVideo> planVideoList = planService.getQwPlanVideoList(planIds);
+        List<ContentPlatformQwPlanVideo> planVideoList = new ArrayList<>();
+        List<List<Long>> partitionList = Lists.partition(planIds, 2000);
+        for (List<Long> partition : partitionList) {
+            planVideoList.addAll(planService.getQwPlanVideoList(partition));
+        }
         Map<Long, Long> planVideoMap = planVideoList.stream()
                 .collect(Collectors.toMap(ContentPlatformQwPlanVideo::getPlanId, ContentPlatformQwPlanVideo::getVideoId));
         List<Long> videoIds = planVideoList.stream().map(ContentPlatformQwPlanVideo::getVideoId).collect(Collectors.toList());
-        List<ContentPlatformVideoAgg> videoList = planService.getVideoContentAggListByVideoIds(videoIds);
+        List<ContentPlatformVideoAgg> videoList = new ArrayList<>();
+        List<List<Long>> videoIdPartitionList = Lists.partition(videoIds, 2000);
+        for (List<Long> partition : videoIdPartitionList) {
+            videoList.addAll(planService.getVideoContentAggListByVideoIds(partition));
+        }
         Map<Long, Double> videoScoreMap = videoList.stream()
                 .collect(Collectors.toMap(ContentPlatformVideoAgg::getVideoId, ContentPlatformVideoAgg::getScore, (a, b) -> a));
         List<ContentPlatformQwDataStat> saveList = new ArrayList<>();
@@ -603,6 +609,7 @@ public class ContentPlatformDatastatJob {
         String outSql = String.format("SELECT * FROM loghubods.qw_out_touliu_behavior_detail WHERE dt=%s;", dt);
         List<Record> outDataList = OdpsUtil.getOdpsData(outSql);
         Long now = System.currentTimeMillis();
+        List<String> existRootSourceIds = new ArrayList<>();
         if (CollectionUtils.isNotEmpty(outDataList)) {
             for (Record record : outDataList) {
                 ContentPlatformQwDataStat item = new ContentPlatformQwDataStat();
@@ -632,6 +639,7 @@ public class ContentPlatformDatastatJob {
                 item.setFirstLevelCount(firstLevelCount);
                 item.setCreateTimestamp(now);
                 saveList.add(item);
+                existRootSourceIds.add(rootSourceId);
             }
         }
         String out2Sql = String.format("SELECT * FROM loghubods.qw_out2_touliu_behavior_detail WHERE dt=%s;", dt);