Merge branch '20250530-wyp-contentPlatformDatastat' into test

api-module/src/main/java/com/tzld/piaoquan/api/job/ContentPlatformVideoJob.java

@@ -21,6 +21,8 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.springframework.util.StringUtils;
 
+import java.math.BigDecimal;
+import java.math.RoundingMode;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -196,7 +198,7 @@ public class ContentPlatformVideoJob {
         if (StringUtils.hasText(param)) {
             aggDt = param;
         }
-        List<String> dtList = DateUtil.getBeforeDays(aggDt, null, 7);
+        List<String> dtList = DateUtil.getBeforeDays(aggDt, aggDt, 6);
         // 轮询查询大数据获取最近14天视频
         for (String dt : dtList) {
             long videoGroupScoreCount = getVideoGroupScoreCount(dt);
@@ -212,8 +214,8 @@ public class ContentPlatformVideoJob {
                 List<ContentPlatformVideoDataStat> saveList = new ArrayList<>();
                 for (Record record : dataList) {
                     ContentPlatformVideoDataStat item = new ContentPlatformVideoDataStat();
-                    String type =(String) record.get(0);
-                    String channel =(String) record.get(1);
+                    String type = (String) record.get(0);
+                    String channel = (String) record.get(1);
                     Long videoId = Long.parseLong((String) record.get(2));
                     Long firstLevel = Long.parseLong((String) record.get(3));
                     Long fission0 = Long.parseLong((String) record.get(4));
@@ -251,14 +253,55 @@ public class ContentPlatformVideoJob {
 
     private List<ContentPlatformVideoDataStatAgg> buildVideoDataStatAggList(String aggDt, List<String> dtList) {
         Long now = System.currentTimeMillis();
+        List<ContentPlatformVideoDataStat> dataStatList = getVideoDatastatList(dtList);
+        Map<Long, Map<String, Map<String, List<ContentPlatformVideoDataStat>>>> videoIdMap = dataStatList.stream()
+                .filter(o -> {
+                    if ("sum".equals(o.getChannel())) {
+                        return o.getFirstLevel() > 300;
+                    } else {
+                        return o.getFirstLevel() > 100;
+                    }
+                })
+                .collect(Collectors.groupingBy(ContentPlatformVideoDataStat::getVideoId,
+                        Collectors.groupingBy(ContentPlatformVideoDataStat::getType,
+                                Collectors.groupingBy(ContentPlatformVideoDataStat::getChannel))));
         List<ContentPlatformVideoDataStatAgg> result = new ArrayList<>();
-        for (ContentPlatformVideoDataStatAgg item : result) {
-            item.setDt(aggDt);
-            item.setCreateTimestamp(now);
+        for (Map.Entry<Long, Map<String, Map<String, List<ContentPlatformVideoDataStat>>>> entry : videoIdMap.entrySet()) {
+            Long videoId = entry.getKey();
+            Map<String, Map<String, List<ContentPlatformVideoDataStat>>> typeMap = entry.getValue();
+            for (Map.Entry<String, Map<String, List<ContentPlatformVideoDataStat>>> typeEntry : typeMap.entrySet()) {
+                String type = typeEntry.getKey();
+                Map<String, List<ContentPlatformVideoDataStat>> channelMap = typeEntry.getValue();
+                for (Map.Entry<String, List<ContentPlatformVideoDataStat>> channelEntry : channelMap.entrySet()) {
+                    String channel = channelEntry.getKey();
+                    List<ContentPlatformVideoDataStat> statList = channelEntry.getValue();
+                    Long sumFirstLevel = statList.stream().map(ContentPlatformVideoDataStat::getFirstLevel).reduce(0L, Long::sum);
+                    Long sumFission0 = statList.stream().map(ContentPlatformVideoDataStat::getFission0).reduce(0L, Long::sum);
+                    ContentPlatformVideoDataStatAgg item = new ContentPlatformVideoDataStatAgg();
+                    item.setDt(aggDt);
+                    item.setVideoId(videoId);
+                    item.setType(type);
+                    item.setChannel(channel);
+                    item.setFirstLevel(sumFirstLevel);
+                    item.setFission0(sumFission0);
+                    if (sumFirstLevel > 0) {
+                        BigDecimal fissionRate = new BigDecimal(sumFission0).divide(new BigDecimal(sumFirstLevel), 4, RoundingMode.HALF_UP);
+                        item.setFissionRate(fissionRate.doubleValue());
+                    }
+                    item.setCreateTimestamp(now);
+                    result.add(item);
+                }
+            }
         }
         return result;
     }
 
+    private List<ContentPlatformVideoDataStat> getVideoDatastatList(List<String> dtList) {
+        ContentPlatformVideoDataStatExample example = new ContentPlatformVideoDataStatExample();
+        example.createCriteria().andDtIn(dtList);
+        return videoDataStatMapper.selectByExample(example);
+    }
+
 }
 
 

api-module/src/main/java/com/tzld/piaoquan/api/service/contentplatform/impl/ContentPlatformPlanServiceImpl.java

@@ -340,6 +340,12 @@ public class ContentPlatformPlanServiceImpl implements ContentPlatformPlanServic
             }
         }
         for (GzhPlanVideoContentItemParam vo : param.getVideoList()) {
+            if (!StringUtils.hasText(vo.getCustomTitle())) {
+                vo.setCustomTitle(null);
+            }
+            if (!StringUtils.hasText(vo.getCustomCover())) {
+                vo.setCustomCover(null);
+            }
             if (existsVideoIds.contains(vo.getVideoId())) {
                 ContentPlatformGzhPlanVideo item = existsVideoMap.get(vo.getVideoId());
                 item.setTitle(vo.getTitle());
@@ -367,7 +373,7 @@ public class ContentPlatformPlanServiceImpl implements ContentPlatformPlanServic
                 }
                 item.setCreateAccountId(loginAccount.getId());
                 item.setCreateTimestamp(System.currentTimeMillis());
-                gzhPlanVideoMapper.insertSelective(item);
+                gzhPlanVideoMapper.insert(item);
             }
         }
         if (CollectionUtils.isNotEmpty(param.getVideoList())) {
@@ -463,7 +469,7 @@ public class ContentPlatformPlanServiceImpl implements ContentPlatformPlanServic
         String type = getVideoContentListType(param.getType());
         String channel = getVideoContentListChannel(param.getSort(), user.getChannel());
         List<ContentPlatformVideo> videoList = planMapperExt.getVideoList(param, dt, datastatDt, type, channel, videoMinScore, offset, param.getPageSize(), sort);
-        List<VideoContentItemVO> list = buildVideoContentItemVOList(videoList, type, user.getChannel(), datastatDt);
+        List<VideoContentItemVO> list = buildVideoContentItemVOList(videoList, type, channel, user.getChannel(), datastatDt);
         result.setObjs(list);
         return result;
     }
@@ -504,7 +510,7 @@ public class ContentPlatformPlanServiceImpl implements ContentPlatformPlanServic
     }
 
     private List<VideoContentItemVO> buildVideoContentItemVOList(List<ContentPlatformVideo> videoList,
-                                                                 String type, String channel, String datastatDt) {
+                                                                 String type, String channel, String userChannel, String datastatDt) {
         if (CollectionUtils.isEmpty(videoList)) {
             return null;
         }
@@ -518,10 +524,10 @@ public class ContentPlatformPlanServiceImpl implements ContentPlatformPlanServic
             Set<Long> ids = new HashSet<>(partition);
             coverMap.putAll(messageAttachmentService.getVideoDetail(ids));
         }
-        List<ContentPlatformVideoDataStatAgg> typeVideoDataStatAggList = getTypeVideoDataStatAggList(datastatDt, type, videoIds);
+        List<ContentPlatformVideoDataStatAgg> typeVideoDataStatAggList = getTypeChannelVideoDataStatAggList(datastatDt, type, channel, videoIds);
         Map<Long, Double> typeVideoScoreMap = typeVideoDataStatAggList.stream().collect(Collectors.toMap(
                 ContentPlatformVideoDataStatAgg::getVideoId, ContentPlatformVideoDataStatAgg::getFissionRate));
-        List<ContentPlatformVideoDataStatAgg> channelVideoDataStatAggList = getChannelVideoDataStatAggList(datastatDt, channel, videoIds);
+        List<ContentPlatformVideoDataStatAgg> channelVideoDataStatAggList = getTypeChannelVideoDataStatAggList(datastatDt, type, userChannel, videoIds);
         Map<Long, Double> channelVideoScoreMap = channelVideoDataStatAggList.stream().collect(Collectors.toMap(
                 ContentPlatformVideoDataStatAgg::getVideoId, ContentPlatformVideoDataStatAgg::getFissionRate));
         List<VideoContentItemVO> result = new ArrayList<>();
@@ -544,15 +550,9 @@ public class ContentPlatformPlanServiceImpl implements ContentPlatformPlanServic
         return result;
     }
 
-    private List<ContentPlatformVideoDataStatAgg> getChannelVideoDataStatAggList(String datastatDt, String channel, List<Long> videoIds) {
-        ContentPlatformVideoDataStatAggExample example = new ContentPlatformVideoDataStatAggExample();
-        example.createCriteria().andVideoIdIn(videoIds).andDtEqualTo(datastatDt).andChannelEqualTo(channel);
-        return videoDataStatAggMapper.selectByExample(example);
-    }
-
-    private List<ContentPlatformVideoDataStatAgg> getTypeVideoDataStatAggList(String datastatDt, String type, List<Long> videoIds) {
+    private List<ContentPlatformVideoDataStatAgg> getTypeChannelVideoDataStatAggList(String datastatDt, String type, String channel, List<Long> videoIds) {
         ContentPlatformVideoDataStatAggExample example = new ContentPlatformVideoDataStatAggExample();
-        example.createCriteria().andVideoIdIn(videoIds).andDtEqualTo(datastatDt).andTypeEqualTo(type);
+        example.createCriteria().andVideoIdIn(videoIds).andDtEqualTo(datastatDt).andTypeEqualTo(type).andChannelEqualTo(channel);
         return videoDataStatAggMapper.selectByExample(example);
     }
 

common-module/src/main/java/com/tzld/piaoquan/growth/common/component/ProxyHttpPoolClient.java

@@ -71,24 +71,31 @@ public class ProxyHttpPoolClient {
     }
 
     public String request(HttpRequestBase request) throws IOException {
-
-        HttpEntity entity = null;
-        CloseableHttpResponse response = request((HttpUriRequest) request);
-        if (response == null) {
-            log.error("call api exception no response");
-            throw new RuntimeException("call api exception no response");
-        }
-        entity = response.getEntity();
-        String content = null;
-        if (entity != null) {
-            content = EntityUtils.toString(entity, "UTF-8");
-        }
-        int httpStatus = response.getStatusLine().getStatusCode();
-        if (httpStatus == HttpStatus.SC_OK) {
-            return content;
+        int retry = 0;
+        while (retry <= 3) {
+            try {
+                retry++;
+                HttpEntity entity = null;
+                CloseableHttpResponse response = request((HttpUriRequest) request);
+                if (response == null) {
+                    log.error("call api exception no response");
+                    throw new RuntimeException("call api exception no response");
+                }
+                entity = response.getEntity();
+                String content = null;
+                if (entity != null) {
+                    content = EntityUtils.toString(entity, "UTF-8");
+                }
+                int httpStatus = response.getStatusLine().getStatusCode();
+                if (httpStatus == HttpStatus.SC_OK) {
+                    return content;
+                }
+                String path = request.getURI().toString();
+                log.error("http call api {} fail response status {} content {}", path, httpStatus, content);
+            } catch (Exception e) {
+                log.error("request error", e);
+            }
         }
-        String path = request.getURI().toString();
-        log.error("http call api {} fail response status {} content {}", path, httpStatus, content);
         return null;
     }
 

offline-module/src/main/java/com/tzld/piaoquan/offline/job/WeComHistoryDataJob.java

@@ -91,41 +91,39 @@ public class WeComHistoryDataJob {
     @Autowired
     private WeComSendService weComSendService;
 
-    private static final String START_TIME_KEY = "start_time_key";
+    private static final String START_TIME_KEY = "start_time_key_%s";
 
     @XxlJob("saveHistoryMessageJob1")
     public ReturnT<String> saveHistoryMessageJob1(String param) {
-        Integer startTimeVal = (Integer) redisTemplate.opsForValue().get(START_TIME_KEY);
-        Long dayStartTime = DateUtil.getBeforeDayStart(0);
-        Long startTime;
-        if (startTimeVal == null) {
-            startTime = DateUtil.getBeforeDayStart(0);
-        } else {
-            if (dayStartTime > startTimeVal) {
-                startTime = dayStartTime;
-            } else {
-                startTime = (long) startTimeVal;
-            }
-        }
-
-        Long endTime = System.currentTimeMillis() / 1000;
         CorpExample example = new CorpExample();
         List<Corp> corps = corpMapper.selectByExample(example);
         if (CollectionUtils.isEmpty(corps)) {
             return ReturnT.SUCCESS;
         }
         for (Corp corp : corps) {
+            String key = String.format(START_TIME_KEY, corp.getId());
+            Integer startTimeVal = (Integer) redisTemplate.opsForValue().get(key);
+            Long dayStartTime = DateUtil.getBeforeDayStart(0);
+            Long startTime;
+            if (startTimeVal == null) {
+                startTime = DateUtil.getBeforeDayStart(0);
+            } else {
+                if (dayStartTime > startTimeVal) {
+                    startTime = dayStartTime;
+                } else {
+                    startTime = (long) startTimeVal;
+                }
+            }
+            Long endTime = System.currentTimeMillis() / 1000;
             selectGroupMsgList(startTime, endTime, corp.getId(), null);
-        }
-        boolean isAllSend = true;
-        for (Corp corp : corps) {
+            boolean isAllSend = true;
             long notSentCount = newSelectAlertHistoryMessage(startTime, endTime + TimeConstant.HOUR, corp.getId(), corp.getName());
             if (notSentCount > 0) {
                 isAllSend = false;
             }
-        }
-        if (isAllSend) {
-            redisTemplate.opsForValue().set(START_TIME_KEY, endTime, 1, TimeUnit.DAYS);
+            if (isAllSend) {
+                redisTemplate.opsForValue().set(key, endTime, 1, TimeUnit.DAYS);
+            }
         }
         return ReturnT.SUCCESS;
     }
@@ -177,7 +175,7 @@ public class WeComHistoryDataJob {
                 LarkRobotUtil.sendMessage(sendDetail.getRemark() + "存在未发送记录，请检查");
                 LarkRobotUtil.sendTipMessage(sendDetail.getRemark() + "存在未发送记录，请检查");
                 //8点前报警
-                if (DateUtil.getHourOfDay() < 8) {
+                if (DateUtil.getHourOfDay() < 8 && corpId == 1) {
                     LarkRobotUtil.sendNotPushMessage("<at user_id=\"all\">所有人</at> " + sendDetail.getRemark() + " 存在未发送记录，请检查");
                 }
             }
@@ -482,6 +480,9 @@ public class WeComHistoryDataJob {
         String cursor = "";
         do {
             String res = getGroupMsgTask(msgId, cursor, corpId);
+            if (StringUtils.isEmpty(res)) {
+                break;
+            }
             JSONObject jsonObject = JSONObject.parseObject(res);
             Integer errCode = jsonObject.getInteger("errcode");
             if (errCode != 0) {
@@ -645,8 +646,8 @@ public class WeComHistoryDataJob {
                 + "?access_token=" + accessToken;
         JSONObject param = new JSONObject();
         param.put("chat_type", "single");
-        param.put("start_time", startTime);
-        param.put("end_time", endTime);
+        param.put("start_time", String.valueOf(startTime));
+        param.put("end_time", String.valueOf(endTime));
         param.put("limit", 100);
         if (StringUtils.isNotEmpty(cursor)) {
             param.put("cursor", cursor);