中间指标导出

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/PublishSortLogRepository.java

@@ -12,6 +12,8 @@ public interface PublishSortLogRepository extends JpaRepository<PublishSortLog,
 
     List<PublishSortLog> findByDateStr(String dateStr);
 
+    List<PublishSortLog> findByDateStrGreaterThanEqual(String dateStr);
+
     List<PublishSortLog> findByDateStrAndTitleIn(String dateStr, List<String> titles);
 
     List<PublishSortLog> findByCrawlerChannelContentIdIn(List<String> crawlerChannelContentIds);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/DataDashboardService.java

@@ -441,16 +441,18 @@ public class DataDashboardService {
 
         // 先删除掉已存在的dateStr数据
         HttpEntity<Object> queryEntity = new HttpEntity<>(httpHeaders);
+        int deleteRowNum = rowNum < 20 ? startRowIndex + dateStrList.size() * rowNum : startRowIndex + (rowNum * 2);
         ResponseEntity<String> queryResponseEntity = restTemplate.exchange(
                 String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/values/%s!A"
-                        + startRowIndex + ":A" + (startRowIndex + (rowNum * 2)), sheetToken, sheetId),
+                        + startRowIndex + ":A" + deleteRowNum, sheetToken, sheetId),
                 HttpMethod.GET, queryEntity, String.class);
         JSONArray values = JSON.parseObject(queryResponseEntity.getBody())
                 .getJSONObject("data")
                 .getJSONObject("valueRange")
                 .getJSONArray("values");
         int count = 0;
-        if (!values.isEmpty() && Objects.nonNull(values.get(0))
+        if (!values.isEmpty() && Objects.nonNull(values.get(0)) && !((JSONArray) values.get(0)).isEmpty()
+                && Objects.nonNull(((JSONArray) values.get(0)).get(0))
                 && dateStrList.contains(((JSONArray) values.get(0)).get(0).toString())) {
             for (Object value : values) {
                 if (((JSONArray) value).get(0) != null) {
@@ -555,23 +557,27 @@ public class DataDashboardService {
     public void scheduleIntermediateIndicatorsExport() {
         List<String> dateStrList = DateUtils.getBeforeDays(null, 3);
         for (String date : dateStrList) {
-            exportFeishuIntermediateIndicators(date, sheetToken, "OuaLWV");
+            exportFeishuIntermediateIndicators(date, dateStrList, sheetToken, "OuaLWV");
         }
     }
 
     public void intermediateIndicatorsExport(String dateStr) {
         List<String> dateStrList = DateUtils.getBeforeDays(dateStr, 3);
         for (String date : dateStrList) {
-            exportFeishuIntermediateIndicators(date, sheetToken, "OuaLWV");
+            exportFeishuIntermediateIndicators(date, dateStrList, sheetToken, "OuaLWV");
         }
     }
 
-    private void exportFeishuIntermediateIndicators(String dateStr, String sheetToken, String sheetId) {
+    private void exportFeishuIntermediateIndicators(String dateStr, List<String> dateStrList, String sheetToken, String sheetId) {
         List<IntermediateIndicatorsExport> newContentsYesData = intermediateIndicatorsData(dateStr);
         if (CollectionUtil.isEmpty(newContentsYesData)) {
             return;
         }
-
+        long dateStrLong = DateUtils.dateStrToTimestamp(dateStr, "yyyyMMdd");
+        List<String> dateStrs = dateStrList.stream().filter(o -> {
+            long date = DateUtils.dateStrToTimestamp(o, "yyyyMMdd");
+            return date >= dateStrLong;
+        }).collect(Collectors.toList());
         int rowNum = newContentsYesData.size();
         List<List<Object>> rows = new ArrayList<>();
         Field[] fields = IntermediateIndicatorsExport.class.getDeclaredFields();
@@ -611,7 +617,7 @@ public class DataDashboardService {
                         Pair.of("AQ", "0.00%")
                 );
 
-        doSendFeishuSheet(Collections.singletonList(dateStr), sheetToken, sheetId, rowNum, rows, 3, styles);
+        doSendFeishuSheet(dateStrs, sheetToken, sheetId, rowNum, rows, 3, styles);
     }
 
     private List<IntermediateIndicatorsExport> intermediateIndicatorsData(String dateStr) {
@@ -642,11 +648,12 @@ public class DataDashboardService {
         }
         Map<String, List<ArticleDetailInfo>> articleDetailInfoMap = articleDetailInfoList.stream()
                 .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
-        List<PublishSortLog> publishSortLogList = publishSortLogRepository.findByDateStr(dateStr);
-        Map<String, Map<Integer, PublishSortLog>> publishSortLogMap = publishSortLogList.stream()
-                .collect(Collectors.groupingBy(PublishSortLog::getGhId, Collectors.toMap(
-                        PublishSortLog::getIndex, o -> o, (existing, replacement) -> replacement
-                )));
+        List<PublishSortLog> publishSortLogList = publishSortLogRepository.findByDateStrGreaterThanEqual(dateStr);
+        Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap = publishSortLogList.stream()
+                .collect(Collectors.groupingBy(PublishSortLog::getGhId,
+                        Collectors.groupingBy(PublishSortLog::getDateStr, Collectors.toMap(
+                                PublishSortLog::getIndex, o -> o, (existing, replacement) -> replacement
+                        ))));
         List<NewSortStrategyExport> newSortStrategyExportList = getNewSortStrategyExportList(todayPublish,
                 articleDetailInfoMap, accountAvgInfoIndexMap);
 
@@ -654,6 +661,7 @@ public class DataDashboardService {
         Map<String, List<Article>> futurePublishMap = futurePublishList.stream().collect(Collectors.groupingBy(Article::getTitle));
 
         Map<String, List<String>> titleTypeMap = new HashMap<>();
+        Map<String, Integer> poolHisPublishTimeMap = new HashMap<>();
         Map<String, Map<Integer, List<String>>> titleTypePoolMap = new HashMap<>();
         for (NewSortStrategyExport data : newSortStrategyExportList) {
             String type = getArticleType(data, small);
@@ -672,9 +680,9 @@ public class DataDashboardService {
             // 发布表现
             setPublishPerformance(item, data, publishSortLogMap);
             // 发布依赖表现
-            setPublishSourcePerformance(item, accountAvgInfoIndexMap, articleDetailInfoMap, list, small);
+            setPublishSourcePerformance(item, accountAvgInfoIndexMap, articleDetailInfoMap, publishSortLogMap, poolHisPublishTimeMap, type, list, small);
             // 发布未来表现 todo
-            setPublishFuturePerformance(item, data, poolLevel, futurePublishMap, small);
+//            setPublishFuturePerformance(item, data, poolLevel, futurePublishMap, publishSortLogMap, small);
 
             titles.add(data.getTitle());
             poolTitles.add(data.getTitle());
@@ -684,12 +692,14 @@ public class DataDashboardService {
     }
 
     private void setPublishFuturePerformance(IntermediateIndicatorsExport item, NewSortStrategyExport data, Integer poolLevel,
-                                             Map<String, List<Article>> futurePublishMap, List<String> small) {
+                                             Map<String, List<Article>> futurePublishMap,
+                                             Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap,
+                                             List<String> small) {
         List<Article> futurePublishList = futurePublishMap.get(data.getTitle());
-//        Integer futurePoolLevel = getArticlePoolLevel(data, futurePublishList, small);
-//        if (futurePoolLevel > poolLevel) {
-//
-//        }
+        Integer futurePoolLevel = getArticlePoolLevel(data, futurePublishList, small);
+        if (futurePoolLevel > poolLevel) {
+
+        }
         item.setPromotionCountL2(item.getPromotionCountL2());
         item.setPromotionCountL3(item.getPromotionCountL3());
         item.setPromotionCountL4(item.getPromotionCountL4());
@@ -814,8 +824,13 @@ public class DataDashboardService {
     private void setPublishSourcePerformance(IntermediateIndicatorsExport item,
                                              Map<String, Map<String, Map<String, AccountAvgInfo>>> accountAvgInfoIndexMap,
                                              Map<String, List<ArticleDetailInfo>> articleDetailInfoMap,
+                                             Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap,
+                                             Map<String, Integer> poolHisPublishTimeMap, String type,
                                              List<Article> hisPublish, List<String> small) {
+        Long hisMinDate = hisPublish.stream().mapToLong(Article::getUpdateTime).min().orElse(0);
+        Integer publishTime = poolHisPublishTimeMap.computeIfAbsent(type, k -> 0);
         for (Article publish : hisPublish) {
+            publishTime++;
             Map<String, Map<String, AccountAvgInfo>> dateAvgMap = accountAvgInfoIndexMap.get(publish.getGhId());
             String hisPublishDate = DateUtils.timestampToYMDStr(publish.getUpdateTime(), "yyyy-MM-dd");
             AccountAvgInfo avgInfo = null;
@@ -861,14 +876,34 @@ public class DataDashboardService {
             if (item.getRedundantViewBase() > 0 && item.getRedundantT0FissionUV() > 0) {
                 item.setAverageFirstLayerFissionToViewBaseRatio(item.getRedundantT0FissionUV() / (double) item.getRedundantViewBase());
             }
-            // todo
-            item.setFirstExplorationIntervalAvg(item.getFirstExplorationIntervalAvg());
-            item.setRedundantAccountArticleRelevanceAvg(item.getRedundantAccountArticleRelevanceAvg());
+            Map<String, Map<Integer, PublishSortLog>> dateSortMap = publishSortLogMap.get(publish.getGhId());
+            if (Objects.nonNull(dateSortMap)) {
+                String publishDateStr = DateUtils.timestampToYMDStr(publish.getUpdateTime(), "yyyyMMdd");
+                Map<Integer, PublishSortLog> indexMap = dateSortMap.get(publishDateStr);
+                if (Objects.nonNull(indexMap)) {
+                    PublishSortLog log = indexMap.get(publish.getItemIndex());
+                    double scoreAvg = item.getRedundantAccountArticleRelevanceAvg();
+                    double score = 0.0;
+                    if (Objects.nonNull(log) && Objects.nonNull(log.getScore())) {
+                        JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
+                        if (scoreMap.containsKey("SimilarityStrategy")) {
+                            score = scoreMap.getDoubleValue("SimilarityStrategy");
+                        }
+                    }
+                    scoreAvg = (scoreAvg * (publishTime - 1) + score) / publishTime;
+                    item.setRedundantAccountArticleRelevanceAvg(scoreAvg);
+                }
+            }
+            double firstExplorationIntervalAvg = Double.isNaN(item.getFirstExplorationIntervalAvg()) ? 0.0 : item.getFirstExplorationIntervalAvg();
+            double explorationInterval = (publish.getUpdateTime() - hisMinDate) / 86400.0;
+            firstExplorationIntervalAvg = (firstExplorationIntervalAvg * (publishTime - 1) + explorationInterval) / publishTime;
+            item.setFirstExplorationIntervalAvg(firstExplorationIntervalAvg);
         }
+        poolHisPublishTimeMap.put(type, publishTime);
     }
 
     private void setPublishPerformance(IntermediateIndicatorsExport item, NewSortStrategyExport data,
-                                       Map<String, Map<Integer, PublishSortLog>> publishSortLogMap) {
+                                       Map<String, Map<String, Map<Integer, PublishSortLog>>> publishSortLogMap) {
         item.setViewCount(item.getViewCount() + data.getViewCount());
         item.setFirstLayerUV(item.getFirstLayerUV() + data.getFirstLevel());
         item.setT0FissionUV(item.getT0FissionUV() + data.getFission0());
@@ -895,19 +930,22 @@ public class DataDashboardService {
         if (item.getT0FissionCount() + item.getT1FissionCount() + item.getT2FissionCount() > 0 && item.getViewBase() > 0) {
             item.setT2CumulativeFissionRate((item.getT0FissionCount() + item.getT1FissionCount() + item.getT2FissionCount()) / (double) item.getFirstLayerUV());
         }
-        Map<Integer, PublishSortLog> indexMap = publishSortLogMap.get(data.getGhId());
-        if (Objects.nonNull(indexMap)) {
-            PublishSortLog log = indexMap.get(data.getPosition());
-            double scoreAvg = item.getAccountArticleRelevanceAvg();
-            double score = 0.0;
-            if (Objects.nonNull(log) && Objects.nonNull(log.getScore())) {
-                JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
-                if (scoreMap.containsKey("SimilarityStrategy")) {
-                    score = scoreMap.getDoubleValue("SimilarityStrategy");
+        Map<String, Map<Integer, PublishSortLog>> dateSortMap = publishSortLogMap.get(data.getGhId());
+        if (Objects.nonNull(dateSortMap)) {
+            Map<Integer, PublishSortLog> indexMap = dateSortMap.get(item.getDateStr());
+            if (Objects.nonNull(indexMap)) {
+                PublishSortLog log = indexMap.get(data.getPosition());
+                double scoreAvg = item.getAccountArticleRelevanceAvg();
+                double score = 0.0;
+                if (Objects.nonNull(log) && Objects.nonNull(log.getScore())) {
+                    JSONObject scoreMap = JSONObject.parseObject(log.getScoreMap());
+                    if (scoreMap.containsKey("SimilarityStrategy")) {
+                        score = scoreMap.getDoubleValue("SimilarityStrategy");
+                    }
                 }
+                scoreAvg = (scoreAvg * (item.getActualArticleReleaseCount() - 1) + score) / item.getActualArticleReleaseCount();
+                item.setAccountArticleRelevanceAvg(scoreAvg);
             }
-            scoreAvg = (scoreAvg * (item.getActualArticleReleaseCount() - 1) + score) / item.getActualArticleReleaseCount();
-            item.setAccountArticleRelevanceAvg(scoreAvg);
         }
     }
 
@@ -975,20 +1013,20 @@ public class DataDashboardService {
     private String getArticleType(NewSortStrategyExport data, List<String> small) {
         if (data.getPosition() == 1) {
             if (small.contains(data.getGhId())) {
-                return "L1-订阅号冷启1";
+                return "L3";
             } else {
-                return "L1-订阅号无限流";
+                return "L4";
             }
         } else if (data.getPosition() == 2) {
-            return "L2-订阅号冷启2";
+            return "L2";
         } else {
-            return "L3-订阅号利用";
+            return "L1";
         }
     }
 
     private List<IntermediateIndicatorsExport> buildIntermediateIndicatorsData(String dateStr) {
         List<IntermediateIndicatorsExport> result = new ArrayList<>();
-        List<String> typeList = Arrays.asList("L3-订阅号利用", "L2-订阅号冷启2", "L1-订阅号冷启1", "L1-订阅号无限流");
+        List<String> typeList = Arrays.asList("L4", "L3", "L2", "L1");
         for (String type : typeList) {
             IntermediateIndicatorsExport item = new IntermediateIndicatorsExport();
             item.setDateStr(dateStr);