发布数据策略报表

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/NewSortStrategyExport.java

@@ -0,0 +1,28 @@
+package com.tzld.longarticle.recommend.server.model;
+
+import lombok.Data;
+
+@Data
+public class NewSortStrategyExport {
+
+    private String dateStr;
+    private String accountName;
+    private String strategy;
+    private long fans;
+    private Integer viewCount;
+    private Double avgViewCount;
+    private Integer firstLevel;
+    private Integer fission0;
+    private Integer fission1;
+    private Integer fission2;
+    private Double readRate = 0.0;
+    private Double readFansRate = 0.0;
+    private Double firstReadRate = 0.0;
+    private Double fission0ReadAvgRate = 0.0;
+    private Double fission1Fission0Rate = 0.0;
+    private Integer position;
+    private String ghId;
+    private String title;
+    private String link;
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/PublishSortLogRepository.java

@@ -5,6 +5,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
 import java.util.List;
+import java.util.Set;
 
 @Repository
 public interface PublishSortLogRepository extends JpaRepository<PublishSortLog, Long> {
@@ -14,4 +15,8 @@ public interface PublishSortLogRepository extends JpaRepository<PublishSortLog,
     List<PublishSortLog> findByDateStrAndTitleIn(String dateStr, List<String> titles);
 
     List<PublishSortLog> findByCrawlerChannelContentIdIn(List<String> crawlerChannelContentIds);
+
+    List<PublishSortLog> findByGhIdInAndDateStrGreaterThanEqual(Set<String> ghIds, String dateStr);
+
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/DataDashboardService.java

@@ -0,0 +1,301 @@
+package com.tzld.longarticle.recommend.server.service;
+
+import cn.hutool.core.collection.CollectionUtil;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.model.NewSortStrategyExport;
+import com.tzld.longarticle.recommend.server.repository.crawler.AccountAvgInfoRepository;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleDetailInfoRepository;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
+import com.tzld.longarticle.recommend.server.repository.crawler.PublishSortLogRepository;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.AccountAvgInfo;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.Article;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.ArticleDetailInfo;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.PublishSortLog;
+import com.tzld.longarticle.recommend.server.util.DateUtils;
+import com.tzld.longarticle.recommend.server.util.MapBuilder;
+import com.tzld.longarticle.recommend.server.util.feishu.FeiShu;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.util.Pair;
+import org.springframework.http.*;
+import org.springframework.stereotype.Service;
+import org.springframework.util.StringUtils;
+import org.springframework.web.client.RestTemplate;
+
+import java.lang.reflect.Field;
+import java.text.SimpleDateFormat;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Service
+@Slf4j
+public class DataDashboardService {
+
+    @Autowired
+    private ArticleRepository articleRepository;
+    @Autowired
+    private ArticleDetailInfoRepository articleDetailInfoRepository;
+    @Autowired
+    private AccountAvgInfoRepository accountAvgInfoRepository;
+    @Autowired
+    private PublishSortLogRepository publishSortLogRepository;
+
+    @ApolloJsonValue("${export.account.ghId:[]}")
+    private static List<String> ghIdList;
+    private static final String sheetToken = "M0pLs3uF6hfL0htn2dMcB9eFn8e";
+
+    public void export(String dateStr) {
+        List<String> dateStrList = getBeforeThreeDays(dateStr);
+        exportFeishuNewSortStrategy(dateStrList, sheetToken, "7d4e12");
+    }
+
+    private List<String> getBeforeThreeDays(String dateStr) {
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
+        // 获取今天的日期
+        LocalDate today = LocalDate.now();
+        if (!StringUtils.hasText(dateStr)) {
+            dateStr = today.format(formatter);
+        }
+        // 解析输入的日期字符串为LocalDate对象
+        LocalDate date = LocalDate.parse(dateStr, formatter);
+        // 获取从输入日期前三天的日期
+        LocalDate startDate = date.minusDays(3);
+        // 存储所有日期的列表
+        List<String> datesList = new ArrayList<>();
+        // 从startDate到today遍历日期
+        while (!startDate.isAfter(today)) {
+            // 将当前日期格式化为"yyyyMMdd"并添加到列表中
+            datesList.add(startDate.format(formatter));
+            // 日期加1天
+            startDate = startDate.plusDays(1);
+        }
+        return datesList;
+    }
+
+
+    private void exportFeishuNewSortStrategy(List<String> dateStrList, String sheetToken, String sheetId) {
+        String minDate = dateStrList.stream().min(String::compareTo).orElse("");
+        List<NewSortStrategyExport> newContentsYesData = newSortStrageData(minDate);
+        if (CollectionUtil.isEmpty(newContentsYesData)) {
+            return;
+        }
+
+        int rowNum = newContentsYesData.size();
+        List<List<Object>> rows = new ArrayList<>();
+        Field[] fields = NewSortStrategyExport.class.getDeclaredFields();
+        for (NewSortStrategyExport datum : newContentsYesData) {
+            List<Object> rowDatas = new ArrayList<>();
+            rows.add(rowDatas);
+
+            for (Field field : fields) {
+                field.setAccessible(true);
+                try {
+                    rowDatas.add(field.get(datum));
+                } catch (IllegalAccessException e) {
+                    log.error("获取值出错：{}", field.getName());
+                } catch (Exception e) {
+                    throw new RuntimeException(e.getMessage());
+                }
+            }
+        }
+
+        List<Pair<String, String>> styles = Arrays
+                .asList(
+                        Pair.of("K", "0.00%"),
+                        Pair.of("L", "0.00%"),
+                        Pair.of("M", "0.00%"),
+                        Pair.of("N", "0.00%"),
+                        Pair.of("O", "0.00%")
+                );
+
+        doSendFeishuSheet(dateStrList, sheetToken, sheetId, rowNum, rows, 2, styles);
+    }
+
+    private List<NewSortStrategyExport> newSortStrageData(String dateStr) {
+        long timestamp = DateUtils.dateStrToTimestamp(dateStr, "yyyyMMdd");
+        List<AccountAvgInfo> accountAvgInfoList = accountAvgInfoRepository.getAllByStatusEquals(1);
+        Set<String> ghIds = accountAvgInfoList.stream().map(AccountAvgInfo::getGhId).collect(Collectors.toSet());
+        List<Article> articleList = articleRepository.getByGhIdInAndUpdateTimeGreaterThan(ghIds, timestamp);
+
+        Set<String> snList = articleList.stream().map(Article::getWxSn).collect(Collectors.toSet());
+        List<ArticleDetailInfo> articleDetailInfoList = articleDetailInfoRepository.getAllByWxSnIn(snList);
+        Map<String, List<ArticleDetailInfo>> articleDetailInfoMap = articleDetailInfoList.stream()
+                .collect(Collectors.groupingBy(ArticleDetailInfo::getWxSn));
+        List<PublishSortLog> sortLogList = publishSortLogRepository.findByGhIdInAndDateStrGreaterThanEqual(ghIds, dateStr);
+        Map<String, Map<String, String>> sortStrategyMap = sortLogList.stream()
+                .collect(Collectors.groupingBy(PublishSortLog::getGhId,
+                        Collectors.toMap(PublishSortLog::getDateStr, PublishSortLog::getStrategy, (existing, replacement) -> replacement)));
+
+        Map<String, Map<String, AccountAvgInfo>> accountAvgInfoIndexMap = accountAvgInfoList.stream().collect(
+                Collectors.groupingBy(AccountAvgInfo::getGhId, Collectors.toMap(AccountAvgInfo::getPosition, o -> o)));
+        List<NewSortStrategyExport> result = new ArrayList<>();
+        for (Article article : articleList) {
+            List<ArticleDetailInfo> articleDetailInfos = articleDetailInfoMap.get(article.getWxSn());
+            if (CollectionUtils.isEmpty(articleDetailInfos)) {
+                continue;
+            }
+            Map<String, String> dateStrategy = sortStrategyMap.get(article.getGhId());
+            Date minDate = articleDetailInfos.stream().map(ArticleDetailInfo::getRecallDt).min(Date::compareTo).orElse(new Date());
+            int sumfirstLevel = 0;
+            int sumFission0 = 0;
+            int sumFission1 = 0;
+            int sumFission2 = 0;
+            for (ArticleDetailInfo articleDetailInfo : articleDetailInfos) {
+                if (articleDetailInfo.getRecallDt().equals(minDate)) {
+                    sumfirstLevel += Optional.ofNullable(articleDetailInfo.getFirstLevel()).orElse(0);
+                    sumFission0 += Optional.ofNullable(articleDetailInfo.getFission0()).orElse(0);
+                    sumFission1 += Optional.ofNullable(articleDetailInfo.getFission1()).orElse(0);
+                    sumFission2 += Optional.ofNullable(articleDetailInfo.getFission2()).orElse(0);
+                }
+            }
+            Map<String, AccountAvgInfo> accountAvgInfoMap = accountAvgInfoIndexMap.get(article.getGhId());
+            AccountAvgInfo avgInfo = accountAvgInfoMap.get(article.getItemIndex().toString());
+            SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+            String date = sdf.format(new Date(article.getUpdateTime() * 1000));
+            NewSortStrategyExport obj = new NewSortStrategyExport();
+            obj.setGhId(article.getGhId());
+            obj.setAccountName(article.getAccountName());
+            obj.setTitle(article.getTitle());
+            obj.setLink(article.getContentUrl());
+            obj.setPosition(article.getItemIndex());
+            obj.setViewCount(article.getShowViewCount());
+            obj.setDateStr(date);
+            obj.setStrategy(dateStrategy.get(date));
+            if (Objects.nonNull(avgInfo)) {
+                obj.setFans(avgInfo.getFans());
+                obj.setAvgViewCount(avgInfo.getReadAvg());
+                if (avgInfo.getReadAvg() > 0) {
+                    obj.setReadRate((article.getShowViewCount() * 1.0) / avgInfo.getReadAvg());
+                    obj.setFission0ReadAvgRate((sumFission0 * 1.0) / avgInfo.getReadAvg());
+                }
+                if (avgInfo.getFans() > 0) {
+                    obj.setReadFansRate((article.getShowViewCount() * 1.0) / avgInfo.getFans());
+                }
+            }
+            obj.setFirstLevel(sumfirstLevel);
+            obj.setFission0(sumFission0);
+            obj.setFission1(sumFission1);
+            obj.setFission2(sumFission2);
+            if (article.getShowViewCount() > 0) {
+                obj.setFirstReadRate((sumfirstLevel * 1.0) / article.getShowViewCount());
+            }
+            if (sumFission0 > 0) {
+                obj.setFission1Fission0Rate((sumFission1 * 1.0) / sumFission0);
+            }
+            result.add(obj);
+        }
+        result.sort(Comparator.comparing(NewSortStrategyExport::getDateStr).reversed()
+                .thenComparing(NewSortStrategyExport::getGhId).thenComparing(NewSortStrategyExport::getPosition));
+        return result;
+    }
+
+
+    private static void doSendFeishuSheet(List<String> dateStrList, String sheetToken, String sheetId,
+                                          int rowNum, List<List<Object>> rows, Integer startRowIndex,
+                                          List<Pair<String, String>> styles) {
+        Pair<String, Integer> token = FeiShu.requestAccessToken();
+        RestTemplate restTemplate = new RestTemplate();
+        HttpHeaders httpHeaders = new HttpHeaders();
+        httpHeaders.add("Authorization", "Bearer " + token.getFirst());
+
+        // 先删除掉已存在的dateStr数据
+        HttpEntity<Object> queryEntity = new HttpEntity<>(httpHeaders);
+        ResponseEntity<String> queryResponseEntity = restTemplate.exchange(
+                String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/values/%s!A" + startRowIndex + ":A" + (startRowIndex + (rowNum * 2)),
+                        sheetToken, sheetId),
+                HttpMethod.GET, queryEntity, String.class);
+        JSONArray values = JSON.parseObject(queryResponseEntity.getBody())
+                .getJSONObject("data")
+                .getJSONObject("valueRange")
+                .getJSONArray("values");
+        int count = 0;
+        if (!values.isEmpty() && Objects.nonNull(values.get(0))
+                && dateStrList.contains(((JSONArray) values.get(0)).get(0).toString())) {
+            for (Object value : values) {
+                if (((JSONArray) value).get(0) != null) {
+                    List<String> dates = ((JSONArray) value).stream().map(Object::toString).collect(Collectors.toList());
+                    if (dateStrList.contains(dates.get(0))) {
+                        count++;
+                        continue;
+                    }
+                }
+                break;
+            }
+        }
+        if (count > 0) {
+            // 删除当前日期已存在的旧数据
+            httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+            HttpEntity<Object> deleteEntity = new HttpEntity<>(
+                    String.format("{\n" +
+                            "    \"dimension\": {\n" +
+                            "        \"sheetId\": \"%s\",\n" +
+                            "        \"majorDimension\": \"ROWS\",\n" +
+                            "        \"startIndex\": %s,\n" +
+                            "        \"endIndex\": %s\n" +
+                            "    }\n" +
+                            "}", sheetId, startRowIndex, count + startRowIndex - 1),
+                    httpHeaders);
+            restTemplate.exchange(String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/dimension_range", sheetToken),
+                    HttpMethod.DELETE, deleteEntity, String.class);
+        }
+
+        // 插入数据
+        HttpEntity<Object> postEntity = new HttpEntity<>(MapBuilder
+                .builder()
+                .put("valueRange", MapBuilder
+                        .builder()
+                        .put("range", String.format("%s!A" + startRowIndex + ":S", sheetId) + (rowNum + startRowIndex - 1))
+                        .put("values", rows)
+                        .build())
+                .build(), httpHeaders);
+        restTemplate.exchange(String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/values_prepend",
+                        sheetToken),
+                HttpMethod.POST, postEntity, String.class);
+//        // 合并单元格
+//        if (isMerge) {
+//            HttpEntity<Object> mergeEntity = new HttpEntity<>(MapBuilder
+//                    .builder()
+//                    .put("range", String.format("%s!A" + startRowIndex + ":A", sheetId) + (rowNum + startRowIndex - 1))
+//                    .put("mergeType", "MERGE_ALL")
+//                    .build(), httpHeaders);
+//            restTemplate.exchange(String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/merge_cells",
+//                            sheetToken),
+//                    HttpMethod.POST, mergeEntity, String.class);
+//        }
+        // 此处先简单处理，调整单元格为”百分比小数点“
+        if (CollectionUtil.isNotEmpty(styles)) {
+            for (Pair<String, String> style : styles) {
+                HttpEntity<Map<Object, Object>> styleEntity = new HttpEntity<>(MapBuilder
+                        .builder()
+                        .put("appendStyle",
+                                MapBuilder
+                                        .builder()
+                                        .put("range", String.format("%s!%s" + startRowIndex + ":%s", sheetId,
+                                                style.getFirst(), style.getFirst())
+                                                + (rowNum + startRowIndex - 1))
+                                        .put("style",
+                                                MapBuilder
+                                                        .builder()
+                                                        .put("formatter", style.getSecond())
+                                                        .build()
+                                        )
+                                        .build()
+                        )
+                        .build(), httpHeaders);
+                restTemplate.exchange(
+                        String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/style",
+                                sheetToken),
+                        HttpMethod.PUT,
+                        styleEntity,
+                        String.class
+                );
+            }
+        }
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/RecommendService.java

@@ -250,7 +250,7 @@ public class RecommendService {
             Content content = rankResult.getContents().get(i - 1);
 
             PublishSortLog sortLog = new PublishSortLog();
-            sortLog.setDateStr(DateUtils.getCurrentDateStr("yyyy-MM-dd"));
+            sortLog.setDateStr(DateUtils.getCurrentDateStr("yyyyMMdd"));
             sortLog.setGhId(param.getGhId());
             sortLog.setAccountName(param.getAccountName());
             sortLog.setCrawlerChannelContentId(content.getCrawlerChannelContentId());

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/util/DateUtils.java

@@ -1,7 +1,9 @@
 package com.tzld.longarticle.recommend.server.util;
 
 import java.text.SimpleDateFormat;
+import java.time.LocalDate;
 import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
 import java.util.Calendar;
 import java.util.Date;
@@ -53,6 +55,23 @@ public final class DateUtils {
         }
     }
 
+    public static long dateStrToTimestamp(String dateStr, String format) {
+        // 定义日期格式
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format);
+
+        // 解析输入的日期字符串为LocalDate对象
+        LocalDate date = LocalDate.parse(dateStr, formatter);
+
+        // 将LocalDate转换为LocalDateTime，时间部分设为00:00:00
+        LocalDateTime dateTime = date.atStartOfDay();
+
+        // 定义北京时区
+        ZoneId zone = ZoneId.of("Asia/Shanghai");
+
+        // 将LocalDateTime转换为时间戳（秒数），使用北京时区
+        return dateTime.toEpochSecond(zone.getRules().getOffset(dateTime));
+    }
+
 
     public static void main(String[] args) {
         Calendar calendar = Calendar.getInstance();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/DataDashboardController.java

@@ -0,0 +1,25 @@
+package com.tzld.longarticle.recommend.server.web;
+
+import com.tzld.longarticle.recommend.server.service.DataDashboardService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/dataDashboard")
+@Slf4j
+public class DataDashboardController {
+
+    @Autowired
+    private DataDashboardService service;
+
+    @GetMapping("/export/newSortStrategy")
+    public void newSortStrategyExport(String dateStr) {
+        new Thread(() -> {
+            service.export(dateStr);
+        }).start();
+    }
+
+}