Merge branch 'master' into wyp/0124-articleAutoDelete

# Conflicts:
#	long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java
#	long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/ContentCountMonitor.java

@@ -0,0 +1,16 @@
+package com.tzld.longarticle.recommend.server.common;
+
+import com.tzld.longarticle.recommend.server.common.constant.LogConstants;
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
+
+@Slf4j
+public class ContentCountMonitor {
+
+    public static void logCount(String type, String name, Integer count) {
+        Marker marker = MarkerFactory.getMarker(LogConstants.MARKER_RECALL);
+        log.info(marker, "{}.{} count: {}", type, name, count);
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/RecommendLoghubAppender.java

@@ -115,6 +115,8 @@ public class RecommendLoghubAppender<E> extends UnsynchronizedAppenderBase<E> {
             appendPlainEvent(eventObject);
         } else if (LogConstants.MARKER_COST.equals(event.getMarker().getName())) {
             appendCostEvent(event);
+        } else if (LogConstants.MARKER_RECALL.equals(event.getMarker().getName())) {
+            appendRecallEvent(event);
         }
     }
 
@@ -153,6 +155,41 @@ public class RecommendLoghubAppender<E> extends UnsynchronizedAppenderBase<E> {
         }
     }
 
+    private void appendRecallEvent(LoggingEvent event) {
+        List<LogItem> logItems = new ArrayList();
+        LogItem item = new LogItem();
+        logItems.add(item);
+        item.SetTime((int) (event.getTimeStamp() / 1000L));
+
+        item.PushBack("thread", event.getThreadName());
+        Object[] args = event.getArgumentArray();
+        try {
+            item.PushBack("type", (String) args[0]);
+            item.PushBack("name", (String) args[1]);
+            item.PushBack("count", String.valueOf(args[2]));
+        } catch (Exception e) {
+            this.addError("invalid event argument ", e);
+        }
+
+        Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
+            event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
+                return Arrays.stream(f.split(",")).anyMatch((i) -> {
+                    return i.equals(v.getKey());
+                });
+            }).forEach((map) -> {
+                item.PushBack((String) map.getKey(), (String) map.getValue());
+            });
+        });
+
+        try {
+            this.producer.send(this.projectConfig.getProject(), this.logStore, "ContentCountMonitor", this.source, logItems,
+                    new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
+                            this.topic, this.source, logItems));
+        } catch (Exception e) {
+            this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, e);
+        }
+    }
+
     private void appendPlainEvent(E eventObject) {
         LoggingEvent event = (LoggingEvent) eventObject;
         List<LogItem> logItems = new ArrayList();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/constant/LogConstants.java

@@ -2,4 +2,6 @@ package com.tzld.longarticle.recommend.server.common.constant;
 
 public class LogConstants {
     public static final String MARKER_COST = "COST";
+
+    public static final String MARKER_RECALL = "RECALL";
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -38,4 +38,16 @@ public interface AigcBaseMapper {
     List<ProduceTaskAtom> getProduceScoreByContentId(List<String> planExeIds);
 
     List<ContentPoolTypeDTO> getContentPoolType(List<String> contentIds);
+
+    List<String> getProducePlanId();
+
+    List<String> getCrawlerPlanByProducePlanIds(List<String> producePlanIds);
+
+    Long getCrawlerContentCountByCrawlerPlanIds(List<String> crawlerPlanIds, Long start, Long end);
+
+    List<String> getCrawlerSuccessPlanByCrawlerPlanIds(List<String> crawlerPlanIds, Long start, Long end);
+
+    Long getProduceContentCountByProducePlanIds(List<String> producePlanIds, Long start, Long end);
+
+    Long getProduceAuditPassCountByProducePlanIds(List<String> producePlanIds, Long start, Long end);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -7,6 +7,7 @@ import com.tzld.longarticle.recommend.server.model.entity.longArticle.*;
 import com.tzld.longarticle.recommend.server.model.param.ArticleVideoPoolSourceParam;
 import org.apache.ibatis.annotations.Mapper;
 
+import java.util.Date;
 import java.util.List;
 
 @Mapper
@@ -75,5 +76,7 @@ public interface LongArticleBaseMapper {
 
     List<String> getDisAuditDeleteGhid();
 
+    Long countMatchSuccessCount(Date start, Date end);
+
     void batchInsertLongArticleAuditDelete(List<LongArticleAuditDelete> deleteList);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/LongArticleAuditDelete.java

@@ -39,6 +39,9 @@ public class LongArticleAuditDelete {
     @Column(name = "status")
     private Integer status;
 
+    @Column(name = "delete_reason")
+    private String deleteReason;
+
     @Column(name = "fail_reason")
     private String failReason;
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/ArticleDangerFindDeleteParam.java

@@ -10,4 +10,7 @@ public class ArticleDangerFindDeleteParam {
     private String title;
     private String wxSn;
 
+    // 1-html手动删除
+    private Integer deleteFlag = 0;
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/ContentFunnelExport.java

@@ -0,0 +1,16 @@
+package com.tzld.longarticle.recommend.server.model.vo;
+
+import lombok.Data;
+
+@Data
+public class ContentFunnelExport {
+
+    private String dateStr;
+    private Long planCrawlerCount = 0L;
+    private Long crawlerCount = 0L;
+    private Long produceCount = 0L;
+    private Long produceAuditPassCount = 0L;
+    private Long matchSuccessCount = 0L;
+    private Long videoAuditPassCount = 0L;
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleAuditService.java

@@ -367,7 +367,7 @@ public class ArticleAuditService {
         List<String> reMatchTraceIds = traceIds.stream().filter(o -> !deleteTraceIds.contains(o)).collect(Collectors.toList());
         buildReMatchRecord(reMatchTraceIds, response.getVideoOSS(), matchVideoMap);
         // 文章删除
-        buildArticleAuditDelete(publishContentIds);
+        buildArticleAuditDelete(publishContentIds, "手动根据videoId删除");
         return null;
     }
 
@@ -404,7 +404,7 @@ public class ArticleAuditService {
         longArticleBaseMapper.batchInsertArticleReMatchRecord(reMatchRecordList);
     }
 
-    public void buildArticleAuditDelete(List<String> publishContentIds) {
+    public void buildArticleAuditDelete(List<String> publishContentIds, String deleteReason) {
         if (CollectionUtils.isEmpty(publishContentIds)) {
             return;
         }
@@ -448,12 +448,12 @@ public class ArticleAuditService {
             try {
                 String pushId = publishPushIdMap.get(publishContentId);
                 if (!StringUtils.hasText(pushId)) {
-                    deleteFailAlarmAdd(alarmList, publishContentId, "无推送记录", 0);
+                    deleteFailAlarmAdd(alarmList, publishContentId, "无推送记录", 0, deleteReason);
                     continue;
                 }
                 PublishGzhPushDTO publishGzhPushDTO = pushDTOMap.get(pushId);
                 if (Objects.isNull(publishGzhPushDTO)) {
-                    deleteFailAlarmAdd(alarmList, publishContentId, "无推送记录", 0);
+                    deleteFailAlarmAdd(alarmList, publishContentId, "无推送记录", 0, deleteReason);
                     continue;
                 }
                 String publishAccountId = pushAccountMap.get(pushId);
@@ -469,12 +469,12 @@ public class ArticleAuditService {
                 }
                 if (!publishGzhPushDTO.getPushType().equals(PushTypeEnum.AUTO_GROUP_PUBLISH.getVal())) {
                     PushTypeEnum pushTypeEnum = PushTypeEnum.from(publishGzhPushDTO.getPushType());
-                    deleteFailAlarmAdd(alarmList, publishContentId, "推送类型为" + pushTypeEnum.getDescription(), index);
+                    deleteFailAlarmAdd(alarmList, publishContentId, "推送类型为" + pushTypeEnum.getDescription(), index, deleteReason);
                     continue;
                 }
                 String groupPushMsgId = pushIdMap.get(pushId);
                 if (!StringUtils.hasText(groupPushMsgId)) {
-                    deleteFailAlarmAdd(alarmList, publishContentId, "无推送MsgId", index);
+                    deleteFailAlarmAdd(alarmList, publishContentId, "无推送MsgId", index, deleteReason);
                     continue;
                 }
                 LongArticleAuditDelete delete = new LongArticleAuditDelete();
@@ -485,6 +485,7 @@ public class ArticleAuditService {
                 delete.setPublishContentId(publishContentId);
                 delete.setIndex(index);
                 delete.setStatus(StatusEnum.ZERO.getCode());
+                delete.setDeleteReason(deleteReason);
                 delete.setCreateTimestamp(System.currentTimeMillis());
                 longArticleAuditDeleteRepository.save(delete);
             } catch (Exception e) {
@@ -494,11 +495,12 @@ public class ArticleAuditService {
         deleteFailAlarm(alarmList, accountTypeMap);
     }
 
-    private void deleteFailAlarmAdd(List<JSONObject> alarmList, String publishContentId, String errMsg, Integer index) {
+    private void deleteFailAlarmAdd(List<JSONObject> alarmList, String publishContentId, String errMsg, Integer index, String deleteReason) {
         JSONObject alarm = new JSONObject();
         alarm.put("publishContentId", publishContentId);
         alarm.put("errMsg", errMsg);
         alarm.put("index", index);
+        alarm.put("deleteReason", deleteReason);
         alarmList.add(alarm);
     }
 
@@ -591,7 +593,7 @@ public class ArticleAuditService {
                             }
                         } else {
                             msgIdList.forEach(delete -> {
-                                deleteFailAlarmAdd(alarmList, delete.getPublishContentId(), "非自动群发", delete.getIndex());
+                                deleteFailAlarmAdd(alarmList, delete.getPublishContentId(), "非自动群发", delete.getIndex(), delete.getDeleteReason());
                                 delete.setStatus(ArticleDeleteStatusEnum.SUCCESS.getCode());
                                 delete.setFinishTimestamp(System.currentTimeMillis());
                                 longArticleAuditDeleteRepository.save(delete);
@@ -646,9 +648,10 @@ public class ArticleAuditService {
             row.put("index", String.valueOf(alarm.getInteger("index")));
             row.put("title", publishContent.getTitle());
             row.put("err_msg", alarm.getString("errMsg"));
+            row.put("delete_reason", alarm.getString("deleteReason"));
             rows.add(row);
         }
-        FeishuTableDTO tableDTO = FeishuTableDTO.createTable("文章删除失败", columns, rows, false);
+        FeishuTableDTO tableDTO = FeishuTableDTO.createTable("文章删除任务添加失败", columns, rows, false);
         JSONObject content = JSONObject.parseObject(JSONObject.toJSONString(tableDTO));
         // 无法自动删除 发送飞书通知 人工删除
         JSONObject bodyParam = new JSONObject();
@@ -678,8 +681,11 @@ public class ArticleAuditService {
                 FieshuTableColumnDataTypeEnum.TEXT.getType(), "title", "标题", null);
         columns.add(titleColumn);
         FeishuTableDTO.Column errMsgColumn = FeishuTableDTO.createFeishuColumns(
-                FieshuTableColumnDataTypeEnum.TEXT.getType(), "err_msg", "原因", null);
+                FieshuTableColumnDataTypeEnum.TEXT.getType(), "err_msg", "添加失败原因", null);
         columns.add(errMsgColumn);
+        FeishuTableDTO.Column deleteReasonColumn = FeishuTableDTO.createFeishuColumns(
+                FieshuTableColumnDataTypeEnum.TEXT.getType(), "delete_reason", "删除原因", null);
+        columns.add(deleteReasonColumn);
         return columns;
     }
 
@@ -717,7 +723,7 @@ public class ArticleAuditService {
 
         // 根据已发布文章查找推送id 并删除
         List<String> publishContentIds = publishContents.stream().map(PublishContentDTO::getId).collect(Collectors.toList());
-        buildArticleAuditDelete(publishContentIds);
+        buildArticleAuditDelete(publishContentIds, param.getDeleteFlag() == 1 ? "html手动删除" : "违规检查删除");
         buildArticleUnsafeTitle(param.getGhId(), titleMd5, param.getTitle());
     }
 
@@ -838,7 +844,7 @@ public class ArticleAuditService {
             List<String> existsIds = exists.stream().map(LongArticleAuditDelete::getPublishContentId).collect(Collectors.toList());
             publishContentIds.removeAll(existsIds);
             if (CollectionUtils.isNotEmpty(publishContentIds)) {
-                buildArticleAuditDelete(publishContentIds);
+                buildArticleAuditDelete(publishContentIds, "单小程序文章定期删除");
             }
             pageNum++;
         }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleVideoAuditService.java

@@ -204,7 +204,7 @@ public class ArticleVideoAuditService {
         List<PublishContentMiniprogramDTO> publishContents = publishContentMapper.getPublishContentByTraceIdIn(traceIds);
         List<String> publishContentIds = publishContents.stream().map(PublishContentMiniprogramDTO::getPublishContentId)
                 .collect(Collectors.toList());
-        articleAuditService.buildArticleAuditDelete(publishContentIds);
+        articleAuditService.buildArticleAuditDelete(publishContentIds, "视频审核不通过删除");
     }
 
     public List<String> getFilterValue(AuditFilterParam param) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/DataDashboardService.java

@@ -96,6 +96,8 @@ public class DataDashboardService {
     private LongArticleTitleAuditRepository titleAuditRepository;
     @Autowired
     private LongArticleCrawlerVideoRepository crawlerVideoRepository;
+    @Autowired
+    private LongArticleTitleAuditRepository longArticleTitleAuditRepository;
 
     @ApolloJsonValue("${export.account.ghId:[]}")
     private static List<String> ghIdList;
@@ -2010,7 +2012,7 @@ public class DataDashboardService {
         List<LongArticleTitleAudit> auditList = titleAuditRepository.getByAuditTimestampBetween(timestamp, timestamp + 86400000);
         List<LongArticleCrawlerVideo> crawlerVideoList = crawlerVideoRepository.getByAuditTimestampBetween(timestamp, timestamp + 86400000);
         Map<String, List<LongArticleCrawlerVideo>> crawlerVideoMap = crawlerVideoList.stream()
-               .collect(Collectors.groupingBy(LongArticleCrawlerVideo::getContentId));
+                .collect(Collectors.groupingBy(LongArticleCrawlerVideo::getContentId));
         List<String> contentIds = auditList.stream().map(LongArticleTitleAudit::getContentId).collect(Collectors.toList());
         List<ContentPoolTypeDTO> poolTypeDTOS = aigcBaseMapper.getContentPoolType(contentIds);
         Map<String, ContentPoolTypeDTO> poolTypeMap = poolTypeDTOS.stream().collect(Collectors.toMap(ContentPoolTypeDTO::getContentId, o -> o));
@@ -2034,7 +2036,7 @@ public class DataDashboardService {
             }
             videoAuditExport.setVideoAuditCount(videoAuditExport.getVideoAuditCount() + videoList.size());
             long videoAuditPassCount = videoList.stream()
-                   .filter(o -> o.getStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
+                    .filter(o -> o.getStatus().equals(ProduceContentAuditStatusEnum.pass.getVal())).count();
             videoAuditExport.setVideoAuditPassCount(videoAuditExport.getVideoAuditPassCount() + videoAuditPassCount);
         }
         for (Map.Entry<String, VideoAuditExport> entry : map.entrySet()) {
@@ -2061,4 +2063,93 @@ public class DataDashboardService {
         return result;
     }
 
+    @XxlJob("contentFunnelExport")
+    public ReturnT<String> contentFunnelExportJob(String param) {
+        List<String> dateStrList = DateUtils.getBeforeDays(null, null, 1);
+        contentFunnelExport(dateStrList);
+        return ReturnT.SUCCESS;
+    }
+
+    public void contentFunnelExport(String dateStr) {
+        if (!StringUtils.hasText(dateStr)) {
+            dateStr = DateUtils.getBeforeDaysDateStr("yyyyMMdd", 1);
+        }
+        contentFunnelExport(Collections.singletonList(dateStr));
+    }
+
+    public void contentFunnelExport(List<String> dateStrList) {
+        List<ContentFunnelExport> exportList = new ArrayList<>();
+        dateStrList = Lists.reverse(dateStrList);
+        for (String dateStr : dateStrList) {
+            exportList.add(buildContentFunnelExport(dateStr));
+        }
+        if (CollectionUtil.isEmpty(exportList)) {
+            return;
+        }
+        int rowNum = exportList.size();
+        List<List<Object>> rows = new ArrayList<>();
+        Field[] fields = ContentFunnelExport.class.getDeclaredFields();
+        for (ContentFunnelExport datum : exportList) {
+            List<Object> rowDatas = new ArrayList<>();
+            rows.add(rowDatas);
+
+            for (Field field : fields) {
+                field.setAccessible(true);
+                try {
+                    rowDatas.add(field.get(datum));
+                } catch (IllegalAccessException e) {
+                    log.error("获取值出错：{}", field.getName());
+                } catch (Exception e) {
+                    throw new RuntimeException(e.getMessage());
+                }
+            }
+        }
+
+        doSendFeishuSheet(dateStrList, dailyDetailSheetToken, "qEipyL", rowNum, rows,
+                2, null, null);
+    }
+
+    private ContentFunnelExport buildContentFunnelExport(String dateStr) {
+        ContentFunnelExport result = new ContentFunnelExport();
+        Long start = DateUtils.getStartOfDay(dateStr, "yyyyMMdd") * 1000;
+        Long end = start + 86400000;
+        List<String> producePlanIds = aigcBaseMapper.getProducePlanId();
+        List<String> crawlerPlanIds = aigcBaseMapper.getCrawlerPlanByProducePlanIds(producePlanIds);
+        Long crawlerCount = aigcBaseMapper.getCrawlerContentCountByCrawlerPlanIds(crawlerPlanIds, start, end);
+        Long planCrawlerCount = 0L;
+        List<String> crawlerSuccessPlanIds = aigcBaseMapper.getCrawlerSuccessPlanByCrawlerPlanIds(crawlerPlanIds, start, end);
+        List<CrawlerPlan> crawlerPlanList = aigcBaseMapper.getCrawlerPlanByPlanIds(crawlerSuccessPlanIds);
+        for (CrawlerPlan crawlerPlan : crawlerPlanList) {
+            JSONObject crawlerModeValue = JSONObject.parseObject(crawlerPlan.getCrawlerModeValue());
+            if (crawlerModeValue == null) {
+                continue;
+            }
+            JSONObject sourceModeValues = crawlerModeValue.getJSONObject("sourceModeValues");
+            if (sourceModeValues == null) {
+                continue;
+            }
+            JSONArray inputModeValues = sourceModeValues.getJSONArray("inputModeValues");
+            if (inputModeValues == null) {
+                continue;
+            }
+            planCrawlerCount += inputModeValues.size();
+        }
+        Long produceCount = aigcBaseMapper.getProduceContentCountByProducePlanIds(producePlanIds, start, end);
+        Long produceAuditPassCount = aigcBaseMapper.getProduceAuditPassCountByProducePlanIds(producePlanIds, start, end);
+        Date dateStart = DateUtils.getStartDateOfDay(start / 1000);
+        Date dateEnd = DateUtils.getStartDateOfDay(end / 1000);
+        Long matchSuccessCount = longArticleBaseMapper.countMatchSuccessCount(dateStart, dateEnd);
+        Long videoAuditPassCount = longArticleTitleAuditRepository.countByStatusAndAuditTimestampBetween(
+                1, start, end);
+
+        result.setDateStr(dateStr);
+        result.setPlanCrawlerCount(planCrawlerCount);
+        result.setCrawlerCount(crawlerCount);
+        result.setProduceCount(produceCount);
+        result.setProduceAuditPassCount(produceAuditPassCount);
+        result.setMatchSuccessCount(matchSuccessCount);
+        result.setVideoAuditPassCount(videoAuditPassCount);
+        return result;
+    }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/FilterService.java

@@ -1,5 +1,6 @@
 package com.tzld.longarticle.recommend.server.service.recommend.filter;
 
+import com.tzld.longarticle.recommend.server.common.ContentCountMonitor;
 import com.tzld.longarticle.recommend.server.common.CostMonitor;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
@@ -46,6 +47,8 @@ public class FilterService {
                     FilterResult ret = strategy.filter(param);
                     long t2 = System.currentTimeMillis();
                     CostMonitor.logCost("Filter", strategy.getClass().getSimpleName(), t2 - t1);
+                    ContentCountMonitor.logCount("Filter", strategy.getClass().getSimpleName(),
+                            Objects.isNull(ret.getContentIds()) ? 0 : ret.getContentIds().size());
                     return ret;
                 } finally {
                     cdl.countDown();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/strategy/SensitiveStrategy.java

@@ -3,11 +3,13 @@ package com.tzld.longarticle.recommend.server.service.recommend.filter.strategy;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.tzld.longarticle.recommend.server.common.CommonThreadPoolExecutor;
 import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleSensitive;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleUnsafeTitle;
-import com.tzld.longarticle.recommend.server.remote.ArticleSensitiveRemoteService;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleSensitiveRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleUnsafeTitleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterParam;
@@ -20,15 +22,13 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import java.util.*;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 import java.util.stream.Collectors;
 
 @Component
 @Slf4j
 public class SensitiveStrategy implements FilterStrategy {
 
-    @Autowired
-    private ArticleSensitiveRemoteService articleSensitiveRemoteService;
     @Autowired
     private ArticleSensitiveRepository articleSensitiveRepository;
     @Autowired
@@ -37,14 +37,22 @@ public class SensitiveStrategy implements FilterStrategy {
     @ApolloJsonValue("${UnSafeTitles:[]}")
     private static List<String> UnSafeTitles;
 
+    private final static ExecutorService pool = new CommonThreadPoolExecutor(
+            5,
+            5,
+            0L, TimeUnit.SECONDS,
+            new LinkedBlockingQueue<>(100),
+            new ThreadFactoryBuilder().setNameFormat("SensitiveStrategy-%d").build(),
+            new ThreadPoolExecutor.AbortPolicy());
+
     Cache<String, Boolean> similarityCache = CacheBuilder.newBuilder()
             .expireAfterWrite(60 * 60, TimeUnit.SECONDS).build();
 
     @Override
     public FilterResult filter(FilterParam param) {
         FilterResult filterResult = new FilterResult();
-        List<String> result = new ArrayList<>(param.getContents().size());
-        List<Content> filterContents = new ArrayList<>();
+        List<String> result = Collections.synchronizedList(new ArrayList<>(param.getContents().size()));
+        List<Content> filterContents = Collections.synchronizedList(new ArrayList<>());
 
 //        Map<String, String> titleMd5Map = new HashMap<>();
 //        Map<String, ArticleSensitive> articleSensitiveMap = new HashMap<>();
@@ -75,25 +83,40 @@ public class SensitiveStrategy implements FilterStrategy {
         Map<String, Boolean> similarityMap = new HashMap<>(similarityCache.getAllPresent(allTitles));
         long t4 = System.currentTimeMillis();
         log.info("SensitiveStrategy get unsafe title cache cost time:{}", t4 - t3);
-        for (Content content : param.getContents()) {
-            try {
-                Boolean isSimilar;
-                if (similarityMap.containsKey(content.getTitle())) {
-                    isSimilar = similarityMap.get(content.getTitle());
-                } else {
-                    isSimilar = TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), unsafeTitleCache,
-                            TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
-                    similarityCache.put(content.getTitle(), isSimilar);
-                }
-                if (isSimilar) {
-                    content.setFilterReason("安全违规");
-                    filterContents.add(content);
-                } else {
-                    result.add(content.getId());
+        List<List<Content>> partitions = Lists.partition(param.getContents(), 500);
+        CountDownLatch cdl = new CountDownLatch(partitions.size());
+        for (List<Content> partition : partitions) {
+            pool.submit(() -> {
+                try {
+                    for (Content content : partition) {
+                        try {
+                            Boolean isSimilar;
+                            if (similarityMap.containsKey(content.getTitle())) {
+                                isSimilar = similarityMap.get(content.getTitle());
+                            } else {
+                                isSimilar = TitleSimilarCheckUtil.isDuplicateContentByCache(content.getTitle(), unsafeTitleCache,
+                                        TitleSimilarCheckUtil.SIMILARITY_THRESHOLD);
+                                similarityCache.put(content.getTitle(), isSimilar);
+                            }
+                            if (isSimilar) {
+                                content.setFilterReason("安全违规");
+                                filterContents.add(content);
+                            } else {
+                                result.add(content.getId());
+                            }
+                        } catch (Exception e) {
+                            log.error("similar check error ", e);
+                        }
+                    }
+                } finally {
+                    cdl.countDown();
                 }
-            } catch (Exception e) {
-                log.error("similar check error ", e);
-            }
+            });
+        }
+        try {
+            cdl.await();
+        } catch (InterruptedException e) {
+            log.error("similar check error ", e);
         }
         long t5 = System.currentTimeMillis();
         log.info("SensitiveStrategy filter cost time:{}", t5 - t4);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/RankService.java

@@ -7,6 +7,7 @@ import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.model.ConfigChangeEvent;
 import com.ctrip.framework.apollo.spring.annotation.ApolloConfigChangeListener;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.ContentCountMonitor;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
@@ -96,6 +97,7 @@ public class RankService implements InitializingBean {
 
     public static void printSortLog(String strategy, String accountName, String position, List<Content> contentList) {
         JSONArray jsonArray = new JSONArray();
+        ContentCountMonitor.logCount("Rank", position, Objects.isNull(contentList) ? 0 : contentList.size());
         if (CollectionUtil.isEmpty(contentList)) {
             return;
         }
@@ -176,6 +178,7 @@ public class RankService implements InitializingBean {
         if (CollectionUtils.isNotEmpty(pool)) {
             pool = contentSourceTypeFilter(param.getStrategy(), pool, 3);
         }
+
         if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
             RankService.printSortLog(strategy, param.getAccountName(), "3-8", pool);
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV5Strategy.java

@@ -2,6 +2,7 @@ package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.ContentCountMonitor;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -6,6 +6,7 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Lists;
+import com.tzld.longarticle.recommend.server.common.ContentCountMonitor;
 import com.tzld.longarticle.recommend.server.common.CostMonitor;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.common.enums.LongArticleTextSimilarityStatusEnum;
@@ -191,12 +192,14 @@ public class RecallService implements ApplicationContextAware {
         List<Content> content = aigcWaitingPublishContentService.getAllContent(param);
         long t2 = System.currentTimeMillis();
         CostMonitor.logCost("Recall", "GetAllContents", t2 - t1);
+        ContentCountMonitor.logCount("Recall", "GetAllContents", Objects.isNull(content) ? 0 : content.size());
         if (whiteAccountList.contains(param.getAccountName())) {
 //            // 临时过滤文章视频不匹配content
 //            filterNotMatchContent(content);
             // 过滤仅保留审核通过content
             filterAuditPassContent(content);
         }
+        ContentCountMonitor.logCount("Recall", "Filter", Objects.isNull(content) ? 0 : content.size());
         if (CollectionUtils.isEmpty(content)) {
             FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
                     "内容召回失败\n"

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/recommend/DataDashboardController.java

@@ -57,6 +57,13 @@ public class DataDashboardController {
         }).start();
     }
 
+    @GetMapping("/export/contentFunnel")
+    public void contentFunnelExport(String dateStr) {
+        new Thread(() -> {
+            service.contentFunnelExport(dateStr);
+        }).start();
+    }
+
     @GetMapping("/export/dailySafeScore")
     public void dailySafeScore(String dateStr) {
         new Thread(() -> {

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -139,4 +139,62 @@
         </foreach>
     </select>
 
+    <select id="getProducePlanId" resultType="java.lang.String">
+        select input.input_source_value
+        from publish_plan plan
+         join publish_plan_input_source input on plan.id = input.plan_id
+        where plan.plan_status = 1 and plan.channel = 5
+    </select>
+
+    <select id="getCrawlerPlanByProducePlanIds" resultType="java.lang.String">
+        select input.input_source_value
+        from produce_plan plan
+         join produce_plan_input_source input on plan.id = input.plan_id
+        where plan.id in
+        <foreach collection="producePlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+    </select>
+
+    <select id="getCrawlerContentCountByCrawlerPlanIds" resultType="java.lang.Long">
+        select count(1)
+        from crawler_plan_result_rel
+        where plan_id in
+        <foreach collection="crawlerPlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and create_timestamp between #{start} and #{end}
+    </select>
+
+    <select id="getCrawlerSuccessPlanByCrawlerPlanIds" resultType="java.lang.String">
+        select distinct plan_id
+        from crawler_plan_result_rel
+        where plan_id in
+        <foreach collection="crawlerPlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and create_timestamp between #{start} and #{end}
+    </select>
+
+    <select id="getProduceContentCountByProducePlanIds" resultType="java.lang.Long">
+        select count(1)
+        from produce_plan_exe_record
+        where plan_id in
+        <foreach collection="producePlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and produce_timestamp between #{start} and #{end}
+    </select>
+
+    <select id="getProduceAuditPassCountByProducePlanIds" resultType="java.lang.Long">
+        select count(1)
+        from produce_plan_exe_record
+        where plan_id in
+        <foreach collection="producePlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and audit_status = 1
+        and audit_timestamp between #{start} and #{end}
+    </select>
+
 </mapper>

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -271,6 +271,12 @@
         select distinct gh_id from long_article_audit_delete where create_timestamp > 1736157600000 and status = 2 and fail_reason like 'reach max api daily quota%'
     </select>
 
+    <select id="countMatchSuccessCount" resultType="java.lang.Long">
+        select count(distinct content_id)
+        from long_articles_crawler_videos
+        where crawler_time between #{start} and #{end}
+    </select>
+
     <insert id="batchInsertLongArticleAuditDelete">
         insert into long_article_audit_delete
         (gh_id, msg_id, index, push_id, push_type, publish_content_id, status, create_timestamp)

long-article-recommend-service/src/main/resources/static/internal/articleDelete.html

@@ -236,7 +236,7 @@
                 'Content-Type': 'application/json',
                 'Access-Control-Allow-Origin': '*'
             },
-            body: JSON.stringify({title: title, ghId: ghId})
+            body: JSON.stringify({title: title, ghId: ghId, deleteFlag: 1})
         })
             .then(response => response.json())
             .then(data => {