Merge branch 'wyp/0124-contentCountMonitor' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/ContentCountMonitor.java

@@ -0,0 +1,16 @@
+package com.tzld.longarticle.recommend.server.common;
+
+import com.tzld.longarticle.recommend.server.common.constant.LogConstants;
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
+
+@Slf4j
+public class ContentCountMonitor {
+
+    public static void logCount(String type, String name, Integer count) {
+        Marker marker = MarkerFactory.getMarker(LogConstants.MARKER_RECALL);
+        log.info(marker, "{}.{} count: {}", type, name, count);
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/RecommendLoghubAppender.java

@@ -115,6 +115,8 @@ public class RecommendLoghubAppender<E> extends UnsynchronizedAppenderBase<E> {
             appendPlainEvent(eventObject);
         } else if (LogConstants.MARKER_COST.equals(event.getMarker().getName())) {
             appendCostEvent(event);
+        } else if (LogConstants.MARKER_RECALL.equals(event.getMarker().getName())) {
+            appendRecallEvent(event);
         }
     }
 
@@ -153,6 +155,41 @@ public class RecommendLoghubAppender<E> extends UnsynchronizedAppenderBase<E> {
         }
     }
 
+    private void appendRecallEvent(LoggingEvent event) {
+        List<LogItem> logItems = new ArrayList();
+        LogItem item = new LogItem();
+        logItems.add(item);
+        item.SetTime((int) (event.getTimeStamp() / 1000L));
+
+        item.PushBack("thread", event.getThreadName());
+        Object[] args = event.getArgumentArray();
+        try {
+            item.PushBack("type", (String) args[0]);
+            item.PushBack("name", (String) args[1]);
+            item.PushBack("count", String.valueOf(args[2]));
+        } catch (Exception e) {
+            this.addError("invalid event argument ", e);
+        }
+
+        Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
+            event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
+                return Arrays.stream(f.split(",")).anyMatch((i) -> {
+                    return i.equals(v.getKey());
+                });
+            }).forEach((map) -> {
+                item.PushBack((String) map.getKey(), (String) map.getValue());
+            });
+        });
+
+        try {
+            this.producer.send(this.projectConfig.getProject(), this.logStore, "ContentCountMonitor", this.source, logItems,
+                    new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
+                            this.topic, this.source, logItems));
+        } catch (Exception e) {
+            this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, e);
+        }
+    }
+
     private void appendPlainEvent(E eventObject) {
         LoggingEvent event = (LoggingEvent) eventObject;
         List<LogItem> logItems = new ArrayList();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/constant/LogConstants.java

@@ -2,4 +2,6 @@ package com.tzld.longarticle.recommend.server.common.constant;
 
 public class LogConstants {
     public static final String MARKER_COST = "COST";
+
+    public static final String MARKER_RECALL = "RECALL";
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/filter/FilterService.java

@@ -1,5 +1,6 @@
 package com.tzld.longarticle.recommend.server.service.recommend.filter;
 
+import com.tzld.longarticle.recommend.server.common.ContentCountMonitor;
 import com.tzld.longarticle.recommend.server.common.CostMonitor;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
@@ -46,6 +47,8 @@ public class FilterService {
                     FilterResult ret = strategy.filter(param);
                     long t2 = System.currentTimeMillis();
                     CostMonitor.logCost("Filter", strategy.getClass().getSimpleName(), t2 - t1);
+                    ContentCountMonitor.logCount("Filter", strategy.getClass().getSimpleName(),
+                            Objects.isNull(ret.getContentIds()) ? 0 : ret.getContentIds().size());
                     return ret;
                 } finally {
                     cdl.countDown();

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/RankService.java

@@ -7,6 +7,7 @@ import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.model.ConfigChangeEvent;
 import com.ctrip.framework.apollo.spring.annotation.ApolloConfigChangeListener;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.ContentCountMonitor;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputSourceTypesEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
@@ -96,6 +97,7 @@ public class RankService implements InitializingBean {
 
     public static void printSortLog(String strategy, String accountName, String position, List<Content> contentList) {
         JSONArray jsonArray = new JSONArray();
+        ContentCountMonitor.logCount("Rank", position, Objects.isNull(contentList) ? 0 : contentList.size());
         if (CollectionUtil.isEmpty(contentList)) {
             return;
         }
@@ -176,6 +178,7 @@ public class RankService implements InitializingBean {
         if (CollectionUtils.isNotEmpty(pool)) {
             pool = contentSourceTypeFilter(param.getStrategy(), pool, 3);
         }
+
         if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
             RankService.printSortLog(strategy, param.getAccountName(), "3-8", pool);
             result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV5Strategy.java

@@ -2,6 +2,7 @@ package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.ContentCountMonitor;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -6,6 +6,7 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Lists;
+import com.tzld.longarticle.recommend.server.common.ContentCountMonitor;
 import com.tzld.longarticle.recommend.server.common.CostMonitor;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.common.enums.LongArticleTextSimilarityStatusEnum;
@@ -191,12 +192,14 @@ public class RecallService implements ApplicationContextAware {
         List<Content> content = aigcWaitingPublishContentService.getAllContent(param);
         long t2 = System.currentTimeMillis();
         CostMonitor.logCost("Recall", "GetAllContents", t2 - t1);
+        ContentCountMonitor.logCount("Recall", "GetAllContents", Objects.isNull(content) ? 0 : content.size());
         if (whiteAccountList.contains(param.getAccountName())) {
 //            // 临时过滤文章视频不匹配content
 //            filterNotMatchContent(content);
             // 过滤仅保留审核通过content
             filterAuditPassContent(content);
         }
+        ContentCountMonitor.logCount("Recall", "Filter", Objects.isNull(content) ? 0 : content.size());
         if (CollectionUtils.isEmpty(content)) {
             FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
                     "内容召回失败\n"