Merge branch 'feature/20241202-improve-perf' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/CostMonitor.java

@@ -0,0 +1,14 @@
+package com.tzld.longarticle.recommend.server.common;
+
+import com.tzld.longarticle.recommend.server.common.constant.LogConstants;
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
+
+@Slf4j
+public class CostMonitor {
+    public static void logCost(String type, String name, long cost) {
+        Marker marker = MarkerFactory.getMarker(LogConstants.MARKER_COST);
+        log.info(marker, "{}.{} cost: {}", type, name, cost);
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/RecommendLoghubAppender.java

@@ -13,6 +13,7 @@ import com.aliyun.openservices.aliyun.log.producer.ProducerConfig;
 import com.aliyun.openservices.aliyun.log.producer.ProjectConfig;
 import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException;
 import com.aliyun.openservices.log.common.LogItem;
+import com.tzld.longarticle.recommend.server.common.constant.LogConstants;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
@@ -103,62 +104,105 @@ public class RecommendLoghubAppender<E> extends UnsynchronizedAppenderBase<E> {
         } catch (Exception var3) {
             this.addError("Failed to append event.", var3);
         }
-
     }
 
     private void appendEvent(E eventObject) {
-        if (eventObject instanceof LoggingEvent) {
-            LoggingEvent event = (LoggingEvent) eventObject;
-            List<LogItem> logItems = new ArrayList();
-            LogItem item = new LogItem();
-            logItems.add(item);
-            item.SetTime((int) (event.getTimeStamp() / 1000L));
-            if (this.formatter != null) {
-                DateTime dateTime = new DateTime(event.getTimeStamp());
-                item.PushBack("time", dateTime.toString(this.formatter));
-            } else {
-                Instant instant = Instant.ofEpochMilli(event.getTimeStamp());
-                item.PushBack("time", this.formatter1.format(instant));
-            }
-
-            item.PushBack("level", event.getLevel().toString());
-            item.PushBack("thread", event.getThreadName());
-            StackTraceElement[] caller = event.getCallerData();
-            if (caller != null && caller.length > 0) {
-                item.PushBack("location", caller[0].toString());
-            }
-
-            String message = event.getFormattedMessage();
-            item.PushBack("message", message);
-            IThrowableProxy iThrowableProxy = event.getThrowableProxy();
-            if (iThrowableProxy != null) {
-                String throwable = this.getExceptionInfo(iThrowableProxy);
-                throwable = throwable + this.fullDump(event.getThrowableProxy().getStackTraceElementProxyArray());
-                item.PushBack("throwable", throwable);
-            }
-
-            if (this.encoder != null) {
-                item.PushBack("log", new String(this.encoder.encode(eventObject)));
-            }
-
-            Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
-                event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
-                    return Arrays.stream(f.split(",")).anyMatch((i) -> {
-                        return i.equals(v.getKey());
-                    });
-                }).forEach((map) -> {
-                    item.PushBack((String) map.getKey(), (String) map.getValue());
+        if (!(eventObject instanceof LoggingEvent)) {
+            return;
+        }
+        LoggingEvent event = (LoggingEvent) eventObject;
+        if (event.getMarker() == null) {
+            appendPlainEvent(eventObject);
+        } else if (LogConstants.MARKER_COST.equals(event.getMarker().getName())) {
+            appendCostEvent(event);
+        }
+    }
+
+    private void appendCostEvent(LoggingEvent event) {
+        List<LogItem> logItems = new ArrayList();
+        LogItem item = new LogItem();
+        logItems.add(item);
+        item.SetTime((int) (event.getTimeStamp() / 1000L));
+
+        item.PushBack("thread", event.getThreadName());
+        Object[] args = event.getArgumentArray();
+        try {
+            item.PushBack("type", (String) args[0]);
+            item.PushBack("name", (String) args[1]);
+            item.PushBack("cost", String.valueOf(args[2]));
+        } catch (Exception e) {
+            this.addError("invalid event argument ", e);
+        }
+
+        Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
+            event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
+                return Arrays.stream(f.split(",")).anyMatch((i) -> {
+                    return i.equals(v.getKey());
                 });
+            }).forEach((map) -> {
+                item.PushBack((String) map.getKey(), (String) map.getValue());
             });
+        });
+
+        try {
+            this.producer.send(this.projectConfig.getProject(), this.logStore, "CostMonitor", this.source, logItems,
+                    new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
+                            this.topic, this.source, logItems));
+        } catch (Exception e) {
+            this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, e);
+        }
+    }
+
+    private void appendPlainEvent(E eventObject) {
+        LoggingEvent event = (LoggingEvent) eventObject;
+        List<LogItem> logItems = new ArrayList();
+        LogItem item = new LogItem();
+        logItems.add(item);
+        item.SetTime((int) (event.getTimeStamp() / 1000L));
+        if (this.formatter != null) {
+            DateTime dateTime = new DateTime(event.getTimeStamp());
+            item.PushBack("time", dateTime.toString(this.formatter));
+        } else {
+            Instant instant = Instant.ofEpochMilli(event.getTimeStamp());
+            item.PushBack("time", this.formatter1.format(instant));
+        }
+
+        item.PushBack("level", event.getLevel().toString());
+        item.PushBack("thread", event.getThreadName());
+        StackTraceElement[] caller = event.getCallerData();
+        if (caller != null && caller.length > 0) {
+            item.PushBack("location", caller[0].toString());
+        }
+
+        String message = event.getFormattedMessage();
+        item.PushBack("message", message);
+        IThrowableProxy iThrowableProxy = event.getThrowableProxy();
+        if (iThrowableProxy != null) {
+            String throwable = this.getExceptionInfo(iThrowableProxy);
+            throwable = throwable + this.fullDump(event.getThrowableProxy().getStackTraceElementProxyArray());
+            item.PushBack("throwable", throwable);
+        }
+
+        if (this.encoder != null) {
+            item.PushBack("log", new String(this.encoder.encode(eventObject)));
+        }
 
-            try {
-                this.producer.send(this.projectConfig.getProject(), this.logStore, this.topic, this.source, logItems,
-                        new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
-                                this.topic, this.source, logItems));
-            } catch (Exception var9) {
-                this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, var9);
-            }
+        Optional.ofNullable(this.mdcFields).ifPresent((f) -> {
+            event.getMDCPropertyMap().entrySet().stream().filter((v) -> {
+                return Arrays.stream(f.split(",")).anyMatch((i) -> {
+                    return i.equals(v.getKey());
+                });
+            }).forEach((map) -> {
+                item.PushBack((String) map.getKey(), (String) map.getValue());
+            });
+        });
 
+        try {
+            this.producer.send(this.projectConfig.getProject(), this.logStore, this.topic, this.source, logItems,
+                    new RecommendLoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore,
+                            this.topic, this.source, logItems));
+        } catch (Exception var9) {
+            this.addError("Failed to send log, project=" + this.project + ", logStore=" + this.logStore + ", topic=" + this.topic + ", source=" + this.source + ", logItem=" + logItems, var9);
         }
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/constant/LogConstants.java

@@ -0,0 +1,5 @@
+package com.tzld.longarticle.recommend.server.common.constant;
+
+public class LogConstants {
+    public static final String MARKER_COST = "COST";
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/aigc/AIGCWaitingPublishContentService.java

@@ -29,7 +29,6 @@ public class AIGCWaitingPublishContentService {
 
 
     public List<Content> getAllContent(RecallParam param) {
-        long start = System.currentTimeMillis();
         List<Content> result = new ArrayList<>();
         JSONObject bodyParam = new JSONObject();
         JSONObject bodyParamParams = new JSONObject();
@@ -58,7 +57,6 @@ public class AIGCWaitingPublishContentService {
         } catch (Exception e) {
             log.error("getAllContent error", e);
         }
-        log.info("getAllContent耗时：{}", System.currentTimeMillis() - start);
         return result;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -2,6 +2,7 @@ package com.tzld.longarticle.recommend.server.service.recommend.recall;
 
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
+import com.tzld.longarticle.recommend.server.common.CostMonitor;
 import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
@@ -108,15 +109,12 @@ public class RecallService implements ApplicationContextAware {
     }
 
     public RecallResult recall(RecallParam param) {
-        long t1 = System.currentTimeMillis();
         List<RecallResult.RecallData> results = new ArrayList<>();
         log.info("RecallParam {}", JSONUtils.toJson(param));
         List<RecallStrategy> strategies = getRecallStrategy(param);
         log.info("RecallStrategy {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies,
                 s -> s.getClass().getSimpleName())));
         List<Content> content = getAllContent(param);
-        long t2 = System.currentTimeMillis();
-        log.info("recall account:{} get content:{}", param.getAccountName(), t2 - t1);
         if (CollectionUtils.isEmpty(content)) {
             return new RecallResult(results);
         }
@@ -164,7 +162,10 @@ public class RecallService implements ApplicationContextAware {
     }
 
     private List<Content> getAllContent(RecallParam param) {
+        long t1 = System.currentTimeMillis();
         List<Content> content = aigcWaitingPublishContentService.getAllContent(param);
+        long t2 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "GetAllContents", t2 - t1);
         if (CollectionUtils.isEmpty(content)) {
             FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
                     "内容召回失败\n"
@@ -175,8 +176,12 @@ public class RecallService implements ApplicationContextAware {
         }
         // category 查询
         setContentCategory(content);
+        long t3 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "GetCategory", t3 - t2);
         // 标题历史均值
         setTitleAvgViewCount(content, param.getGhId(), param.getType());
+        long t4 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "SetAvgViewCount", t4 - t3);
         return content;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/strategy/DefaultRecallStrategy.java

@@ -1,5 +1,6 @@
 package com.tzld.longarticle.recommend.server.service.recommend.recall.strategy;
 
+import com.tzld.longarticle.recommend.server.common.CostMonitor;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.recommend.filter.FilterResult;
@@ -24,15 +25,20 @@ public class DefaultRecallStrategy implements RecallStrategy {
 
     @Override
     public RecallResult.RecallData recall(RecallParam param) {
+        long t1 = System.currentTimeMillis();
         List<Content> content = param.getContent();
         // 处理 content
         FilterParam filterParam = FilterParamFactory.create(param, content);
         FilterResult filterResult = filterService.filter(filterParam);
+        long t2 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "ProcessFilter", t2 - t1);
         // 处理 content
         RecallResult.RecallData result = new RecallResult.RecallData();
         result.setContents(content.stream().filter(o -> filterResult.getContentIds().contains(o.getId()))
                 .collect(Collectors.toList()));
         result.setFilterContents(filterResult.getFilterContent());
+        long t3 = System.currentTimeMillis();
+        CostMonitor.logCost("Recall", "SetResults", t3 - t2);
         return result;
     }