metric log

etl-core/src/main/java/com/tzld/crawler/etl/common/enums/ExceptionEnum.java

@@ -9,7 +9,7 @@ package com.tzld.crawler.etl.common.enums;
 public enum ExceptionEnum {
 
     SYSTEM_ERROR(1000, "系统错误"),
-    DATA_ERROR(1001, "数据异常,请联系管理员"),
+    DATA_ERROR(1001, "数据异常"),
     PARAM_ERROR(1002, "参数不对"),
     INVOKE_VIDEOAPI_ERROR(1003, "调用 longvideo api 接口服务失败"),
     URL_FORBIDDEN(1004, "don't have permission to access the url on remote server."),

etl-core/src/main/java/com/tzld/crawler/etl/enums/MetricTypeEnum.java

@@ -0,0 +1,56 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.enums;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+
+/**
+ * @author ehlxr
+ * @since 2023-07-18 14:32.
+ */
+public enum MetricTypeEnum {
+    SUCCESS("success", "成功任务"),
+    DURATION("duration", "处理时长"),
+    DUPLICATED("duplicated", "重复视频数"),
+    FAIL("fail", "失败原因"),
+    INVALID_URL("invalidUrl", "抓取链接无效"),
+    DOWNLOAD_INTERRUPT("downloadInterrupt", "文件下载中断"),
+    VIDEO_SYNTHESIS_FAILED("videoSynthesisFailed", "音视频合成失败"),
+    UNKNOWN_EXCEPTION("unknownException", "系统处理异常"),
+    VIDEO_SEND_FAILED("videoSendFailed", "发布视频失败");
+
+    private final String type;
+    private final String desc;
+
+    MetricTypeEnum(String type, String desc) {
+        this.type = type;
+        this.desc = desc;
+    }
+
+    @JsonValue
+    public String getType() {
+        return type;
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/model/dto/MetricLogDto.java

@@ -0,0 +1,162 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.model.dto;
+
+import com.tzld.crawler.etl.enums.MetricTypeEnum;
+
+/**
+ * 监控度量日志
+ *
+ * @author ehlxr
+ * @since 2023-07-18 13:17.
+ */
+public class MetricLogDto {
+    /**
+     * 平台名
+     */
+    private String platform;
+    /**
+     * 模式名
+     */
+    private String strategy;
+    /**
+     * 数据类型码
+     */
+    private MetricTypeEnum type;
+    /**
+     * 信息
+     */
+    private String msg;
+
+    /**
+     * 数据
+     */
+    private Object data;
+
+    public static MetricLogDtoBuilder newBuiler() {
+        return new MetricLogDtoBuilder();
+    }
+
+    @Override
+    public String toString() {
+        return "MetricLogDto{" +
+                "platform='" + platform + '\'' +
+                ", strategy='" + strategy + '\'' +
+                ", type=" + type +
+                ", msg='" + msg + '\'' +
+                ", data=" + data +
+                '}';
+    }
+
+    public Object getData() {
+        return data;
+    }
+
+    public void setData(Object data) {
+        this.data = data;
+    }
+
+    public String getPlatform() {
+        return platform;
+    }
+
+    public void setPlatform(String platform) {
+        this.platform = platform;
+    }
+
+    public String getStrategy() {
+        return strategy;
+    }
+
+    public void setStrategy(String strategy) {
+        this.strategy = strategy;
+    }
+
+    public MetricTypeEnum getType() {
+        return type;
+    }
+
+    public void setType(MetricTypeEnum type) {
+        this.type = type;
+    }
+
+    public String getMsg() {
+        return msg;
+    }
+
+    public void setMsg(String msg) {
+        this.msg = msg;
+    }
+
+    public static final class MetricLogDtoBuilder {
+        private String platform;
+        private String strategy;
+        private MetricTypeEnum type;
+        private String msg;
+        private Object data;
+
+        private MetricLogDtoBuilder() {
+        }
+
+        public static MetricLogDtoBuilder aMetricLogDto() {
+            return new MetricLogDtoBuilder();
+        }
+
+        public MetricLogDtoBuilder platform(String platform) {
+            this.platform = platform;
+            return this;
+        }
+
+        public MetricLogDtoBuilder strategy(String strategy) {
+            this.strategy = strategy;
+            return this;
+        }
+
+        public MetricLogDtoBuilder type(MetricTypeEnum type) {
+            this.type = type;
+            return this;
+        }
+
+        public MetricLogDtoBuilder msg(String msg) {
+            this.msg = msg;
+            return this;
+        }
+
+        public MetricLogDtoBuilder data(Object data) {
+            this.data = data;
+            return this;
+        }
+
+        public MetricLogDto build() {
+            MetricLogDto metricLogDto = new MetricLogDto();
+            metricLogDto.setPlatform(platform);
+            metricLogDto.setStrategy(strategy);
+            metricLogDto.setType(type);
+            metricLogDto.setMsg(msg);
+            metricLogDto.setData(data);
+            return metricLogDto;
+        }
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/model/dto/StrategyDataDto.java

@@ -24,13 +24,13 @@
 
 package com.tzld.crawler.etl.model.dto;
 
-import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.model.vo.CrawlerEtlParam;
 
 /**
  * @author ehlxr
  * @since 2023-06-20 10:36.
  */
-public class StrategyDataDto extends CrawlerVideoVO {
+public class StrategyDataDto extends CrawlerEtlParam {
     private double titleScore;
 
     public double getTitleScore() {

etl-core/src/main/java/com/tzld/crawler/etl/model/vo/CrawlerVideoVO.java → etl-core/src/main/java/com/tzld/crawler/etl/model/vo/CrawlerEtlParam.java

@@ -5,7 +5,7 @@ import javax.validation.constraints.NotNull;
 /**
  * @author ehlxr
  */
-public class CrawlerVideoVO {
+public class CrawlerEtlParam {
     /**
      * 站内用户ID
      */
@@ -122,9 +122,14 @@ public class CrawlerVideoVO {
      */
     private String audioUrl;
 
+    /**
+     * MQ 消息ID
+     */
+    private String messageId;
+
     @Override
     public String toString() {
-        return "CrawlerVideoVO{" +
+        return "CrawlerEtlParam{" +
                 "userId=" + userId +
                 ", outUserId='" + outUserId + '\'' +
                 ", userName='" + userName + '\'' +
@@ -149,9 +154,18 @@ public class CrawlerVideoVO {
                 ", coverOssPath='" + coverOssPath + '\'' +
                 ", videoOssPath='" + videoOssPath + '\'' +
                 ", audioUrl='" + audioUrl + '\'' +
+                ", messageId='" + messageId + '\'' +
                 '}';
     }
 
+    public String getMessageId() {
+        return messageId;
+    }
+
+    public void setMessageId(String messageId) {
+        this.messageId = messageId;
+    }
+
     public String getAudioUrl() {
         return audioUrl;
     }

etl-core/src/main/java/com/tzld/crawler/etl/mq/EtlMQConsumer.java

@@ -30,7 +30,7 @@ import com.aliyun.mq.http.MQConsumer;
 import com.aliyun.mq.http.model.Message;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.model.vo.CrawlerEtlParam;
 import com.tzld.crawler.etl.service.EtlService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -86,15 +86,42 @@ public class EtlMQConsumer {
         new Thread(this::consumeMsg).start();
     }
 
+    private void consumeMsg() {
+        final MQConsumer consumer = mqClient.getConsumer(instanceId, topic, groupId, null);
+        do {
+            List<String> handles = new ArrayList<>();
+            try {
+                List<Message> messages = consumer.consumeMessage(10, 10);
+                if (messages == null || messages.isEmpty()) {
+                    log.info("No new message, continue");
+                    continue;
+                }
+
+                messages.forEach(message -> {
+                    log.info("Receive message: {} from topic: {}, group: {}", message, topic, groupId);
+                    handles.add(message.getReceiptHandle());
+                    CrawlerEtlParam param = JSONObject.parseObject(message.getMessageBodyString(), CrawlerEtlParam.class);
+                    param.setMessageId(message.getMessageId());
+                    priorityPool.execute(new RunnablePriority(param));
+                });
+            } catch (Throwable e) {
+                log.error("Consume message from topic: {}, group: {} error", topic, groupId, e);
+            }
+            if (!handles.isEmpty()) {
+                consumer.ackMessage(handles);
+            }
+        } while (true);
+    }
+
     class RunnablePriority implements Runnable, Comparable<RunnablePriority> {
-        private final CrawlerVideoVO video;
+        private final CrawlerEtlParam param;
 
-        public RunnablePriority(CrawlerVideoVO video) {
-            this.video = video;
+        public RunnablePriority(CrawlerEtlParam param) {
+            this.param = param;
         }
 
         public Integer getPriority() {
-            return priorityMap.getOrDefault(video.getPlatform(), -1);
+            return priorityMap.getOrDefault(param.getPlatform(), -1);
         }
 
         /**
@@ -116,45 +143,19 @@ public class EtlMQConsumer {
                 log.info("Thread pool info \nTaskCount: {}\nCompletedTaskCount: {}\nActiveCount: {}\nQueue size: {}\nQueue elements: {}",
                         priorityPool.getTaskCount(), priorityPool.getCompletedTaskCount(), priorityPool.getActiveCount(),
                         priorityPool.getQueue().size(), priorityPool.getQueue());
-                log.info("deal video: {} priority: {}", video, getPriority());
-                etlService.deal(video);
+                log.info("deal: {} priority: {}", param, getPriority());
+                etlService.deal(param);
                 log.info("Thread pool info \nTaskCount: {}\nCompletedTaskCount: {}\nActiveCount: {}\nQueue size: {}\nQueue elements: {}",
                         priorityPool.getTaskCount(), priorityPool.getCompletedTaskCount(), priorityPool.getActiveCount(),
                         priorityPool.getQueue().size(), priorityPool.getQueue());
             } catch (Exception e) {
-                log.error("deal video {} error.", video, e);
+                log.error("deal {} error.", param, e);
             }
         }
 
         @Override
         public String toString() {
-            return video.getPlatform() + ":" + video.getOutVideoId();
+            return param.getPlatform() + ":" + param.getOutVideoId();
         }
     }
-
-    private void consumeMsg() {
-        final MQConsumer consumer = mqClient.getConsumer(instanceId, topic, groupId, null);
-        do {
-            List<String> handles = new ArrayList<>();
-            try {
-                List<Message> messages = consumer.consumeMessage(10, 10);
-                if (messages == null || messages.isEmpty()) {
-                    log.info("No new message, continue");
-                    continue;
-                }
-
-                messages.forEach(message -> {
-                    log.info("Receive message: {} from topic: {}, group: {}", message, topic, groupId);
-                    handles.add(message.getReceiptHandle());
-                    CrawlerVideoVO video = JSONObject.parseObject(message.getMessageBodyString(), CrawlerVideoVO.class);
-                    priorityPool.execute(new RunnablePriority(video));
-                });
-            } catch (Throwable e) {
-                log.error("Consume message from topic: {}, group: {} error", topic, groupId, e);
-            }
-            if (!handles.isEmpty()) {
-                consumer.ackMessage(handles);
-            }
-        } while (true);
-    }
 }

etl-core/src/main/java/com/tzld/crawler/etl/service/EtlService.java

@@ -24,14 +24,14 @@
 
 package com.tzld.crawler.etl.service;
 
-import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.model.vo.CrawlerEtlParam;
 
 /**
  * @author ehlxr
  * @since 2023-06-09 15:22.
  */
 public interface EtlService {
-    void deal(CrawlerVideoVO video);
+    void deal(CrawlerEtlParam video);
 }
 
 

etl-core/src/main/java/com/tzld/crawler/etl/service/SlsService.java

@@ -27,6 +27,8 @@ package com.tzld.crawler.etl.service;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import com.tzld.commons.aliyun.log.AliyunLogManager;
+import com.tzld.crawler.etl.model.dto.MetricLogDto;
+import com.tzld.crawler.etl.util.JsonUtil;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
@@ -45,6 +47,8 @@ public class SlsService {
     private final AliyunLogManager aliyunLogManager;
     @Value("${aliyun.log.logstore.crawler:}")
     private String crawlerLogstore;
+    @Value("${aliyun.log.logstore.metric:}")
+    private String metricLogstore;
     @Value("${aliyun.log.project:}")
     private String logProject;
 
@@ -59,6 +63,15 @@ public class SlsService {
         pool = Executors.newFixedThreadPool(2);
     }
 
+    public void metric(MetricLogDto logDto, String logStore) {
+        Map<String, Object> map = JsonUtil.obj2Map(logDto);
+        pool.execute(() -> aliyunLogManager.sendLog(logProject, logStore, "", map));
+    }
+
+    public void metric(MetricLogDto logDto) {
+        metric(logDto, metricLogstore);
+    }
+
     public void log(Map<String, Object> param) {
         HashMap<String, Object> map = Maps.newHashMap(param);
         map.put("timestamp", System.currentTimeMillis() / 1000);

etl-core/src/main/java/com/tzld/crawler/etl/service/impl/EtlServiceImpl.java

@@ -25,6 +25,7 @@
 package com.tzld.crawler.etl.service.impl;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.base.Stopwatch;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.tzld.commons.aliyun.oss.AliyunOssManager;
@@ -36,13 +37,15 @@ import com.tzld.crawler.etl.common.exception.CommonException;
 import com.tzld.crawler.etl.dao.mapper.CrawlerUserV3Mapper;
 import com.tzld.crawler.etl.dao.mapper.CrawlerVideoMapper;
 import com.tzld.crawler.etl.dao.mapper.ext.CrawlerVideoExtMapper;
+import com.tzld.crawler.etl.enums.MetricTypeEnum;
+import com.tzld.crawler.etl.model.dto.MetricLogDto;
 import com.tzld.crawler.etl.model.dto.StrategyDataDto;
 import com.tzld.crawler.etl.model.dto.VideoInfoDto;
 import com.tzld.crawler.etl.model.param.CrawlerVideoSendParam;
 import com.tzld.crawler.etl.model.po.CrawlerUserV3;
 import com.tzld.crawler.etl.model.po.CrawlerUserV3Example;
 import com.tzld.crawler.etl.model.po.CrawlerVideo;
-import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.model.vo.CrawlerEtlParam;
 import com.tzld.crawler.etl.model.vo.WxVideoVO;
 import com.tzld.crawler.etl.service.EtlService;
 import com.tzld.crawler.etl.service.SlsService;
@@ -114,15 +117,16 @@ public class EtlServiceImpl implements EtlService {
     private String ffprobePath;
     @Value("${ffprobe.path:ffmpeg}")
     private String ffmpegPath;
-
     @ApolloJsonValue("${fail.retry.times:{}}")
     private Map<String, Integer> failRetryTimes;
 
     private Executor pool;
 
+    private final ThreadLocal<CrawlerEtlParam> tlParam = new ThreadLocal<>();
+
     public EtlServiceImpl(StrategyHandlerService strategyHandlerService, AliyunOssManager aliyunOssManager,
-            LongVideoFeign longVideoFeign, CrawlerVideoMapper crawlerVideoMapper, SlsService slsService,
-            CrawlerVideoExtMapper crawlerVideoExtMapper, CrawlerUserV3Mapper crawlerUserV3Mapper) {
+                          LongVideoFeign longVideoFeign, CrawlerVideoMapper crawlerVideoMapper, SlsService slsService,
+                          CrawlerVideoExtMapper crawlerVideoExtMapper, CrawlerUserV3Mapper crawlerUserV3Mapper) {
         this.strategyHandlerService = strategyHandlerService;
         this.aliyunOssManager = aliyunOssManager;
         this.longVideoFeign = longVideoFeign;
@@ -133,7 +137,9 @@ public class EtlServiceImpl implements EtlService {
     }
 
     @Override
-    public void deal(CrawlerVideoVO param) {
+    public void deal(CrawlerEtlParam param) {
+        tlParam.set(param);
+        Stopwatch sw = Stopwatch.createStarted();
         CrawlerVideo crawlerVideo = new CrawlerVideo();
         BeanUtils.copyProperties(param, crawlerVideo);
         long id = 0L;
@@ -141,14 +147,13 @@ public class EtlServiceImpl implements EtlService {
             // 参数校验
             String errorMessage = CustomValidator.validate(param);
             if (!Strings.isNullOrEmpty(errorMessage)) {
-                log.error("param validate failed. {}", errorMessage);
-                return;
+                throw new CommonException(ExceptionEnum.PARAM_ERROR, "param validate failed." + errorMessage);
             }
 
             // 保存数据库（去重校验）
             id = save2db(crawlerVideo);
             if (id <= 0) {
-                return;
+                throw new CommonException(ExceptionEnum.DATA_ERROR, "save2db failed.");
             }
 
             // 策略应用
@@ -157,8 +162,7 @@ public class EtlServiceImpl implements EtlService {
 
             StrategyDataDto data = strategyHandlerService.execute(strategies, param);
             if (data == null) {
-                log.info("{} filter by strategies {}", param, strategies);
-                return;
+                throw new CommonException(ExceptionEnum.DATA_ERROR, param + " filter by strategies " + strategies);
             }
 
             // 音频、视频文件下载、合成，上传 OSS、清理视频信息
@@ -177,6 +181,8 @@ public class EtlServiceImpl implements EtlService {
 
             // 视频写入飞书
             async2Feishu(data, videoId);
+
+            metric(MetricTypeEnum.SUCCESS);
         } catch (Exception e) {
             log.error("etl server deal {} failed.", param, e);
             // 回滚数据
@@ -184,6 +190,9 @@ public class EtlServiceImpl implements EtlService {
                 crawlerVideoMapper.deleteByPrimaryKey(id);
             }
             throw new CommonException(ExceptionEnum.SYSTEM_ERROR, "etl server deal error: " + e.getMessage());
+        } finally {
+            metric(MetricTypeEnum.DURATION, sw.stop());
+            tlParam.remove();
         }
     }
 
@@ -198,6 +207,7 @@ public class EtlServiceImpl implements EtlService {
                 // 根据站外视频 ID 唯一约束 key 做去重校验
                 log.info("out video id {} of platform {} strategy {} has exist!", crawlerVideo.getOutVideoId(),
                         crawlerVideo.getPlatform(), crawlerVideo.getStrategy());
+                metric(MetricTypeEnum.DUPLICATED);
                 return Pair.of(false, 0L);
             }
         }, "video2db", String.format("save video info [%s] to db", crawlerVideo));
@@ -437,6 +447,20 @@ public class EtlServiceImpl implements EtlService {
         retryFuncR(c -> Pair.of(func.test(c), null), t, type, errorMsg);
     }
 
+    private void metric(MetricTypeEnum typeEnum, Object data) {
+        CrawlerEtlParam param = tlParam.get();
+        slsService.metric(MetricLogDto.newBuiler()
+                .platform(param.getPlatform())
+                .strategy(param.getStrategy())
+                .type(typeEnum)
+                .data(data)
+                .msg(param.toString()).build());
+    }
+
+    private void metric(MetricTypeEnum typeEnum) {
+        metric(typeEnum, "");
+    }
+
     @PostConstruct
     public void init() {
         pool = Executors.newFixedThreadPool(1);

etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/StrategyAbstractHandler.java

@@ -25,7 +25,7 @@
 package com.tzld.crawler.etl.service.strategy;
 
 import com.tzld.crawler.etl.model.dto.StrategyDataDto;
-import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.model.vo.CrawlerEtlParam;
 
 /**
  * @author ehlxr
@@ -35,7 +35,7 @@ public abstract class StrategyAbstractHandler {
     /**
      * 业务处理实现抽象方法
      */
-    public abstract StrategyDataDto execute(CrawlerVideoVO param);
+    public abstract StrategyDataDto execute(CrawlerEtlParam param);
 
 
     /**

etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/StrategyHandlerService.java

@@ -25,7 +25,7 @@
 package com.tzld.crawler.etl.service.strategy;
 
 import com.tzld.crawler.etl.model.dto.StrategyDataDto;
-import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.model.vo.CrawlerEtlParam;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeansException;
@@ -68,7 +68,7 @@ public class StrategyHandlerService implements ApplicationContextAware {
     /**
      * 业务处理类的路由方法
      */
-    public StrategyDataDto execute(List<String> strategies, CrawlerVideoVO param) {
+    public StrategyDataDto execute(List<String> strategies, CrawlerEtlParam param) {
         StrategyDataDto result = null;
         for (String strategy : strategies) {
             StrategyAbstractHandler handler = HANDLER_MAP.get(strategy);

etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/handler/TitleScoreHandler.java

@@ -28,7 +28,7 @@ import com.alibaba.fastjson.JSONArray;
 import com.huaban.analysis.jieba.JiebaSegmenter;
 import com.huaban.analysis.jieba.SegToken;
 import com.tzld.crawler.etl.model.dto.StrategyDataDto;
-import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.model.vo.CrawlerEtlParam;
 import com.tzld.crawler.etl.service.strategy.StrategyAbstractHandler;
 import com.tzld.crawler.etl.util.FeishuUtils;
 import org.slf4j.Logger;
@@ -76,7 +76,7 @@ public class TitleScoreHandler extends StrategyAbstractHandler {
     private List<String> stopWorlds = new ArrayList<>();
 
     @Override
-    public StrategyDataDto execute(CrawlerVideoVO param) {
+    public StrategyDataDto execute(CrawlerEtlParam param) {
         double score = 0;
         StrategyDataDto result = new StrategyDataDto();
         BeanUtils.copyProperties(param, result);

etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/handler/VideoFilterHandler.java

@@ -25,7 +25,7 @@
 package com.tzld.crawler.etl.service.strategy.handler;
 
 import com.tzld.crawler.etl.model.dto.StrategyDataDto;
-import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.model.vo.CrawlerEtlParam;
 import com.tzld.crawler.etl.service.strategy.StrategyAbstractHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,7 +40,7 @@ public class VideoFilterHandler extends StrategyAbstractHandler {
     private static final Logger log = LoggerFactory.getLogger(VideoFilterHandler.class);
 
     @Override
-    public StrategyDataDto execute(CrawlerVideoVO param) {
+    public StrategyDataDto execute(CrawlerEtlParam param) {
         log.info("video filter {}", param);
         return null;
     }

etl-core/src/main/java/com/tzld/crawler/etl/util/JsonUtil.java

@@ -10,6 +10,7 @@ import com.tzld.crawler.etl.common.base.Constant;
 
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -147,6 +148,11 @@ public class JsonUtil {
         }
     }
 
+    public static HashMap<String, Object> obj2Map(Object obj) {
+        return om().convertValue(obj, new TypeReference<HashMap<String, Object>>() {
+        });
+    }
+
     public static void main(String[] args) {
         try {
             User user1 = new User();

etl-server/src/main/java/com/tzld/crawler/etl/controller/IndexController.java

@@ -1,7 +1,7 @@
 package com.tzld.crawler.etl.controller;
 
 import com.alibaba.fastjson.JSONObject;
-import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.model.vo.CrawlerEtlParam;
 import com.tzld.crawler.etl.service.EtlService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -31,12 +31,12 @@ public class IndexController {
 
     public static void main(String[] args) {
         String s = "{\"user_id\": 6281907,\"out_user_id\": \"53322270\",\"platform\": \"xiaoniangao\",\"strategy\": \"author\",\"out_video_id\": \"5067863383\",\"video_title\": \"早上起床前念四句话，身体有可能变得更健康\uD83D\uDC8E\",\"cover_url\": \"https://cdn-xphoto2.xiaoniangao.cn/5067863386@690w_385h_0e_1pr%7C690x385-5rc_0r.jpg?OSSAccessKeyId=LTAI4G2W1FsgwzAWYpPoB3v6&Expires=1688140805&Signature=F%2BIn%2FdAfjM4UWnG%2F96qAUcHWhpI%3D\",\"video_url\": \"http://cdn-xalbum-baishan.xiaoniangao.cn/5067863383?Expires=1704038400&OSSAccessKeyId=LTAI5tB7cRkYiqHcTdkVprwb&Signature=kD2sBhazSa4L5q/XolJ8BkxW6SI%3D\",\"duration\": 40,\"publish_time\": \"2023-06-08 23:01:47\",\"play_cnt\": 602,\"like_cnt\": 0,\"share_cnt\": 0,\"collection_cnt\": 0,\"comment_cnt\": 0,\"crawler_rule\": {\"period\": {\"max\": 3,\"min\": 3},\"duration\": {\"max\": 999999999999999,\"min\": 40},\"play_cnt\": {\"max\": 999999999999999,\"min\": 500}},\"width\": 450,\"height\": 254}";
-        System.out.println(JSONObject.parseObject(s, CrawlerVideoVO.class));
+        System.out.println(JSONObject.parseObject(s, CrawlerEtlParam.class));
     }
 
     @PostMapping("/etl")
-    public String deal(@RequestBody CrawlerVideoVO video) {
-        etlService.deal(video);
+    public String deal(@RequestBody CrawlerEtlParam param) {
+        etlService.deal(param);
         return "ok";
     }
 }

etl-server/src/main/resources/application-dev.yml

@@ -15,6 +15,7 @@ aliyun:
     project: crawler-log-dev
     logstore:
       crawler: crawler-log-dev
+      metric: etl-metric
 
 longvideo:
   feign:

etl-server/src/main/resources/application-prod.yml

@@ -15,6 +15,7 @@ aliyun:
     project: crawler-log-prod
     logstore:
       crawler: crawler-log-prod
+      metric: etl-metric
 
 longvideo:
   feign: