etl init

.gitignore

@@ -0,0 +1,22 @@
+# ---> Java
+*.class
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.ear
+target/
+.idea/
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+*.iml
+*.DS_Store
+
+# eclipse
+*.project
+.settings
+*.classpath

README.md

@@ -0,0 +1 @@
+# crawler ETL 

etl-core/pom.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.tzld.piaoquan</groupId>
+        <artifactId>crawler-etl</artifactId>
+        <version>1.0.0</version>
+    </parent>
+    <artifactId>etl-core</artifactId>
+    <name>crawler-etl-core</name>
+    <description>crawler-etl-core</description>
+
+</project>

etl-core/src/main/java/com/tzld/crawler/etl/common/base/CommonRequest.java

@@ -0,0 +1,40 @@
+package com.tzld.crawler.etl.common.base;
+
+import com.tzld.crawler.etl.model.dto.BaseInfoDto;
+
+/**
+ * 请求参数
+ *
+ * @author supeng
+ */
+public class CommonRequest<T> {
+    /**
+     * 基础信息
+     */
+    BaseInfoDto baseInfo;
+    /**
+     * 请求参数
+     */
+    T params;
+
+    public BaseInfoDto getBaseInfo() {
+        return baseInfo;
+    }
+
+    public void setBaseInfo(BaseInfoDto baseInfo) {
+        this.baseInfo = baseInfo;
+    }
+
+    public T getParams() {
+        return params;
+    }
+
+    public void setParams(T params) {
+        this.params = params;
+    }
+
+    @Override
+    public String toString() {
+        return "CommonRequest{" + "baseInfo=" + baseInfo + ", params=" + params + '}';
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/common/base/CommonResponse.java

@@ -0,0 +1,114 @@
+package com.tzld.crawler.etl.common.base;
+
+import com.tzld.crawler.etl.common.enums.ExceptionEnum;
+
+/**
+ * Common Response
+ */
+public class CommonResponse<T> {
+
+    private static final int SUCCESS_CODE = 0;
+    private static final String SUCCESS_MSG = "success";
+
+    /**
+     * 返回状态码，0 表示业务成功
+     */
+    private int code = 0;
+    /**
+     * 返回消息
+     */
+    private String msg = "success";
+    /**
+     * 业务成功时返回数据
+     */
+    private T data;
+    /**
+     * 重定向
+     */
+    private String redirect;
+
+    public static <T> CommonResponse<T> success() {
+        CommonResponse<T> commonResponse = new CommonResponse<>();
+        commonResponse.setCode(SUCCESS_CODE);
+        commonResponse.setMsg(SUCCESS_MSG);
+        return commonResponse;
+    }
+
+    public static <T> CommonResponse<T> success(T data) {
+        CommonResponse<T> commonResponse = new CommonResponse<>();
+        commonResponse.setCode(SUCCESS_CODE);
+        commonResponse.setMsg(SUCCESS_MSG);
+        commonResponse.setData(data);
+        return commonResponse;
+    }
+
+    public static <T> CommonResponse<T> create() {
+        return create(SUCCESS_CODE, SUCCESS_MSG, null);
+    }
+
+    public static <T> CommonResponse<T> create(T data) {
+        return create(SUCCESS_CODE, SUCCESS_MSG, data);
+    }
+
+    public static <T> CommonResponse<T> create(int code, String msg) {
+        return create(code, msg, null);
+    }
+
+    public static <T> CommonResponse<T> create(ExceptionEnum exceptionEnum) {
+        return create(exceptionEnum.getCode(), exceptionEnum.getMsg(), null);
+    }
+
+    public static <T> CommonResponse<T> create(int code, String msg, T data) {
+        CommonResponse<T> commonResponse = new CommonResponse<>();
+        commonResponse.setCode(code);
+        commonResponse.setMsg(msg);
+        commonResponse.setData(data);
+        return commonResponse;
+    }
+
+    public boolean isSuccess() {
+        return this.code == SUCCESS_CODE;
+    }
+
+    public int getCode() {
+        return code;
+    }
+
+    public void setCode(int code) {
+        this.code = code;
+    }
+
+    public String getMsg() {
+        return msg;
+    }
+
+    public void setMsg(String msg) {
+        this.msg = msg;
+    }
+
+    public T getData() {
+        return data;
+    }
+
+    public void setData(T data) {
+        this.data = data;
+    }
+
+    public String getRedirect() {
+        return redirect;
+    }
+
+    public void setRedirect(String redirect) {
+        this.redirect = redirect;
+    }
+
+    @Override
+    public String toString() {
+        return "CommonResponse{" +
+                "code=" + code +
+                ", msg='" + msg + '\'' +
+                ", data=" + data +
+                ", redirect='" + redirect + '\'' +
+                '}';
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/common/base/Constant.java

@@ -0,0 +1,21 @@
+package com.tzld.crawler.etl.common.base;
+
+/**
+ * 常量
+ *
+ * @author supeng
+ * @date 2020/08/19
+ */
+public interface Constant {
+    /**
+     * traceID
+     */
+    String LOG_TRACE_ID = "logTraceId";
+
+    /**
+     * 接口请求开始时间戳
+     */
+    String REQUEST_START_TIME = "request_start_time";
+
+    String STANDARD_FORMAT = "yyyy-MM-dd HH:mm:ss";
+}

etl-core/src/main/java/com/tzld/crawler/etl/common/enums/ExceptionEnum.java

@@ -0,0 +1,33 @@
+package com.tzld.crawler.etl.common.enums;
+
+/**
+ * 异常
+ *
+ * @author supeng
+ * @date 2020/08/31
+ */
+public enum ExceptionEnum {
+
+    SYSTEM_ERROR(1000, "系统错误"),
+    DATA_ERROR(1001, "数据异常,请联系管理员"),
+    PARAM_ERROR(1002, "参数不对"),
+    INVOKE_VIDEOAPI_ERROR(1003, "调用 longvideo api 接口服务失败"),
+
+    ;
+
+    private final int code;
+    private final String msg;
+
+    ExceptionEnum(int code, String msg) {
+        this.code = code;
+        this.msg = msg;
+    }
+
+    public int getCode() {
+        return code;
+    }
+
+    public String getMsg() {
+        return msg;
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/common/exception/CommonException.java

@@ -0,0 +1,94 @@
+package com.tzld.crawler.etl.common.exception;
+
+import com.tzld.crawler.etl.common.enums.ExceptionEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 异常
+ *
+ * @author supeng
+ * @date 2020/08/28
+ */
+public class CommonException extends RuntimeException {
+
+    /**
+     *
+     */
+    private static final long serialVersionUID = 1L;
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(CommonException.class);
+    /**
+     * 异常
+     */
+    private ExceptionEnum exceptionEnum;
+    /**
+     * 错误码
+     */
+    private int code;
+    /**
+     * 异常信息
+     */
+    private String msg;
+
+
+    public CommonException(Throwable throwable) {
+        super(throwable);
+    }
+
+    public CommonException(int code, String msg) {
+        super(msg);
+        this.code = code;
+        this.msg = msg;
+    }
+
+    public CommonException(ExceptionEnum exceptionEnum, String msg) {
+        super(msg);
+        this.code = exceptionEnum.getCode();
+        this.msg = msg;
+    }
+
+    public CommonException(ExceptionEnum exceptionEnum) {
+        super(exceptionEnum.getMsg());
+        this.exceptionEnum = exceptionEnum;
+        this.code = exceptionEnum.getCode();
+        this.msg = exceptionEnum.getMsg();
+    }
+
+    public CommonException(int code, String msg, Throwable throwable) {
+        super(msg, throwable);
+        this.code = code;
+        this.msg = msg;
+    }
+
+    public CommonException(ExceptionEnum exceptionEnum, Throwable throwable) {
+        super(exceptionEnum.getMsg(), throwable);
+        this.exceptionEnum = exceptionEnum;
+        this.code = exceptionEnum.getCode();
+        this.msg = exceptionEnum.getMsg();
+    }
+
+    public int getCode() {
+        return code;
+    }
+
+    public void setCode(int code) {
+        this.code = code;
+    }
+
+    public String getMsg() {
+        return msg;
+    }
+
+    public void setMsg(String msg) {
+        this.msg = msg;
+    }
+
+    @Override
+    public void printStackTrace() {
+        if (exceptionEnum != null) {
+            LOGGER.info("exception code = {}, msg = {}", exceptionEnum.getCode(), exceptionEnum.getMsg());
+        }
+        LOGGER.info("exception code = {}, msg = {}", code, msg);
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/config/FeignConfig.java

@@ -0,0 +1,45 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.config;
+
+import feign.codec.Encoder;
+import feign.form.spring.SpringFormEncoder;
+import org.springframework.beans.factory.ObjectFactory;
+import org.springframework.boot.autoconfigure.http.HttpMessageConverters;
+import org.springframework.cloud.openfeign.support.SpringEncoder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-12 15:03.
+ */
+@Configuration
+public class FeignConfig {
+    @Bean
+    public Encoder feignFormEncoder(ObjectFactory<HttpMessageConverters> converters) {
+        return new SpringFormEncoder(new SpringEncoder(converters));
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/config/LoggerLevelRefresher.java

@@ -0,0 +1,66 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2022 xrv <xrg@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.config;
+
+import com.ctrip.framework.apollo.model.ConfigChangeEvent;
+import com.ctrip.framework.apollo.spring.annotation.ApolloConfigChangeListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.logging.LogLevel;
+import org.springframework.boot.logging.LoggingSystem;
+import org.springframework.stereotype.Component;
+
+/**
+ * 结合配置中心动态刷新（logging.level.）日志级别
+ *
+ * @author ehlxr
+ * @since 2022-01-06 17:43.
+ */
+@Component
+public class LoggerLevelRefresher {
+    private static final Logger logger = LoggerFactory.getLogger(LoggerLevelRefresher.class);
+    private static final String LOGGER_TAG = "logging.level.";
+    private final LoggingSystem loggingSystem;
+
+    @Autowired
+    public LoggerLevelRefresher(LoggingSystem loggingSystem) {
+        this.loggingSystem = loggingSystem;
+    }
+
+    @SuppressWarnings("unused")
+    @ApolloConfigChangeListener(interestedKeyPrefixes = LOGGER_TAG)
+    private void onChange(ConfigChangeEvent changeEvent) {
+        for (String key : changeEvent.changedKeys()) {
+            String strLevel = changeEvent.getChange(key).getNewValue();
+            loggingSystem.setLogLevel(key.replaceAll(LOGGER_TAG, ""),
+                    LogLevel.valueOf(strLevel.toUpperCase()));
+
+            logger.info("logging changed: {}, oldValue: {}, newValue: {}",
+                    key, changeEvent.getChange(key).getOldValue(), strLevel);
+        }
+    }
+
+}

etl-core/src/main/java/com/tzld/crawler/etl/config/SwaggerConfig.java

@@ -0,0 +1,62 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2021 xrv <xrg@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import springfox.documentation.builders.ApiInfoBuilder;
+import springfox.documentation.builders.PathSelectors;
+import springfox.documentation.builders.RequestHandlerSelectors;
+import springfox.documentation.oas.annotations.EnableOpenApi;
+import springfox.documentation.service.ApiInfo;
+import springfox.documentation.spi.DocumentationType;
+import springfox.documentation.spring.web.plugins.Docket;
+
+/**
+ * @author ehlxr
+ * @since 2021-07-23 10:03.
+ */
+@Configuration
+@EnableOpenApi
+public class SwaggerConfig {
+    @Bean
+    public Docket createRestApi() {
+        return new Docket(DocumentationType.OAS_30)
+                .apiInfo(apiInfo())
+                .directModelSubstitute(Byte.class, Integer.class)
+                .select()
+                .apis(RequestHandlerSelectors.basePackage("com.tzld.crawler.etl.controller"))
+                .paths(PathSelectors.any())
+                .build();
+    }
+
+    private ApiInfo apiInfo() {
+        return new ApiInfoBuilder()
+                .title("ETL swagger api")
+                .description("Crawler ETL swagger api")
+                .version("1.0")
+                .build();
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/config/SwaggerHostResolver.java

@@ -0,0 +1,77 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2022 xrv <xrg@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.config;
+
+import com.google.common.base.Strings;
+import io.swagger.v3.oas.models.OpenAPI;
+import io.swagger.v3.oas.models.servers.Server;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import springfox.documentation.oas.web.OpenApiTransformationContext;
+import springfox.documentation.oas.web.WebMvcOpenApiTransformationFilter;
+import springfox.documentation.spi.DocumentationType;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.Collections;
+
+/**
+ * @author ehlxr
+ * @since 2022-02-10 11:00.
+ * <p>
+ * https://github.com/springfox/springfox/issues/3483
+ * <p>
+ * https://github.com/springfox/springfox/issues/3445
+ */
+@Component
+public class SwaggerHostResolver implements WebMvcOpenApiTransformationFilter {
+    @Value("${swagger.host:}")
+    private String hostUri;
+
+    @Value("${server.servlet.context-path:}")
+    private String contextPath;
+
+    @Override
+    public OpenAPI transform(OpenApiTransformationContext<HttpServletRequest> context) {
+        OpenAPI openApi = context.getSpecification();
+        if (Strings.isNullOrEmpty(hostUri)) {
+            context.request().ifPresent(httpServletRequest -> {
+                String referer = httpServletRequest.getHeader("referer");
+                if (!Strings.isNullOrEmpty(referer)) {
+                    hostUri = referer.substring(0, referer.indexOf(contextPath));
+                }
+            });
+        }
+
+        Server server = new Server();
+        server.setUrl(hostUri);
+        openApi.setServers(Collections.singletonList(server));
+        return openApi;
+    }
+
+    @Override
+    public boolean supports(DocumentationType delimiter) {
+        return delimiter == DocumentationType.OAS_30;
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/dao/generator/MybatisGeneratorMain.java

@@ -0,0 +1,33 @@
+package com.tzld.crawler.etl.dao.generator;
+
+import org.mybatis.generator.api.MyBatisGenerator;
+import org.mybatis.generator.config.Configuration;
+import org.mybatis.generator.config.xml.ConfigurationParser;
+import org.mybatis.generator.exception.InvalidConfigurationException;
+import org.mybatis.generator.exception.XMLParserException;
+import org.mybatis.generator.internal.DefaultShellCallback;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author ehlxr
+ */
+public class MybatisGeneratorMain {
+
+    public static void main(String[] args)
+            throws SQLException, IOException, InterruptedException, InvalidConfigurationException, XMLParserException {
+        List<String> warnings = new ArrayList<>();
+        boolean overwrite = true;
+        File configFile = new File(MybatisGeneratorMain.class.getResource("/mybatis-generator-config.xml").getFile());
+        ConfigurationParser cp = new ConfigurationParser(warnings);
+        Configuration config = cp.parseConfiguration(configFile);
+        DefaultShellCallback callback = new DefaultShellCallback(overwrite);
+        MyBatisGenerator myBatisGenerator = new MyBatisGenerator(config, callback, warnings);
+        myBatisGenerator.generate(null);
+        System.out.println("generate finish");
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/dao/mapper/CrawlerVideoMapper.java

@@ -0,0 +1,97 @@
+package com.tzld.crawler.etl.dao.mapper;
+
+import com.tzld.crawler.etl.model.po.CrawlerVideo;
+import com.tzld.crawler.etl.model.po.CrawlerVideoExample;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+public interface CrawlerVideoMapper {
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    long countByExample(CrawlerVideoExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    int deleteByExample(CrawlerVideoExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    int deleteByPrimaryKey(Long id);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    int insert(CrawlerVideo record);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    int insertSelective(CrawlerVideo record);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    List<CrawlerVideo> selectByExample(CrawlerVideoExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    CrawlerVideo selectByPrimaryKey(Long id);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    int updateByExampleSelective(@Param("record") CrawlerVideo record, @Param("example") CrawlerVideoExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    int updateByExample(@Param("record") CrawlerVideo record, @Param("example") CrawlerVideoExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    int updateByPrimaryKeySelective(CrawlerVideo record);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    int updateByPrimaryKey(CrawlerVideo record);
+}

etl-core/src/main/java/com/tzld/crawler/etl/filter/AuthFilter.java

@@ -0,0 +1,38 @@
+package com.tzld.crawler.etl.filter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.core.annotation.Order;
+
+import javax.servlet.*;
+import javax.servlet.annotation.WebFilter;
+import java.io.IOException;
+
+/**
+ * auth filter
+ *
+ * @author supeng
+ * @date 2020/08/31
+ */
+@Order(value = 2)
+@WebFilter(filterName = "authFilter", urlPatterns = "/api")
+public class AuthFilter implements Filter {
+    private static final Logger LOGGER = LoggerFactory.getLogger(AuthFilter.class);
+
+    @Override
+    public void init(FilterConfig filterConfig) throws ServletException {
+        LOGGER.info("authFilter init");
+
+    }
+
+    @Override
+    public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+        //TODO token校验
+        filterChain.doFilter(servletRequest, servletResponse);
+    }
+
+    @Override
+    public void destroy() {
+        LOGGER.info("authFilter destroy");
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/filter/LogTraceFilter.java

@@ -0,0 +1,41 @@
+package com.tzld.crawler.etl.filter;
+
+import com.tzld.crawler.etl.common.base.Constant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.core.annotation.Order;
+
+import javax.servlet.*;
+import javax.servlet.annotation.WebFilter;
+import java.io.IOException;
+import java.util.UUID;
+
+/**
+ * 日志增加traceId
+ *
+ * @author supeng
+ * @date 2020/08/25
+ */
+@Order(value = 1)
+@WebFilter(filterName = "logTraceFilter", urlPatterns = "/*")
+public class LogTraceFilter implements Filter {
+    private static final Logger LOGGER = LoggerFactory.getLogger(LogTraceFilter.class);
+
+    @Override
+    public void init(FilterConfig filterConfig) {
+        LOGGER.info("logTraceFilter init");
+    }
+
+    @Override
+    public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+        MDC.put(Constant.LOG_TRACE_ID, UUID.randomUUID().toString());
+        filterChain.doFilter(servletRequest, servletResponse);
+        MDC.remove(Constant.LOG_TRACE_ID);
+    }
+
+    @Override
+    public void destroy() {
+        LOGGER.info("logTraceFilter destroy");
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/handle/GlobalExceptionHandle.java

@@ -0,0 +1,75 @@
+package com.tzld.crawler.etl.handle;
+
+import com.tzld.crawler.etl.common.base.CommonResponse;
+import com.tzld.crawler.etl.common.enums.ExceptionEnum;
+import com.tzld.crawler.etl.common.exception.CommonException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.CollectionUtils;
+import org.springframework.validation.BindException;
+import org.springframework.validation.ObjectError;
+import org.springframework.web.bind.MethodArgumentNotValidException;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.RestControllerAdvice;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.List;
+
+/**
+ * 全局异常处理器
+ *
+ * @author liuzhiheng
+ * @version 1.0
+ * @date 2020年11月20日 上午11:04:48
+ */
+@RestControllerAdvice
+public class GlobalExceptionHandle {
+    private final static Logger LOGGER = LoggerFactory.getLogger(GlobalExceptionHandle.class);
+
+    @ExceptionHandler
+    public Object handleException(HttpServletRequest req, Exception exception) {
+        String uri = req.getRequestURI();
+        CommonResponse<Object> response = new CommonResponse<>();
+        // 业务异常
+        if (exception instanceof CommonException) {
+            CommonException e = (CommonException) exception;
+            response.setCode(e.getCode());
+            response.setMsg(e.getMsg());
+            LOGGER.warn("uri:" + uri + "\n" + "CustomException log.", exception);
+        } else if (exception instanceof MethodArgumentNotValidException) {
+            // 参数校验异常
+            MethodArgumentNotValidException e = (MethodArgumentNotValidException) exception;
+            List<ObjectError> errorList = e.getBindingResult().getAllErrors();
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.append("|");
+            if (!CollectionUtils.isEmpty(errorList)) {
+                for (ObjectError objectError : errorList) {
+                    errorMsg.append(objectError.getDefaultMessage()).append("|");
+                }
+            }
+            response.setCode(ExceptionEnum.PARAM_ERROR.getCode());
+            response.setMsg(errorMsg.toString());
+            LOGGER.warn("uri:" + uri + "\n" + "MethodArgumentNotValidException log.", exception);
+        } else if (exception instanceof BindException) {
+            // 参数绑定异常
+            BindException e = (BindException) exception;
+            List<ObjectError> errorList = e.getBindingResult().getAllErrors();
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.append("|");
+            if (!CollectionUtils.isEmpty(errorList)) {
+                for (ObjectError objectError : errorList) {
+                    errorMsg.append(objectError.getDefaultMessage()).append("|");
+                }
+            }
+            response.setCode(ExceptionEnum.PARAM_ERROR.getCode());
+            response.setMsg(errorMsg.toString());
+            LOGGER.warn("uri:" + uri + "\n" + "BindException log.", exception);
+        } else {
+            response.setCode(ExceptionEnum.SYSTEM_ERROR.getCode());
+            response.setMsg(ExceptionEnum.SYSTEM_ERROR.getMsg());
+            LOGGER.error("uri:" + uri + "\n" + "unknown exception log.", exception);
+        }
+        return response;
+    }
+
+}

etl-core/src/main/java/com/tzld/crawler/etl/model/dto/BaseInfoDto.java

@@ -0,0 +1,170 @@
+package com.tzld.crawler.etl.model.dto;
+
+import io.swagger.annotations.ApiModelProperty;
+
+/**
+ * 基础信息
+ *
+ * @author supeng
+ * @date 2020/08/28
+ */
+public class BaseInfoDto {
+    /// 用户信息
+    @ApiModelProperty(value = "公共参数-token值")
+    private String token;
+    @ApiModelProperty(value = "公共参数-登录用户ID")
+    private Long loginUid;
+
+    /// 应用信息
+    @ApiModelProperty(value = "公共参数-应用版本号")
+    private Integer appVersionCode;
+    @ApiModelProperty(value = "公共参数-产品代号")
+    private Integer appType;
+
+    /// 设备信息
+    @ApiModelProperty(value = "公共参数-手机设备的唯一码")
+    private String machineCode;
+    @ApiModelProperty(value = "公共参数-ios,android")
+    private String platform;
+    @ApiModelProperty(value = "公共参数-系统版本（例：ios10.1）")
+    private String systemVersion;
+    @ApiModelProperty(value = "公共参数-手机信息")
+    private String machineInfo;
+    @ApiModelProperty(value = "公共参数-网络类型 WI-FI 5G 4G 3G 2G")
+    private String networkType;
+    @ApiModelProperty(value = "公共参数-客户端ip")
+    private String clientIp;
+
+    // pageSource相关的参数
+    @ApiModelProperty(value = "公共参数-页面来源")
+    private String pageSource;
+
+    // 某次操作相关的参数
+    @ApiModelProperty(value = "公共参数-前端请求时间")
+    private Long clientTimestamp;
+    @ApiModelProperty(value = "公共参数-sessionId")
+    private String sessionId;
+    @ApiModelProperty(value = "公共参数-requestId，每次请求客户端生成唯一ID，不超过64位")
+    private String requestId;
+
+    public String getToken() {
+        return token;
+    }
+
+    public void setToken(String token) {
+        this.token = token;
+    }
+
+    public Long getLoginUid() {
+        return loginUid;
+    }
+
+    public void setLoginUid(Long loginUid) {
+        this.loginUid = loginUid;
+    }
+
+    public Integer getAppVersionCode() {
+        return appVersionCode;
+    }
+
+    public void setAppVersionCode(Integer appVersionCode) {
+        this.appVersionCode = appVersionCode;
+    }
+
+    public Integer getAppType() {
+        return appType;
+    }
+
+    public void setAppType(Integer appType) {
+        this.appType = appType;
+    }
+
+    public String getMachineCode() {
+        return machineCode;
+    }
+
+    public void setMachineCode(String machineCode) {
+        this.machineCode = machineCode;
+    }
+
+    public String getPlatform() {
+        return platform;
+    }
+
+    public void setPlatform(String platform) {
+        this.platform = platform;
+    }
+
+    public String getSystemVersion() {
+        return systemVersion;
+    }
+
+    public void setSystemVersion(String systemVersion) {
+        this.systemVersion = systemVersion;
+    }
+
+    public String getMachineInfo() {
+        return machineInfo;
+    }
+
+    public void setMachineInfo(String machineInfo) {
+        this.machineInfo = machineInfo;
+    }
+
+    public String getNetworkType() {
+        return networkType;
+    }
+
+    public void setNetworkType(String networkType) {
+        this.networkType = networkType;
+    }
+
+    public String getClientIp() {
+        return clientIp;
+    }
+
+    public void setClientIp(String clientIp) {
+        this.clientIp = clientIp;
+    }
+
+    public String getPageSource() {
+        return pageSource;
+    }
+
+    public void setPageSource(String pageSource) {
+        this.pageSource = pageSource;
+    }
+
+    public Long getClientTimestamp() {
+        return clientTimestamp;
+    }
+
+    public void setClientTimestamp(Long clientTimestamp) {
+        this.clientTimestamp = clientTimestamp;
+    }
+
+    public String getSessionId() {
+        return sessionId;
+    }
+
+    public void setSessionId(String sessionId) {
+        this.sessionId = sessionId;
+    }
+
+    public String getRequestId() {
+        return requestId;
+    }
+
+    public void setRequestId(String requestId) {
+        this.requestId = requestId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format(
+                "BaseInfoDTO [token=%s, appVersionCode=%s, appType=%s, machineCode=%s, platform=%s, systemVersion=%s, machineInfo=%s, networkType=%s, clientIp=%s, pageSource=%s, clientTimestamp=%s, sessionId=%s, requestId=%s]",
+                token, appVersionCode, appType, machineCode, platform, systemVersion, machineInfo, networkType, clientIp,
+                pageSource, clientTimestamp, sessionId, requestId);
+    }
+
+}

etl-core/src/main/java/com/tzld/crawler/etl/model/dto/StrategyDataDto.java

@@ -0,0 +1,50 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.model.dto;
+
+import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-20 10:36.
+ */
+public class StrategyDataDto extends CrawlerVideoVO {
+    private double titleScore;
+
+    public double getTitleScore() {
+        return titleScore;
+    }
+
+    public void setTitleScore(double titleScore) {
+        this.titleScore = titleScore;
+    }
+
+    @Override
+    public String toString() {
+        return "StrategyDataDto{" +
+                "titleScore=" + titleScore +
+                "} " + super.toString();
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/model/param/CrawlerVideoSendParam.java

@@ -0,0 +1,173 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrg@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package com.tzld.crawler.etl.model.param;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 19:22.
+ */
+@ApiModel
+public class CrawlerVideoSendParam {
+    @ApiModelProperty(value = "登录用户ID")
+    private Long loginUid;
+    @ApiModelProperty(value = "产品类型")
+    private Integer appType;
+    @ApiModelProperty(value = "爬虫视频所在渠道内的ID")
+    private String crawlerSrcId;
+    @ApiModelProperty(value = "爬虫渠道code")
+    private String crawlerSrcCode;
+    @ApiModelProperty(value = "爬虫视频所在渠道内发布视频")
+    private Long crawlerSrcPublishTimestamp;
+    @ApiModelProperty(value = "爬虫任务创建时间")
+    private Long crawlerTaskTimestamp;
+    @ApiModelProperty(value = "视频地址")
+    private String videoPath;
+    @ApiModelProperty(value = "封面地址")
+    private String coverImgPath;
+    @ApiModelProperty(value = "标题")
+    private String title;
+    @ApiModelProperty(value = "总时长:秒")
+    private Long totalTime;
+    @ApiModelProperty(value = "可视状态: 1有效，2 已删除，3 已屏蔽,4关注可见，5分享可见，6自己可见")
+    private Integer viewStatus;
+    @ApiModelProperty(value = "公共参数-版本号")
+    private Integer versionCode;
+
+    @Override
+    public String toString() {
+        return "CrawlerVideoSendParam{" +
+                "loginUid=" + loginUid +
+                ", appType=" + appType +
+                ", crawlerSrcId='" + crawlerSrcId + '\'' +
+                ", crawlerSrcCode='" + crawlerSrcCode + '\'' +
+                ", crawlerSrcPublishTimestamp=" + crawlerSrcPublishTimestamp +
+                ", crawlerTaskTimestamp=" + crawlerTaskTimestamp +
+                ", videoPath='" + videoPath + '\'' +
+                ", coverImgPath='" + coverImgPath + '\'' +
+                ", title='" + title + '\'' +
+                ", totalTime=" + totalTime +
+                ", viewStatus=" + viewStatus +
+                ", versionCode=" + versionCode +
+                '}';
+    }
+
+    public Long getLoginUid() {
+        return loginUid;
+    }
+
+    public void setLoginUid(Long loginUid) {
+        this.loginUid = loginUid;
+    }
+
+    public Integer getAppType() {
+        return appType;
+    }
+
+    public void setAppType(Integer appType) {
+        this.appType = appType;
+    }
+
+    public String getCrawlerSrcId() {
+        return crawlerSrcId;
+    }
+
+    public void setCrawlerSrcId(String crawlerSrcId) {
+        this.crawlerSrcId = crawlerSrcId;
+    }
+
+    public String getCrawlerSrcCode() {
+        return crawlerSrcCode;
+    }
+
+    public void setCrawlerSrcCode(String crawlerSrcCode) {
+        this.crawlerSrcCode = crawlerSrcCode;
+    }
+
+    public Long getCrawlerSrcPublishTimestamp() {
+        return crawlerSrcPublishTimestamp;
+    }
+
+    public void setCrawlerSrcPublishTimestamp(Long crawlerSrcPublishTimestamp) {
+        this.crawlerSrcPublishTimestamp = crawlerSrcPublishTimestamp;
+    }
+
+    public Long getCrawlerTaskTimestamp() {
+        return crawlerTaskTimestamp;
+    }
+
+    public void setCrawlerTaskTimestamp(Long crawlerTaskTimestamp) {
+        this.crawlerTaskTimestamp = crawlerTaskTimestamp;
+    }
+
+    public String getVideoPath() {
+        return videoPath;
+    }
+
+    public void setVideoPath(String videoPath) {
+        this.videoPath = videoPath;
+    }
+
+    public String getCoverImgPath() {
+        return coverImgPath;
+    }
+
+    public void setCoverImgPath(String coverImgPath) {
+        this.coverImgPath = coverImgPath;
+    }
+
+    public String getTitle() {
+        return title;
+    }
+
+    public void setTitle(String title) {
+        this.title = title;
+    }
+
+    public Long getTotalTime() {
+        return totalTime;
+    }
+
+    public void setTotalTime(Long totalTime) {
+        this.totalTime = totalTime;
+    }
+
+    public Integer getViewStatus() {
+        return viewStatus;
+    }
+
+    public void setViewStatus(Integer viewStatus) {
+        this.viewStatus = viewStatus;
+    }
+
+    public Integer getVersionCode() {
+        return versionCode;
+    }
+
+    public void setVersionCode(Integer versionCode) {
+        this.versionCode = versionCode;
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/model/po/CrawlerVideo.java

@@ -0,0 +1,772 @@
+package com.tzld.crawler.etl.model.po;
+
+import java.util.Date;
+
+/**
+ * This class was generated by MyBatis Generator.
+ * This class corresponds to the database table crawler_video
+ */
+public class CrawlerVideo {
+    /**
+     * Database Column Remarks:
+     * ID
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.id
+     *
+     * @mbg.generated
+     */
+    private Long id;
+
+    /**
+     * Database Column Remarks:
+     * 站内视频ID
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.video_id
+     *
+     * @mbg.generated
+     */
+    private Long videoId;
+
+    /**
+     * Database Column Remarks:
+     * 站内用户ID
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.user_id
+     *
+     * @mbg.generated
+     */
+    private Long userId;
+
+    /**
+     * Database Column Remarks:
+     * 站外用户ID
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.out_user_id
+     *
+     * @mbg.generated
+     */
+    private String outUserId;
+
+    /**
+     * Database Column Remarks:
+     * 平台:youtube，wechat，小年糕，好看视频
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.platform
+     *
+     * @mbg.generated
+     */
+    private String platform;
+
+    /**
+     * Database Column Remarks:
+     * 策略:定向爬虫策略，热榜爬虫策略，小时榜爬虫策略，推荐榜爬虫策略
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.strategy
+     *
+     * @mbg.generated
+     */
+    private String strategy;
+
+    /**
+     * Database Column Remarks:
+     * 站外视频ID
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.out_video_id
+     *
+     * @mbg.generated
+     */
+    private String outVideoId;
+
+    /**
+     * Database Column Remarks:
+     * 视频标题
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.video_title
+     *
+     * @mbg.generated
+     */
+    private String videoTitle;
+
+    /**
+     * Database Column Remarks:
+     * 视频封面
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.cover_url
+     *
+     * @mbg.generated
+     */
+    private String coverUrl;
+
+    /**
+     * Database Column Remarks:
+     * 视频播放地址
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.video_url
+     *
+     * @mbg.generated
+     */
+    private String videoUrl;
+
+    /**
+     * Database Column Remarks:
+     * 视频时长
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.duration
+     *
+     * @mbg.generated
+     */
+    private Long duration;
+
+    /**
+     * Database Column Remarks:
+     * 站外视频发布时间
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.publish_time
+     *
+     * @mbg.generated
+     */
+    private String publishTime;
+
+    /**
+     * Database Column Remarks:
+     * 播放量
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.play_cnt
+     *
+     * @mbg.generated
+     */
+    private Integer playCnt;
+
+    /**
+     * Database Column Remarks:
+     * 点赞量
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.like_cnt
+     *
+     * @mbg.generated
+     */
+    private Integer likeCnt;
+
+    /**
+     * Database Column Remarks:
+     * 分享量
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.share_cnt
+     *
+     * @mbg.generated
+     */
+    private Integer shareCnt;
+
+    /**
+     * Database Column Remarks:
+     * 收藏量
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.collection_cnt
+     *
+     * @mbg.generated
+     */
+    private Integer collectionCnt;
+
+    /**
+     * Database Column Remarks:
+     * 评论量
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.comment_cnt
+     *
+     * @mbg.generated
+     */
+    private Integer commentCnt;
+
+    /**
+     * Database Column Remarks:
+     * 抓取时条件:{'play_cnt': 0, 'comment_cnt': 0, 'like_cnt': 0, 'duration': 60, 'publish_time': 10, 'video_width': 720, 'video_height': 720}
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.crawler_rule
+     *
+     * @mbg.generated
+     */
+    private String crawlerRule;
+
+    /**
+     * Database Column Remarks:
+     * 宽
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.width
+     *
+     * @mbg.generated
+     */
+    private Integer width;
+
+    /**
+     * Database Column Remarks:
+     * 高
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.height
+     *
+     * @mbg.generated
+     */
+    private Integer height;
+
+    /**
+     * Database Column Remarks:
+     * 创建时间
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.create_time
+     *
+     * @mbg.generated
+     */
+    private Date createTime;
+
+    /**
+     * Database Column Remarks:
+     * 更新时间
+     * <p>
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database column crawler_video.update_time
+     *
+     * @mbg.generated
+     */
+    private Date updateTime;
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.id
+     *
+     * @return the value of crawler_video.id
+     * @mbg.generated
+     */
+    public Long getId() {
+        return id;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.id
+     *
+     * @param id the value for crawler_video.id
+     * @mbg.generated
+     */
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.video_id
+     *
+     * @return the value of crawler_video.video_id
+     * @mbg.generated
+     */
+    public Long getVideoId() {
+        return videoId;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.video_id
+     *
+     * @param videoId the value for crawler_video.video_id
+     * @mbg.generated
+     */
+    public void setVideoId(Long videoId) {
+        this.videoId = videoId;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.user_id
+     *
+     * @return the value of crawler_video.user_id
+     * @mbg.generated
+     */
+    public Long getUserId() {
+        return userId;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.user_id
+     *
+     * @param userId the value for crawler_video.user_id
+     * @mbg.generated
+     */
+    public void setUserId(Long userId) {
+        this.userId = userId;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.out_user_id
+     *
+     * @return the value of crawler_video.out_user_id
+     * @mbg.generated
+     */
+    public String getOutUserId() {
+        return outUserId;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.out_user_id
+     *
+     * @param outUserId the value for crawler_video.out_user_id
+     * @mbg.generated
+     */
+    public void setOutUserId(String outUserId) {
+        this.outUserId = outUserId;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.platform
+     *
+     * @return the value of crawler_video.platform
+     * @mbg.generated
+     */
+    public String getPlatform() {
+        return platform;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.platform
+     *
+     * @param platform the value for crawler_video.platform
+     * @mbg.generated
+     */
+    public void setPlatform(String platform) {
+        this.platform = platform;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.strategy
+     *
+     * @return the value of crawler_video.strategy
+     * @mbg.generated
+     */
+    public String getStrategy() {
+        return strategy;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.strategy
+     *
+     * @param strategy the value for crawler_video.strategy
+     * @mbg.generated
+     */
+    public void setStrategy(String strategy) {
+        this.strategy = strategy;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.out_video_id
+     *
+     * @return the value of crawler_video.out_video_id
+     * @mbg.generated
+     */
+    public String getOutVideoId() {
+        return outVideoId;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.out_video_id
+     *
+     * @param outVideoId the value for crawler_video.out_video_id
+     * @mbg.generated
+     */
+    public void setOutVideoId(String outVideoId) {
+        this.outVideoId = outVideoId;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.video_title
+     *
+     * @return the value of crawler_video.video_title
+     * @mbg.generated
+     */
+    public String getVideoTitle() {
+        return videoTitle;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.video_title
+     *
+     * @param videoTitle the value for crawler_video.video_title
+     * @mbg.generated
+     */
+    public void setVideoTitle(String videoTitle) {
+        this.videoTitle = videoTitle;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.cover_url
+     *
+     * @return the value of crawler_video.cover_url
+     * @mbg.generated
+     */
+    public String getCoverUrl() {
+        return coverUrl;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.cover_url
+     *
+     * @param coverUrl the value for crawler_video.cover_url
+     * @mbg.generated
+     */
+    public void setCoverUrl(String coverUrl) {
+        this.coverUrl = coverUrl;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.video_url
+     *
+     * @return the value of crawler_video.video_url
+     * @mbg.generated
+     */
+    public String getVideoUrl() {
+        return videoUrl;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.video_url
+     *
+     * @param videoUrl the value for crawler_video.video_url
+     * @mbg.generated
+     */
+    public void setVideoUrl(String videoUrl) {
+        this.videoUrl = videoUrl;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.duration
+     *
+     * @return the value of crawler_video.duration
+     * @mbg.generated
+     */
+    public Long getDuration() {
+        return duration;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.duration
+     *
+     * @param duration the value for crawler_video.duration
+     * @mbg.generated
+     */
+    public void setDuration(Long duration) {
+        this.duration = duration;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.publish_time
+     *
+     * @return the value of crawler_video.publish_time
+     * @mbg.generated
+     */
+    public String getPublishTime() {
+        return publishTime;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.publish_time
+     *
+     * @param publishTime the value for crawler_video.publish_time
+     * @mbg.generated
+     */
+    public void setPublishTime(String publishTime) {
+        this.publishTime = publishTime;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.play_cnt
+     *
+     * @return the value of crawler_video.play_cnt
+     * @mbg.generated
+     */
+    public Integer getPlayCnt() {
+        return playCnt;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.play_cnt
+     *
+     * @param playCnt the value for crawler_video.play_cnt
+     * @mbg.generated
+     */
+    public void setPlayCnt(Integer playCnt) {
+        this.playCnt = playCnt;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.like_cnt
+     *
+     * @return the value of crawler_video.like_cnt
+     * @mbg.generated
+     */
+    public Integer getLikeCnt() {
+        return likeCnt;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.like_cnt
+     *
+     * @param likeCnt the value for crawler_video.like_cnt
+     * @mbg.generated
+     */
+    public void setLikeCnt(Integer likeCnt) {
+        this.likeCnt = likeCnt;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.share_cnt
+     *
+     * @return the value of crawler_video.share_cnt
+     * @mbg.generated
+     */
+    public Integer getShareCnt() {
+        return shareCnt;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.share_cnt
+     *
+     * @param shareCnt the value for crawler_video.share_cnt
+     * @mbg.generated
+     */
+    public void setShareCnt(Integer shareCnt) {
+        this.shareCnt = shareCnt;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.collection_cnt
+     *
+     * @return the value of crawler_video.collection_cnt
+     * @mbg.generated
+     */
+    public Integer getCollectionCnt() {
+        return collectionCnt;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.collection_cnt
+     *
+     * @param collectionCnt the value for crawler_video.collection_cnt
+     * @mbg.generated
+     */
+    public void setCollectionCnt(Integer collectionCnt) {
+        this.collectionCnt = collectionCnt;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.comment_cnt
+     *
+     * @return the value of crawler_video.comment_cnt
+     * @mbg.generated
+     */
+    public Integer getCommentCnt() {
+        return commentCnt;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.comment_cnt
+     *
+     * @param commentCnt the value for crawler_video.comment_cnt
+     * @mbg.generated
+     */
+    public void setCommentCnt(Integer commentCnt) {
+        this.commentCnt = commentCnt;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.crawler_rule
+     *
+     * @return the value of crawler_video.crawler_rule
+     * @mbg.generated
+     */
+    public String getCrawlerRule() {
+        return crawlerRule;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.crawler_rule
+     *
+     * @param crawlerRule the value for crawler_video.crawler_rule
+     * @mbg.generated
+     */
+    public void setCrawlerRule(String crawlerRule) {
+        this.crawlerRule = crawlerRule;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.width
+     *
+     * @return the value of crawler_video.width
+     * @mbg.generated
+     */
+    public Integer getWidth() {
+        return width;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.width
+     *
+     * @param width the value for crawler_video.width
+     * @mbg.generated
+     */
+    public void setWidth(Integer width) {
+        this.width = width;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.height
+     *
+     * @return the value of crawler_video.height
+     * @mbg.generated
+     */
+    public Integer getHeight() {
+        return height;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.height
+     *
+     * @param height the value for crawler_video.height
+     * @mbg.generated
+     */
+    public void setHeight(Integer height) {
+        this.height = height;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.create_time
+     *
+     * @return the value of crawler_video.create_time
+     * @mbg.generated
+     */
+    public Date getCreateTime() {
+        return createTime;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.create_time
+     *
+     * @param createTime the value for crawler_video.create_time
+     * @mbg.generated
+     */
+    public void setCreateTime(Date createTime) {
+        this.createTime = createTime;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method returns the value of the database column crawler_video.update_time
+     *
+     * @return the value of crawler_video.update_time
+     * @mbg.generated
+     */
+    public Date getUpdateTime() {
+        return updateTime;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method sets the value of the database column crawler_video.update_time
+     *
+     * @param updateTime the value for crawler_video.update_time
+     * @mbg.generated
+     */
+    public void setUpdateTime(Date updateTime) {
+        this.updateTime = updateTime;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    @Override
+    public String toString() {
+        String sb = getClass().getSimpleName() +
+                " [" +
+                "Hash = " + hashCode() +
+                ", id=" + id +
+                ", videoId=" + videoId +
+                ", userId=" + userId +
+                ", outUserId=" + outUserId +
+                ", platform=" + platform +
+                ", strategy=" + strategy +
+                ", outVideoId=" + outVideoId +
+                ", videoTitle=" + videoTitle +
+                ", coverUrl=" + coverUrl +
+                ", videoUrl=" + videoUrl +
+                ", duration=" + duration +
+                ", publishTime=" + publishTime +
+                ", playCnt=" + playCnt +
+                ", likeCnt=" + likeCnt +
+                ", shareCnt=" + shareCnt +
+                ", collectionCnt=" + collectionCnt +
+                ", commentCnt=" + commentCnt +
+                ", crawlerRule=" + crawlerRule +
+                ", width=" + width +
+                ", height=" + height +
+                ", createTime=" + createTime +
+                ", updateTime=" + updateTime +
+                "]";
+        return sb;
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/model/po/CrawlerVideoExample.java

@@ -0,0 +1,1713 @@
+package com.tzld.crawler.etl.model.po;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+public class CrawlerVideoExample {
+    /**
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    protected String orderByClause;
+
+    /**
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    protected boolean distinct;
+
+    /**
+     * This field was generated by MyBatis Generator.
+     * This field corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    protected List<Criteria> oredCriteria;
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public CrawlerVideoExample() {
+        oredCriteria = new ArrayList<Criteria>();
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public String getOrderByClause() {
+        return orderByClause;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public void setOrderByClause(String orderByClause) {
+        this.orderByClause = orderByClause;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public boolean isDistinct() {
+        return distinct;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public void setDistinct(boolean distinct) {
+        this.distinct = distinct;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public List<Criteria> getOredCriteria() {
+        return oredCriteria;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public void or(Criteria criteria) {
+        oredCriteria.add(criteria);
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public Criteria or() {
+        Criteria criteria = createCriteriaInternal();
+        oredCriteria.add(criteria);
+        return criteria;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public Criteria createCriteria() {
+        Criteria criteria = createCriteriaInternal();
+        if (oredCriteria.size() == 0) {
+            oredCriteria.add(criteria);
+        }
+        return criteria;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    protected Criteria createCriteriaInternal() {
+        Criteria criteria = new Criteria();
+        return criteria;
+    }
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public void clear() {
+        oredCriteria.clear();
+        orderByClause = null;
+        distinct = false;
+    }
+
+    /**
+     * This class was generated by MyBatis Generator.
+     * This class corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    protected abstract static class GeneratedCriteria {
+        protected List<Criterion> criteria;
+
+        protected GeneratedCriteria() {
+            super();
+            criteria = new ArrayList<Criterion>();
+        }
+
+        public boolean isValid() {
+            return criteria.size() > 0;
+        }
+
+        public List<Criterion> getAllCriteria() {
+            return criteria;
+        }
+
+        public List<Criterion> getCriteria() {
+            return criteria;
+        }
+
+        protected void addCriterion(String condition) {
+            if (condition == null) {
+                throw new RuntimeException("Value for condition cannot be null");
+            }
+            criteria.add(new Criterion(condition));
+        }
+
+        protected void addCriterion(String condition, Object value, String property) {
+            if (value == null) {
+                throw new RuntimeException("Value for " + property + " cannot be null");
+            }
+            criteria.add(new Criterion(condition, value));
+        }
+
+        protected void addCriterion(String condition, Object value1, Object value2, String property) {
+            if (value1 == null || value2 == null) {
+                throw new RuntimeException("Between values for " + property + " cannot be null");
+            }
+            criteria.add(new Criterion(condition, value1, value2));
+        }
+
+        public Criteria andIdIsNull() {
+            addCriterion("id is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdIsNotNull() {
+            addCriterion("id is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdEqualTo(Long value) {
+            addCriterion("id =", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdNotEqualTo(Long value) {
+            addCriterion("id <>", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdGreaterThan(Long value) {
+            addCriterion("id >", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdGreaterThanOrEqualTo(Long value) {
+            addCriterion("id >=", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdLessThan(Long value) {
+            addCriterion("id <", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdLessThanOrEqualTo(Long value) {
+            addCriterion("id <=", value, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdIn(List<Long> values) {
+            addCriterion("id in", values, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdNotIn(List<Long> values) {
+            addCriterion("id not in", values, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdBetween(Long value1, Long value2) {
+            addCriterion("id between", value1, value2, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andIdNotBetween(Long value1, Long value2) {
+            addCriterion("id not between", value1, value2, "id");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdIsNull() {
+            addCriterion("video_id is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdIsNotNull() {
+            addCriterion("video_id is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdEqualTo(Long value) {
+            addCriterion("video_id =", value, "videoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdNotEqualTo(Long value) {
+            addCriterion("video_id <>", value, "videoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdGreaterThan(Long value) {
+            addCriterion("video_id >", value, "videoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdGreaterThanOrEqualTo(Long value) {
+            addCriterion("video_id >=", value, "videoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdLessThan(Long value) {
+            addCriterion("video_id <", value, "videoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdLessThanOrEqualTo(Long value) {
+            addCriterion("video_id <=", value, "videoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdIn(List<Long> values) {
+            addCriterion("video_id in", values, "videoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdNotIn(List<Long> values) {
+            addCriterion("video_id not in", values, "videoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdBetween(Long value1, Long value2) {
+            addCriterion("video_id between", value1, value2, "videoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoIdNotBetween(Long value1, Long value2) {
+            addCriterion("video_id not between", value1, value2, "videoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdIsNull() {
+            addCriterion("user_id is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdIsNotNull() {
+            addCriterion("user_id is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdEqualTo(Long value) {
+            addCriterion("user_id =", value, "userId");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdNotEqualTo(Long value) {
+            addCriterion("user_id <>", value, "userId");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdGreaterThan(Long value) {
+            addCriterion("user_id >", value, "userId");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdGreaterThanOrEqualTo(Long value) {
+            addCriterion("user_id >=", value, "userId");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdLessThan(Long value) {
+            addCriterion("user_id <", value, "userId");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdLessThanOrEqualTo(Long value) {
+            addCriterion("user_id <=", value, "userId");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdIn(List<Long> values) {
+            addCriterion("user_id in", values, "userId");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdNotIn(List<Long> values) {
+            addCriterion("user_id not in", values, "userId");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdBetween(Long value1, Long value2) {
+            addCriterion("user_id between", value1, value2, "userId");
+            return (Criteria) this;
+        }
+
+        public Criteria andUserIdNotBetween(Long value1, Long value2) {
+            addCriterion("user_id not between", value1, value2, "userId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdIsNull() {
+            addCriterion("out_user_id is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdIsNotNull() {
+            addCriterion("out_user_id is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdEqualTo(String value) {
+            addCriterion("out_user_id =", value, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdNotEqualTo(String value) {
+            addCriterion("out_user_id <>", value, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdGreaterThan(String value) {
+            addCriterion("out_user_id >", value, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdGreaterThanOrEqualTo(String value) {
+            addCriterion("out_user_id >=", value, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdLessThan(String value) {
+            addCriterion("out_user_id <", value, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdLessThanOrEqualTo(String value) {
+            addCriterion("out_user_id <=", value, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdLike(String value) {
+            addCriterion("out_user_id like", value, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdNotLike(String value) {
+            addCriterion("out_user_id not like", value, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdIn(List<String> values) {
+            addCriterion("out_user_id in", values, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdNotIn(List<String> values) {
+            addCriterion("out_user_id not in", values, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdBetween(String value1, String value2) {
+            addCriterion("out_user_id between", value1, value2, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutUserIdNotBetween(String value1, String value2) {
+            addCriterion("out_user_id not between", value1, value2, "outUserId");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformIsNull() {
+            addCriterion("platform is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformIsNotNull() {
+            addCriterion("platform is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformEqualTo(String value) {
+            addCriterion("platform =", value, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformNotEqualTo(String value) {
+            addCriterion("platform <>", value, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformGreaterThan(String value) {
+            addCriterion("platform >", value, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformGreaterThanOrEqualTo(String value) {
+            addCriterion("platform >=", value, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformLessThan(String value) {
+            addCriterion("platform <", value, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformLessThanOrEqualTo(String value) {
+            addCriterion("platform <=", value, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformLike(String value) {
+            addCriterion("platform like", value, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformNotLike(String value) {
+            addCriterion("platform not like", value, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformIn(List<String> values) {
+            addCriterion("platform in", values, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformNotIn(List<String> values) {
+            addCriterion("platform not in", values, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformBetween(String value1, String value2) {
+            addCriterion("platform between", value1, value2, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlatformNotBetween(String value1, String value2) {
+            addCriterion("platform not between", value1, value2, "platform");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyIsNull() {
+            addCriterion("strategy is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyIsNotNull() {
+            addCriterion("strategy is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyEqualTo(String value) {
+            addCriterion("strategy =", value, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyNotEqualTo(String value) {
+            addCriterion("strategy <>", value, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyGreaterThan(String value) {
+            addCriterion("strategy >", value, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyGreaterThanOrEqualTo(String value) {
+            addCriterion("strategy >=", value, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyLessThan(String value) {
+            addCriterion("strategy <", value, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyLessThanOrEqualTo(String value) {
+            addCriterion("strategy <=", value, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyLike(String value) {
+            addCriterion("strategy like", value, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyNotLike(String value) {
+            addCriterion("strategy not like", value, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyIn(List<String> values) {
+            addCriterion("strategy in", values, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyNotIn(List<String> values) {
+            addCriterion("strategy not in", values, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyBetween(String value1, String value2) {
+            addCriterion("strategy between", value1, value2, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andStrategyNotBetween(String value1, String value2) {
+            addCriterion("strategy not between", value1, value2, "strategy");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdIsNull() {
+            addCriterion("out_video_id is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdIsNotNull() {
+            addCriterion("out_video_id is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdEqualTo(String value) {
+            addCriterion("out_video_id =", value, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdNotEqualTo(String value) {
+            addCriterion("out_video_id <>", value, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdGreaterThan(String value) {
+            addCriterion("out_video_id >", value, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdGreaterThanOrEqualTo(String value) {
+            addCriterion("out_video_id >=", value, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdLessThan(String value) {
+            addCriterion("out_video_id <", value, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdLessThanOrEqualTo(String value) {
+            addCriterion("out_video_id <=", value, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdLike(String value) {
+            addCriterion("out_video_id like", value, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdNotLike(String value) {
+            addCriterion("out_video_id not like", value, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdIn(List<String> values) {
+            addCriterion("out_video_id in", values, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdNotIn(List<String> values) {
+            addCriterion("out_video_id not in", values, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdBetween(String value1, String value2) {
+            addCriterion("out_video_id between", value1, value2, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andOutVideoIdNotBetween(String value1, String value2) {
+            addCriterion("out_video_id not between", value1, value2, "outVideoId");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleIsNull() {
+            addCriterion("video_title is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleIsNotNull() {
+            addCriterion("video_title is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleEqualTo(String value) {
+            addCriterion("video_title =", value, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleNotEqualTo(String value) {
+            addCriterion("video_title <>", value, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleGreaterThan(String value) {
+            addCriterion("video_title >", value, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleGreaterThanOrEqualTo(String value) {
+            addCriterion("video_title >=", value, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleLessThan(String value) {
+            addCriterion("video_title <", value, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleLessThanOrEqualTo(String value) {
+            addCriterion("video_title <=", value, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleLike(String value) {
+            addCriterion("video_title like", value, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleNotLike(String value) {
+            addCriterion("video_title not like", value, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleIn(List<String> values) {
+            addCriterion("video_title in", values, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleNotIn(List<String> values) {
+            addCriterion("video_title not in", values, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleBetween(String value1, String value2) {
+            addCriterion("video_title between", value1, value2, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoTitleNotBetween(String value1, String value2) {
+            addCriterion("video_title not between", value1, value2, "videoTitle");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlIsNull() {
+            addCriterion("cover_url is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlIsNotNull() {
+            addCriterion("cover_url is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlEqualTo(String value) {
+            addCriterion("cover_url =", value, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlNotEqualTo(String value) {
+            addCriterion("cover_url <>", value, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlGreaterThan(String value) {
+            addCriterion("cover_url >", value, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlGreaterThanOrEqualTo(String value) {
+            addCriterion("cover_url >=", value, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlLessThan(String value) {
+            addCriterion("cover_url <", value, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlLessThanOrEqualTo(String value) {
+            addCriterion("cover_url <=", value, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlLike(String value) {
+            addCriterion("cover_url like", value, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlNotLike(String value) {
+            addCriterion("cover_url not like", value, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlIn(List<String> values) {
+            addCriterion("cover_url in", values, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlNotIn(List<String> values) {
+            addCriterion("cover_url not in", values, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlBetween(String value1, String value2) {
+            addCriterion("cover_url between", value1, value2, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andCoverUrlNotBetween(String value1, String value2) {
+            addCriterion("cover_url not between", value1, value2, "coverUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlIsNull() {
+            addCriterion("video_url is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlIsNotNull() {
+            addCriterion("video_url is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlEqualTo(String value) {
+            addCriterion("video_url =", value, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlNotEqualTo(String value) {
+            addCriterion("video_url <>", value, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlGreaterThan(String value) {
+            addCriterion("video_url >", value, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlGreaterThanOrEqualTo(String value) {
+            addCriterion("video_url >=", value, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlLessThan(String value) {
+            addCriterion("video_url <", value, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlLessThanOrEqualTo(String value) {
+            addCriterion("video_url <=", value, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlLike(String value) {
+            addCriterion("video_url like", value, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlNotLike(String value) {
+            addCriterion("video_url not like", value, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlIn(List<String> values) {
+            addCriterion("video_url in", values, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlNotIn(List<String> values) {
+            addCriterion("video_url not in", values, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlBetween(String value1, String value2) {
+            addCriterion("video_url between", value1, value2, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andVideoUrlNotBetween(String value1, String value2) {
+            addCriterion("video_url not between", value1, value2, "videoUrl");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationIsNull() {
+            addCriterion("duration is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationIsNotNull() {
+            addCriterion("duration is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationEqualTo(Long value) {
+            addCriterion("duration =", value, "duration");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationNotEqualTo(Long value) {
+            addCriterion("duration <>", value, "duration");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationGreaterThan(Long value) {
+            addCriterion("duration >", value, "duration");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationGreaterThanOrEqualTo(Long value) {
+            addCriterion("duration >=", value, "duration");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationLessThan(Long value) {
+            addCriterion("duration <", value, "duration");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationLessThanOrEqualTo(Long value) {
+            addCriterion("duration <=", value, "duration");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationIn(List<Long> values) {
+            addCriterion("duration in", values, "duration");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationNotIn(List<Long> values) {
+            addCriterion("duration not in", values, "duration");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationBetween(Long value1, Long value2) {
+            addCriterion("duration between", value1, value2, "duration");
+            return (Criteria) this;
+        }
+
+        public Criteria andDurationNotBetween(Long value1, Long value2) {
+            addCriterion("duration not between", value1, value2, "duration");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeIsNull() {
+            addCriterion("publish_time is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeIsNotNull() {
+            addCriterion("publish_time is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeEqualTo(String value) {
+            addCriterion("publish_time =", value, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeNotEqualTo(String value) {
+            addCriterion("publish_time <>", value, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeGreaterThan(String value) {
+            addCriterion("publish_time >", value, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeGreaterThanOrEqualTo(String value) {
+            addCriterion("publish_time >=", value, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeLessThan(String value) {
+            addCriterion("publish_time <", value, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeLessThanOrEqualTo(String value) {
+            addCriterion("publish_time <=", value, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeLike(String value) {
+            addCriterion("publish_time like", value, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeNotLike(String value) {
+            addCriterion("publish_time not like", value, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeIn(List<String> values) {
+            addCriterion("publish_time in", values, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeNotIn(List<String> values) {
+            addCriterion("publish_time not in", values, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeBetween(String value1, String value2) {
+            addCriterion("publish_time between", value1, value2, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPublishTimeNotBetween(String value1, String value2) {
+            addCriterion("publish_time not between", value1, value2, "publishTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntIsNull() {
+            addCriterion("play_cnt is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntIsNotNull() {
+            addCriterion("play_cnt is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntEqualTo(Integer value) {
+            addCriterion("play_cnt =", value, "playCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntNotEqualTo(Integer value) {
+            addCriterion("play_cnt <>", value, "playCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntGreaterThan(Integer value) {
+            addCriterion("play_cnt >", value, "playCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntGreaterThanOrEqualTo(Integer value) {
+            addCriterion("play_cnt >=", value, "playCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntLessThan(Integer value) {
+            addCriterion("play_cnt <", value, "playCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntLessThanOrEqualTo(Integer value) {
+            addCriterion("play_cnt <=", value, "playCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntIn(List<Integer> values) {
+            addCriterion("play_cnt in", values, "playCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntNotIn(List<Integer> values) {
+            addCriterion("play_cnt not in", values, "playCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntBetween(Integer value1, Integer value2) {
+            addCriterion("play_cnt between", value1, value2, "playCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andPlayCntNotBetween(Integer value1, Integer value2) {
+            addCriterion("play_cnt not between", value1, value2, "playCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntIsNull() {
+            addCriterion("like_cnt is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntIsNotNull() {
+            addCriterion("like_cnt is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntEqualTo(Integer value) {
+            addCriterion("like_cnt =", value, "likeCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntNotEqualTo(Integer value) {
+            addCriterion("like_cnt <>", value, "likeCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntGreaterThan(Integer value) {
+            addCriterion("like_cnt >", value, "likeCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntGreaterThanOrEqualTo(Integer value) {
+            addCriterion("like_cnt >=", value, "likeCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntLessThan(Integer value) {
+            addCriterion("like_cnt <", value, "likeCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntLessThanOrEqualTo(Integer value) {
+            addCriterion("like_cnt <=", value, "likeCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntIn(List<Integer> values) {
+            addCriterion("like_cnt in", values, "likeCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntNotIn(List<Integer> values) {
+            addCriterion("like_cnt not in", values, "likeCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntBetween(Integer value1, Integer value2) {
+            addCriterion("like_cnt between", value1, value2, "likeCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andLikeCntNotBetween(Integer value1, Integer value2) {
+            addCriterion("like_cnt not between", value1, value2, "likeCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntIsNull() {
+            addCriterion("share_cnt is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntIsNotNull() {
+            addCriterion("share_cnt is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntEqualTo(Integer value) {
+            addCriterion("share_cnt =", value, "shareCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntNotEqualTo(Integer value) {
+            addCriterion("share_cnt <>", value, "shareCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntGreaterThan(Integer value) {
+            addCriterion("share_cnt >", value, "shareCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntGreaterThanOrEqualTo(Integer value) {
+            addCriterion("share_cnt >=", value, "shareCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntLessThan(Integer value) {
+            addCriterion("share_cnt <", value, "shareCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntLessThanOrEqualTo(Integer value) {
+            addCriterion("share_cnt <=", value, "shareCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntIn(List<Integer> values) {
+            addCriterion("share_cnt in", values, "shareCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntNotIn(List<Integer> values) {
+            addCriterion("share_cnt not in", values, "shareCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntBetween(Integer value1, Integer value2) {
+            addCriterion("share_cnt between", value1, value2, "shareCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andShareCntNotBetween(Integer value1, Integer value2) {
+            addCriterion("share_cnt not between", value1, value2, "shareCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntIsNull() {
+            addCriterion("collection_cnt is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntIsNotNull() {
+            addCriterion("collection_cnt is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntEqualTo(Integer value) {
+            addCriterion("collection_cnt =", value, "collectionCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntNotEqualTo(Integer value) {
+            addCriterion("collection_cnt <>", value, "collectionCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntGreaterThan(Integer value) {
+            addCriterion("collection_cnt >", value, "collectionCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntGreaterThanOrEqualTo(Integer value) {
+            addCriterion("collection_cnt >=", value, "collectionCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntLessThan(Integer value) {
+            addCriterion("collection_cnt <", value, "collectionCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntLessThanOrEqualTo(Integer value) {
+            addCriterion("collection_cnt <=", value, "collectionCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntIn(List<Integer> values) {
+            addCriterion("collection_cnt in", values, "collectionCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntNotIn(List<Integer> values) {
+            addCriterion("collection_cnt not in", values, "collectionCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntBetween(Integer value1, Integer value2) {
+            addCriterion("collection_cnt between", value1, value2, "collectionCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCollectionCntNotBetween(Integer value1, Integer value2) {
+            addCriterion("collection_cnt not between", value1, value2, "collectionCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntIsNull() {
+            addCriterion("comment_cnt is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntIsNotNull() {
+            addCriterion("comment_cnt is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntEqualTo(Integer value) {
+            addCriterion("comment_cnt =", value, "commentCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntNotEqualTo(Integer value) {
+            addCriterion("comment_cnt <>", value, "commentCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntGreaterThan(Integer value) {
+            addCriterion("comment_cnt >", value, "commentCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntGreaterThanOrEqualTo(Integer value) {
+            addCriterion("comment_cnt >=", value, "commentCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntLessThan(Integer value) {
+            addCriterion("comment_cnt <", value, "commentCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntLessThanOrEqualTo(Integer value) {
+            addCriterion("comment_cnt <=", value, "commentCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntIn(List<Integer> values) {
+            addCriterion("comment_cnt in", values, "commentCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntNotIn(List<Integer> values) {
+            addCriterion("comment_cnt not in", values, "commentCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntBetween(Integer value1, Integer value2) {
+            addCriterion("comment_cnt between", value1, value2, "commentCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCommentCntNotBetween(Integer value1, Integer value2) {
+            addCriterion("comment_cnt not between", value1, value2, "commentCnt");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleIsNull() {
+            addCriterion("crawler_rule is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleIsNotNull() {
+            addCriterion("crawler_rule is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleEqualTo(String value) {
+            addCriterion("crawler_rule =", value, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleNotEqualTo(String value) {
+            addCriterion("crawler_rule <>", value, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleGreaterThan(String value) {
+            addCriterion("crawler_rule >", value, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleGreaterThanOrEqualTo(String value) {
+            addCriterion("crawler_rule >=", value, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleLessThan(String value) {
+            addCriterion("crawler_rule <", value, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleLessThanOrEqualTo(String value) {
+            addCriterion("crawler_rule <=", value, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleLike(String value) {
+            addCriterion("crawler_rule like", value, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleNotLike(String value) {
+            addCriterion("crawler_rule not like", value, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleIn(List<String> values) {
+            addCriterion("crawler_rule in", values, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleNotIn(List<String> values) {
+            addCriterion("crawler_rule not in", values, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleBetween(String value1, String value2) {
+            addCriterion("crawler_rule between", value1, value2, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andCrawlerRuleNotBetween(String value1, String value2) {
+            addCriterion("crawler_rule not between", value1, value2, "crawlerRule");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthIsNull() {
+            addCriterion("width is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthIsNotNull() {
+            addCriterion("width is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthEqualTo(Integer value) {
+            addCriterion("width =", value, "width");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthNotEqualTo(Integer value) {
+            addCriterion("width <>", value, "width");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthGreaterThan(Integer value) {
+            addCriterion("width >", value, "width");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthGreaterThanOrEqualTo(Integer value) {
+            addCriterion("width >=", value, "width");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthLessThan(Integer value) {
+            addCriterion("width <", value, "width");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthLessThanOrEqualTo(Integer value) {
+            addCriterion("width <=", value, "width");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthIn(List<Integer> values) {
+            addCriterion("width in", values, "width");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthNotIn(List<Integer> values) {
+            addCriterion("width not in", values, "width");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthBetween(Integer value1, Integer value2) {
+            addCriterion("width between", value1, value2, "width");
+            return (Criteria) this;
+        }
+
+        public Criteria andWidthNotBetween(Integer value1, Integer value2) {
+            addCriterion("width not between", value1, value2, "width");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightIsNull() {
+            addCriterion("height is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightIsNotNull() {
+            addCriterion("height is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightEqualTo(Integer value) {
+            addCriterion("height =", value, "height");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightNotEqualTo(Integer value) {
+            addCriterion("height <>", value, "height");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightGreaterThan(Integer value) {
+            addCriterion("height >", value, "height");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightGreaterThanOrEqualTo(Integer value) {
+            addCriterion("height >=", value, "height");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightLessThan(Integer value) {
+            addCriterion("height <", value, "height");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightLessThanOrEqualTo(Integer value) {
+            addCriterion("height <=", value, "height");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightIn(List<Integer> values) {
+            addCriterion("height in", values, "height");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightNotIn(List<Integer> values) {
+            addCriterion("height not in", values, "height");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightBetween(Integer value1, Integer value2) {
+            addCriterion("height between", value1, value2, "height");
+            return (Criteria) this;
+        }
+
+        public Criteria andHeightNotBetween(Integer value1, Integer value2) {
+            addCriterion("height not between", value1, value2, "height");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeIsNull() {
+            addCriterion("create_time is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeIsNotNull() {
+            addCriterion("create_time is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeEqualTo(Date value) {
+            addCriterion("create_time =", value, "createTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeNotEqualTo(Date value) {
+            addCriterion("create_time <>", value, "createTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeGreaterThan(Date value) {
+            addCriterion("create_time >", value, "createTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeGreaterThanOrEqualTo(Date value) {
+            addCriterion("create_time >=", value, "createTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeLessThan(Date value) {
+            addCriterion("create_time <", value, "createTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeLessThanOrEqualTo(Date value) {
+            addCriterion("create_time <=", value, "createTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeIn(List<Date> values) {
+            addCriterion("create_time in", values, "createTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeNotIn(List<Date> values) {
+            addCriterion("create_time not in", values, "createTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeBetween(Date value1, Date value2) {
+            addCriterion("create_time between", value1, value2, "createTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andCreateTimeNotBetween(Date value1, Date value2) {
+            addCriterion("create_time not between", value1, value2, "createTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeIsNull() {
+            addCriterion("update_time is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeIsNotNull() {
+            addCriterion("update_time is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeEqualTo(Date value) {
+            addCriterion("update_time =", value, "updateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeNotEqualTo(Date value) {
+            addCriterion("update_time <>", value, "updateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeGreaterThan(Date value) {
+            addCriterion("update_time >", value, "updateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeGreaterThanOrEqualTo(Date value) {
+            addCriterion("update_time >=", value, "updateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeLessThan(Date value) {
+            addCriterion("update_time <", value, "updateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeLessThanOrEqualTo(Date value) {
+            addCriterion("update_time <=", value, "updateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeIn(List<Date> values) {
+            addCriterion("update_time in", values, "updateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeNotIn(List<Date> values) {
+            addCriterion("update_time not in", values, "updateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeBetween(Date value1, Date value2) {
+            addCriterion("update_time between", value1, value2, "updateTime");
+            return (Criteria) this;
+        }
+
+        public Criteria andUpdateTimeNotBetween(Date value1, Date value2) {
+            addCriterion("update_time not between", value1, value2, "updateTime");
+            return (Criteria) this;
+        }
+    }
+
+    /**
+     * This class was generated by MyBatis Generator.
+     * This class corresponds to the database table crawler_video
+     *
+     * @mbg.generated do_not_delete_during_merge
+     */
+    public static class Criteria extends GeneratedCriteria {
+
+        protected Criteria() {
+            super();
+        }
+    }
+
+    /**
+     * This class was generated by MyBatis Generator.
+     * This class corresponds to the database table crawler_video
+     *
+     * @mbg.generated
+     */
+    public static class Criterion {
+        private final String condition;
+
+        private Object value;
+
+        private Object secondValue;
+
+        private boolean noValue;
+
+        private boolean singleValue;
+
+        private boolean betweenValue;
+
+        private boolean listValue;
+
+        private final String typeHandler;
+
+        protected Criterion(String condition) {
+            super();
+            this.condition = condition;
+            this.typeHandler = null;
+            this.noValue = true;
+        }
+
+        protected Criterion(String condition, Object value, String typeHandler) {
+            super();
+            this.condition = condition;
+            this.value = value;
+            this.typeHandler = typeHandler;
+            if (value instanceof List<?>) {
+                this.listValue = true;
+            } else {
+                this.singleValue = true;
+            }
+        }
+
+        protected Criterion(String condition, Object value) {
+            this(condition, value, null);
+        }
+
+        protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
+            super();
+            this.condition = condition;
+            this.value = value;
+            this.secondValue = secondValue;
+            this.typeHandler = typeHandler;
+            this.betweenValue = true;
+        }
+
+        protected Criterion(String condition, Object value, Object secondValue) {
+            this(condition, value, secondValue, null);
+        }
+
+        public String getCondition() {
+            return condition;
+        }
+
+        public Object getValue() {
+            return value;
+        }
+
+        public Object getSecondValue() {
+            return secondValue;
+        }
+
+        public boolean isNoValue() {
+            return noValue;
+        }
+
+        public boolean isSingleValue() {
+            return singleValue;
+        }
+
+        public boolean isBetweenValue() {
+            return betweenValue;
+        }
+
+        public boolean isListValue() {
+            return listValue;
+        }
+
+        public String getTypeHandler() {
+            return typeHandler;
+        }
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/model/vo/CrawlerVideoVO.java

@@ -0,0 +1,290 @@
+package com.tzld.crawler.etl.model.vo;
+
+/**
+ * @author ehlxr
+ */
+public class CrawlerVideoVO {
+    /**
+     * 站内用户ID
+     */
+    private Long userId;
+
+    /**
+     * 站外用户ID
+     */
+    private String outUserId;
+    /**
+     * 站外用户昵称
+     */
+    private String userName;
+    /**
+     * 站外用户头像
+     */
+    private String avatarUrl;
+
+    /**
+     * 平台:youtube，wechat，小年糕，好看视频
+     */
+    private String platform;
+
+    /**
+     * 策略:定向爬虫策略，热榜爬虫策略，小时榜爬虫策略，推荐榜爬虫策略
+     */
+    private String strategy;
+
+    /**
+     * 站外视频ID
+     */
+    private String outVideoId;
+
+    /**
+     * 视频标题
+     */
+    private String videoTitle;
+
+    /**
+     * 视频封面
+     */
+    private String coverUrl;
+
+    /**
+     * 视频播放地址
+     */
+    private String videoUrl;
+
+    /**
+     * 视频时长
+     */
+    private Long duration;
+
+    /**
+     * 站外视频发布时间
+     */
+    private String publishTime;
+
+    /**
+     * 播放量
+     */
+    private Integer playCnt;
+
+    /**
+     * 点赞量
+     */
+    private Integer likeCnt;
+
+    /**
+     * 分享量
+     */
+    private Integer shareCnt;
+
+    /**
+     * 收藏量
+     */
+    private Integer collectionCnt;
+
+    /**
+     * 评论量
+     */
+    private Integer commentCnt;
+
+    /**
+     * 抓取时条件:{'play_cnt': 0, 'comment_cnt': 0, 'like_cnt': 0, 'duration': 60, 'publish_time': 10, 'video_width': 720, 'video_height': 720}
+     */
+    private String crawlerRule;
+
+    /**
+     * 宽
+     */
+    private Integer width;
+
+    /**
+     * 高
+     */
+    private Integer height;
+
+    @Override
+    public String toString() {
+        return "CrawlerVideoVO{" +
+                "userId=" + userId +
+                ", outUserId='" + outUserId + '\'' +
+                ", userName='" + userName + '\'' +
+                ", avatarUrl='" + avatarUrl + '\'' +
+                ", platform='" + platform + '\'' +
+                ", strategy='" + strategy + '\'' +
+                ", outVideoId='" + outVideoId + '\'' +
+                ", videoTitle='" + videoTitle + '\'' +
+                ", coverUrl='" + coverUrl + '\'' +
+                ", videoUrl='" + videoUrl + '\'' +
+                ", duration=" + duration +
+                ", publishTime='" + publishTime + '\'' +
+                ", playCnt=" + playCnt +
+                ", likeCnt=" + likeCnt +
+                ", shareCnt=" + shareCnt +
+                ", collectionCnt=" + collectionCnt +
+                ", commentCnt=" + commentCnt +
+                ", crawlerRule='" + crawlerRule + '\'' +
+                ", width=" + width +
+                ", height=" + height +
+                '}';
+    }
+
+    public String getUserName() {
+        return userName;
+    }
+
+    public void setUserName(String userName) {
+        this.userName = userName;
+    }
+
+    public String getAvatarUrl() {
+        return avatarUrl;
+    }
+
+    public void setAvatarUrl(String avatarUrl) {
+        this.avatarUrl = avatarUrl;
+    }
+
+    public Long getUserId() {
+        return userId;
+    }
+
+    public void setUserId(Long userId) {
+        this.userId = userId;
+    }
+
+    public String getOutUserId() {
+        return outUserId;
+    }
+
+    public void setOutUserId(String outUserId) {
+        this.outUserId = outUserId;
+    }
+
+    public String getPlatform() {
+        return platform;
+    }
+
+    public void setPlatform(String platform) {
+        this.platform = platform;
+    }
+
+    public String getStrategy() {
+        return strategy;
+    }
+
+    public void setStrategy(String strategy) {
+        this.strategy = strategy;
+    }
+
+    public String getOutVideoId() {
+        return outVideoId;
+    }
+
+    public void setOutVideoId(String outVideoId) {
+        this.outVideoId = outVideoId;
+    }
+
+    public String getVideoTitle() {
+        return videoTitle;
+    }
+
+    public void setVideoTitle(String videoTitle) {
+        this.videoTitle = videoTitle;
+    }
+
+    public String getCoverUrl() {
+        return coverUrl;
+    }
+
+    public void setCoverUrl(String coverUrl) {
+        this.coverUrl = coverUrl;
+    }
+
+    public String getVideoUrl() {
+        return videoUrl;
+    }
+
+    public void setVideoUrl(String videoUrl) {
+        this.videoUrl = videoUrl;
+    }
+
+    public Long getDuration() {
+        return duration;
+    }
+
+    public void setDuration(Long duration) {
+        this.duration = duration;
+    }
+
+    public String getPublishTime() {
+        return publishTime;
+    }
+
+    public void setPublishTime(String publishTime) {
+        this.publishTime = publishTime;
+    }
+
+    public Integer getPlayCnt() {
+        return playCnt;
+    }
+
+    public void setPlayCnt(Integer playCnt) {
+        this.playCnt = playCnt;
+    }
+
+    public Integer getLikeCnt() {
+        return likeCnt;
+    }
+
+    public void setLikeCnt(Integer likeCnt) {
+        this.likeCnt = likeCnt;
+    }
+
+    public Integer getShareCnt() {
+        return shareCnt;
+    }
+
+    public void setShareCnt(Integer shareCnt) {
+        this.shareCnt = shareCnt;
+    }
+
+    public Integer getCollectionCnt() {
+        return collectionCnt;
+    }
+
+    public void setCollectionCnt(Integer collectionCnt) {
+        this.collectionCnt = collectionCnt;
+    }
+
+    public Integer getCommentCnt() {
+        return commentCnt;
+    }
+
+    public void setCommentCnt(Integer commentCnt) {
+        this.commentCnt = commentCnt;
+    }
+
+    public String getCrawlerRule() {
+        return crawlerRule;
+    }
+
+    public void setCrawlerRule(String crawlerRule) {
+        this.crawlerRule = crawlerRule;
+    }
+
+    public Integer getWidth() {
+        return width;
+    }
+
+    public void setWidth(Integer width) {
+        this.width = width;
+    }
+
+    public Integer getHeight() {
+        return height;
+    }
+
+    public void setHeight(Integer height) {
+        this.height = height;
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/model/vo/WxVideoVO.java

@@ -0,0 +1,48 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.model.vo;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 19:32.
+ */
+public class WxVideoVO {
+    private Long id;
+
+    @Override
+    public String toString() {
+        return "WxVideoVO{" +
+                "id=" + id +
+                '}';
+    }
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/mq/EtlMQConsumer.java

@@ -0,0 +1,158 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.mq;
+
+import com.alibaba.fastjson.JSONObject;
+import com.aliyun.mq.http.MQClient;
+import com.aliyun.mq.http.MQConsumer;
+import com.aliyun.mq.http.model.Message;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.service.EtlService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 15:24.
+ */
+@Component
+public class EtlMQConsumer {
+    private static final Logger log = LoggerFactory.getLogger(EtlMQConsumer.class);
+    private final EtlService etlService;
+    @Value("${rocketmq.accessKey:}")
+    private String accessKey;
+    @Value("${rocketmq.secretKey:}")
+    private String secretKey;
+    @Value("${rocketmq.httpEndpoint:}")
+    private String httpEndpoint;
+    @Value("${rocketmq.instanceId:}")
+    private String instanceId;
+    @Value("${rocketmq.crawler.etl.topic:}")
+    private String topic;
+    @Value("${rocketmq.crawler.etl.groupid:}")
+    private String groupId;
+
+    @ApolloJsonValue("${crawler.etl.priority:{}}")
+    private Map<String, Integer> priorityMap;
+
+    private MQClient mqClient;
+    private ThreadPoolExecutor priorityPool;
+
+    public EtlMQConsumer(EtlService etlService) {
+        this.etlService = etlService;
+    }
+
+    @PostConstruct
+    public void init() {
+        mqClient = new MQClient(httpEndpoint, accessKey, secretKey);
+        priorityPool = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().availableProcessors() * 2, 60L,
+                TimeUnit.SECONDS, new PriorityBlockingQueue<>(),
+                new ThreadFactoryBuilder().setNameFormat("priority-etl-pool-%d").build());
+
+        new Thread(this::consumeMsg).start();
+    }
+
+    class RunnablePriority implements Runnable, Comparable<RunnablePriority> {
+        private final CrawlerVideoVO video;
+
+        public RunnablePriority(CrawlerVideoVO video) {
+            this.video = video;
+        }
+
+        public Integer getPriority() {
+            return priorityMap.getOrDefault(video.getPlatform(), -1);
+        }
+
+        /**
+         * 优先级比较
+         */
+        @Override
+        public int compareTo(RunnablePriority o) {
+            if (this.getPriority() < o.getPriority()) {
+                return 1;
+            } else if (this.getPriority() > o.getPriority()) {
+                return -1;
+            }
+            return 0;
+        }
+
+        @Override
+        public void run() {
+            try {
+                log.info("Thread pool info \nTaskCount: {}\nCompletedTaskCount: {}\nActiveCount: {}\nQueue size: {}\nQueue elements: {}",
+                        priorityPool.getTaskCount(), priorityPool.getCompletedTaskCount(), priorityPool.getActiveCount(),
+                        priorityPool.getQueue().size(), priorityPool.getQueue());
+                log.info("deal video: {} priority: {}", video, getPriority());
+                etlService.deal(video);
+                log.info("Thread pool info \nTaskCount: {}\nCompletedTaskCount: {}\nActiveCount: {}\nQueue size: {}\nQueue elements: {}",
+                        priorityPool.getTaskCount(), priorityPool.getCompletedTaskCount(), priorityPool.getActiveCount(),
+                        priorityPool.getQueue().size(), priorityPool.getQueue());
+            } catch (Exception e) {
+                log.error("deal video {} error.", video, e);
+            }
+        }
+
+        @Override
+        public String toString() {
+            return video.getPlatform() + ":" + video.getOutVideoId();
+        }
+    }
+
+    private void consumeMsg() {
+        final MQConsumer consumer = mqClient.getConsumer(instanceId, topic, groupId, null);
+        do {
+            try {
+                List<Message> messages = consumer.consumeMessage(10, 10);
+                if (messages == null || messages.isEmpty()) {
+                    log.info("No new message, continue");
+                    continue;
+                }
+
+                List<String> handles = new ArrayList<>();
+                messages.forEach(message -> {
+                    log.info("Receive message: {} from topic: {}, group: {}", message, topic, groupId);
+                    CrawlerVideoVO video = JSONObject.parseObject(message.getMessageBodyString(), CrawlerVideoVO.class);
+                    priorityPool.execute(new RunnablePriority(video));
+                    handles.add(message.getReceiptHandle());
+                });
+                consumer.ackMessage(handles);
+            } catch (Throwable e) {
+                log.error("Consume message from topic: {}, group: {} error", topic, groupId, e);
+            }
+        } while (true);
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/service/EtlService.java

@@ -0,0 +1,37 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.service;
+
+import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 15:22.
+ */
+public interface EtlService {
+    void deal(CrawlerVideoVO video);
+}
+
+

etl-core/src/main/java/com/tzld/crawler/etl/service/SlsService.java

@@ -0,0 +1,87 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.service;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.tzld.commons.aliyun.log.AliyunLogManager;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-19 19:41.
+ */
+@Component
+public class SlsService {
+    private final AliyunLogManager aliyunLogManager;
+    @Value("${aliyun.log.logstore.crawler:}")
+    private String crawlerLogstore;
+    @Value("${aliyun.log.project:}")
+    private String logProject;
+
+    private Executor pool;
+
+    public SlsService(AliyunLogManager aliyunLogManager) {
+        this.aliyunLogManager = aliyunLogManager;
+    }
+
+    @PostConstruct
+    public void init() {
+        pool = Executors.newFixedThreadPool(2);
+    }
+
+    public void log(Map<String, Object> param) {
+        HashMap<String, Object> map = Maps.newHashMap(param);
+        map.put("timestamp", System.currentTimeMillis() / 1000);
+        pool.execute(() -> aliyunLogManager.sendLog(logProject, crawlerLogstore, "", param));
+    }
+
+    public void log(String k1, Object v1) {
+        log(ImmutableMap.of(k1, v1));
+    }
+
+    public void log(String k1, Object v1, String k2, Object v2) {
+        log(ImmutableMap.of(k1, v1, k2, v2));
+    }
+
+    public void log(String k1, Object v1, String k2, Object v2, String k3, Object v3) {
+        log(ImmutableMap.of(k1, v1, k2, v2, k3, v3));
+    }
+
+    public void log(String k1, Object v1, String k2, Object v2, String k3, Object v3, String k4, Object v4) {
+        log(ImmutableMap.of(k1, v1, k2, v2, k3, v3, k4, v4));
+    }
+
+    public void log(String k1, Object v1, String k2, Object v2, String k3, Object v3, String k4, Object v4, String k5, Object v5) {
+        log(ImmutableMap.of(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5));
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/service/feign/LongVideoFeign.java

@@ -0,0 +1,49 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrg@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.service.feign;
+
+import com.tzld.crawler.etl.common.base.CommonResponse;
+import com.tzld.crawler.etl.model.param.CrawlerVideoSendParam;
+import com.tzld.crawler.etl.model.vo.WxVideoVO;
+import com.tzld.crawler.etl.service.feign.fallback.LongVideoFeignFallbackFactory;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 17:42.
+ */
+@FeignClient(value = "longvideoapi", url = "${longvideo.feign.url:}",
+        path = "/longvideoapi", fallbackFactory = LongVideoFeignFallbackFactory.class)
+public interface LongVideoFeign {
+    /**
+     * 发布爬虫视频
+     */
+    @RequestMapping(value = "/crawler/video/send", method = RequestMethod.POST, consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
+    CommonResponse<WxVideoVO> crawlerVideoSend(CrawlerVideoSendParam request);
+}

etl-core/src/main/java/com/tzld/crawler/etl/service/feign/fallback/LongVideoFeignFallbackFactory.java

@@ -0,0 +1,66 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2022 xrv <xrg@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.service.feign.fallback;
+
+import com.tzld.crawler.etl.common.base.CommonResponse;
+import com.tzld.crawler.etl.common.enums.ExceptionEnum;
+import com.tzld.crawler.etl.model.param.CrawlerVideoSendParam;
+import com.tzld.crawler.etl.model.vo.WxVideoVO;
+import com.tzld.crawler.etl.service.feign.LongVideoFeign;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.cloud.openfeign.FallbackFactory;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author ehlxr
+ * @since 2022-10-10 11:43.
+ */
+@Service
+public class LongVideoFeignFallbackFactory implements FallbackFactory<LongVideoFeignFallback> {
+    @Override
+    public LongVideoFeignFallback create(Throwable cause) {
+        return new LongVideoFeignFallback(cause);
+    }
+}
+
+class LongVideoFeignFallback implements LongVideoFeign {
+    private final Logger log = LoggerFactory.getLogger(this.getClass());
+    private final Throwable th;
+
+    LongVideoFeignFallback(Throwable th) {
+        this.th = th;
+    }
+
+    @Override
+    public CommonResponse<WxVideoVO> crawlerVideoSend(CrawlerVideoSendParam request) {
+        CommonResponse<WxVideoVO> response = new CommonResponse<>();
+        response.setMsg(th.getMessage());
+        response.setCode(ExceptionEnum.INVOKE_VIDEOAPI_ERROR.getCode());
+
+        log.error("request: {}, error: {}", request, th.getMessage());
+        return response;
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/service/impl/EtlServiceImpl.java

@@ -0,0 +1,269 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.service.impl;
+
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.collect.Lists;
+import com.tzld.commons.aliyun.oss.AliyunOssManager;
+import com.tzld.crawler.etl.common.base.CommonResponse;
+import com.tzld.crawler.etl.common.base.Constant;
+import com.tzld.crawler.etl.common.enums.ExceptionEnum;
+import com.tzld.crawler.etl.common.exception.CommonException;
+import com.tzld.crawler.etl.dao.mapper.CrawlerVideoMapper;
+import com.tzld.crawler.etl.model.dto.StrategyDataDto;
+import com.tzld.crawler.etl.model.param.CrawlerVideoSendParam;
+import com.tzld.crawler.etl.model.po.CrawlerVideo;
+import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.model.vo.WxVideoVO;
+import com.tzld.crawler.etl.service.EtlService;
+import com.tzld.crawler.etl.service.SlsService;
+import com.tzld.crawler.etl.service.feign.LongVideoFeign;
+import com.tzld.crawler.etl.service.strategy.StrategyHandlerService;
+import com.tzld.crawler.etl.util.FeishuUtils;
+import com.tzld.crawler.etl.util.FileUtils;
+import com.tzld.crawler.etl.util.MD5Util;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.time.*;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 15:23.
+ */
+@Service
+public class EtlServiceImpl implements EtlService {
+    private final static Logger log = LoggerFactory.getLogger(EtlServiceImpl.class);
+    private final StrategyHandlerService strategyHandlerService;
+    private final AliyunOssManager aliyunOssManager;
+    private final LongVideoFeign longVideoFeign;
+    private final CrawlerVideoMapper crawlerVideoMapper;
+    private final SlsService slsService;
+
+    @Value("${download.file.path:/data/crawler/videos}")
+    private String downloadPath;
+    @Value("${aliyun.oss.video.bucket:art-pubbucket}")
+    private String ossBucket;
+    @Value("${env}")
+    private String env;
+    @Value("${feishu.appid:}")
+    private String feishuAppid;
+    @Value("${feishu.appsecret:}")
+    private String feishuAppsecret;
+    @ApolloJsonValue("${feishu.sheet.token.map:{}}")
+    private Map<String, String> feishuSheetTokenMap;
+    @ApolloJsonValue("${feishu.sheet.id.map:{}}")
+    private Map<String, String> feishuSheetIdMap;
+    @ApolloJsonValue("${feishu.range.map:{}}")
+    private Map<String, String> feishuRangeMap;
+    @Value("${admin.ums.url:https://testadmin.piaoquantv.com/ums/user/%d/post}")
+    private String adminUmsUrl;
+
+    private Executor pool;
+
+    public EtlServiceImpl(StrategyHandlerService strategyHandlerService, AliyunOssManager aliyunOssManager,
+                          LongVideoFeign longVideoFeign, CrawlerVideoMapper crawlerVideoMapper, SlsService slsService) {
+        this.strategyHandlerService = strategyHandlerService;
+        this.aliyunOssManager = aliyunOssManager;
+        this.longVideoFeign = longVideoFeign;
+        this.crawlerVideoMapper = crawlerVideoMapper;
+        this.slsService = slsService;
+    }
+
+    @Override
+    public void deal(CrawlerVideoVO param) {
+        String title = param.getVideoTitle();
+        String videoUrl = param.getVideoUrl();
+        String coverUrl = param.getCoverUrl();
+        String platform = param.getPlatform();
+        String strategy = param.getStrategy();
+        try {
+            // 1.策略应用
+            // 获取用户选择策略
+            List<String> stategies = Lists.newArrayList("titleScore");
+
+            StrategyDataDto data = strategyHandlerService.execute(stategies, param);
+            if (data == null) {
+                return;
+            }
+
+            // 2.视频文件下载、上传 OSS、清理视频信息
+            String videoPath = url2oss(videoUrl, "longvideo/crawler_local/data", title, platform, strategy);
+
+            // 3.视频封面下载、上传 OSS、清理视频信息
+            String coverPath = url2oss(coverUrl, "longvideo/crawler_local/image", title, platform, strategy);
+
+            // 4.视频发布
+            CrawlerVideoSendParam request = new CrawlerVideoSendParam();
+            request.setLoginUid(data.getUserId());
+            request.setAppType(888888);
+            request.setTitle(title);
+            request.setVideoPath(videoPath);
+            request.setCoverImgPath(coverPath);
+            request.setTotalTime(data.getDuration());
+            request.setVersionCode(1);
+            request.setViewStatus(1);
+            request.setCrawlerSrcId(data.getOutVideoId());
+            request.setCrawlerSrcCode(platform.toUpperCase());
+            LocalDateTime localDateTime = LocalDateTime.parse(data.getPublishTime(), DateTimeFormatter.ofPattern(Constant.STANDARD_FORMAT));
+            request.setCrawlerSrcPublishTimestamp(localDateTime.toInstant(ZoneOffset.of("+8")).toEpochMilli());
+            request.setCrawlerTaskTimestamp(System.currentTimeMillis());
+            CommonResponse<WxVideoVO> response = longVideoFeign.crawlerVideoSend(request);
+            log.info("crawler data send request: {}, response: {}", request, response);
+            if (!response.isSuccess()) {
+                throw new CommonException(ExceptionEnum.INVOKE_VIDEOAPI_ERROR, "invoke crawler data send failed!" + response);
+            }
+
+            // 5.视频元信息更新
+
+            // 6.保存信息到数据库
+            CrawlerVideo crawlerVideo = new CrawlerVideo();
+            BeanUtils.copyProperties(data, crawlerVideo);
+            crawlerVideo.setVideoId(response.getData().getId());
+            String insertSql = "insert into crawler_video(video_id, user_id, out_user_id, platform, strategy, out_video_id, video_title," +
+                    " cover_url, video_url, duration, publish_time, play_cnt, crawler_rule, width, height) values(" + crawlerVideo.getVideoId() + "," +
+                    crawlerVideo.getUserId() + "," +
+                    crawlerVideo.getOutUserId() + "," +
+                    crawlerVideo.getPlatform() + "," +
+                    crawlerVideo.getStrategy() + "," +
+                    crawlerVideo.getOutVideoId() + "," +
+                    crawlerVideo.getVideoTitle() + "," +
+                    crawlerVideo.getCoverUrl() + "," +
+                    crawlerVideo.getVideoUrl() + "," +
+                    crawlerVideo.getDuration() + "," +
+                    crawlerVideo.getPublishTime() + "," +
+                    crawlerVideo.getPlayCnt() + "," +
+                    crawlerVideo.getCrawlerRule() + "," +
+                    crawlerVideo.getWidth() + "," +
+                    crawlerVideo.getHeight() + ")";
+            slsService.log("message", "insert_sql: " + insertSql, "crawler", platform, "mode", strategy);
+            crawlerVideoMapper.insertSelective(crawlerVideo);
+            slsService.log("message", "视频信息写入数据库成功", "crawler", platform, "mode", strategy);
+
+            // 7.视频写入飞书
+            pool.execute(() -> {
+                try {
+                    String fsResp = FeishuUtils.insertRows(feishuAppid, feishuAppsecret, feishuSheetTokenMap.get(platform),
+                            feishuSheetIdMap.get(platform), 1, 2);
+                    log.debug("insert columns to feishu sheet response is {}", fsResp);
+
+                    List<List<Object>> values = new ArrayList<>();
+                    List<Object> value = new ArrayList<>();
+                    DateTimeFormatter formatter = DateTimeFormatter.ofPattern(Constant.STANDARD_FORMAT);
+                    value.add(data.getTitleScore());
+                    value.add(data.getOutVideoId());
+                    value.add(formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.systemDefault())));
+                    value.add(strategy);
+                    value.add(crawlerVideo.getVideoId());
+                    value.add(data.getVideoTitle());
+                    value.add(String.format(adminUmsUrl, crawlerVideo.getVideoId()));
+                    value.add(data.getPlayCnt());
+                    value.add(data.getCommentCnt());
+                    value.add(data.getLikeCnt());
+                    value.add(data.getShareCnt());
+                    value.add(data.getDuration());
+                    value.add(data.getWidth() + "*" + data.getHeight());
+                    value.add(data.getPublishTime());
+                    value.add(data.getUserName());
+                    value.add(data.getUserId());
+                    value.add(data.getAvatarUrl());
+                    value.add(data.getCoverUrl());
+                    value.add(data.getVideoUrl());
+                    values.add(value);
+
+                    fsResp = FeishuUtils.updateValues(feishuAppid, feishuAppsecret, feishuSheetTokenMap.get(platform),
+                            feishuRangeMap.get(platform), values);
+                    log.debug("update feishu sheet value response is {}", fsResp);
+
+                    slsService.log("message", "视频已保存至云文档", "crawler", platform, "mode", strategy);
+                } catch (Exception e) {
+                    log.error("save data to feishu sheet error. platform {}, strategy {}", platform, strategy, e);
+                }
+            });
+        } catch (Exception e) {
+            log.error("etl server deal {} failed.", param, e);
+            throw new CommonException(ExceptionEnum.SYSTEM_ERROR, "etl server deal error: " + e.getMessage());
+        }
+    }
+
+    private String url2oss(String fileUrl, String filePath, String title, String crawler, String mode) throws IOException {
+        String titleMmd5 = MD5Util.md5(title);
+        String curDate = LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE);
+        String videoFilePath = filePath + File.separator + env + File.separator + curDate;
+        String localFilePath = downloadPath + File.separator + videoFilePath;
+
+        File localFile = new File(localFilePath);
+        if (!localFile.exists()) {
+            if (!localFile.mkdirs()) {
+                throw new CommonException(ExceptionEnum.SYSTEM_ERROR, "mkdir " + localFilePath + " error!");
+            }
+        }
+
+        // 下载文件
+        int retry = 0;
+        boolean downloadFlag = false;
+        while (retry < 3) {
+            retry++;
+            downloadFlag = FileUtils.download(fileUrl, localFilePath + File.separator + titleMmd5);
+            if (downloadFlag) {
+                break;
+            }
+            log.warn("download file failed. retry times: {}, from file {} to {}. ", fileUrl, filePath, retry);
+        }
+        if (!downloadFlag) {
+            throw new CommonException(ExceptionEnum.SYSTEM_ERROR, "download file " + fileUrl + " failed!");
+        }
+
+        // 文件上传 OSS
+        slsService.log("message", "开始上传视频... ", "crawler", crawler, "mode", mode);
+        log.info("begin upload: {}/{} to oss key: {}/{}", localFilePath, titleMmd5, videoFilePath, titleMmd5);
+        aliyunOssManager.putObject(ossBucket, videoFilePath + File.separator + titleMmd5, Files.newInputStream(Paths.get(localFilePath + File.separator + titleMmd5)));
+
+        // 文件清理
+        log.info("begin delete: {}/{}", localFilePath, titleMmd5);
+        Files.deleteIfExists(Paths.get(new File(localFilePath + File.separator + titleMmd5).getPath()));
+
+        return videoFilePath + File.separator + titleMmd5;
+    }
+
+    @PostConstruct
+    public void init() {
+        pool = Executors.newFixedThreadPool(1);
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/StrategyAbstractHandler.java

@@ -0,0 +1,45 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.service.strategy;
+
+import com.tzld.crawler.etl.model.dto.StrategyDataDto;
+import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 15:07.
+ */
+public abstract class StrategyAbstractHandler {
+    /**
+     * 业务处理实现抽象方法
+     */
+    public abstract StrategyDataDto execute(CrawlerVideoVO param);
+
+
+    /**
+     * 返回业务类型
+     */
+    public abstract String eventTypes();
+}

etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/StrategyHandlerService.java

@@ -0,0 +1,83 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.service.strategy;
+
+import com.tzld.crawler.etl.model.dto.StrategyDataDto;
+import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 15:06.
+ */
+@Service
+public class StrategyHandlerService implements ApplicationContextAware {
+    private static final Logger log = LoggerFactory.getLogger(StrategyHandlerService.class);
+    private final Map<String, StrategyAbstractHandler> HANDLER_MAP = new HashMap<>();
+
+    private ApplicationContext applicationContext;
+
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+        this.applicationContext = applicationContext;
+    }
+
+    @PostConstruct
+    public void init() {
+        // 获取 AbstractHandler 类的所有实现类
+        Map<String, StrategyAbstractHandler> type = applicationContext.getBeansOfType(StrategyAbstractHandler.class);
+        for (Map.Entry<String, StrategyAbstractHandler> entry : type.entrySet()) {
+            StrategyAbstractHandler value = entry.getValue();
+            log.info("load strategy {}", value.eventTypes());
+            HANDLER_MAP.put(value.eventTypes(), value);
+        }
+    }
+
+    /**
+     * 业务处理类的路由方法
+     */
+    public StrategyDataDto execute(List<String> strategies, CrawlerVideoVO param) {
+        StrategyDataDto result = null;
+        for (String strategy : strategies) {
+            StrategyAbstractHandler handler = HANDLER_MAP.get(strategy);
+            if (handler == null) {
+                return null;
+            }
+            result = handler.execute(param);
+            param = result;
+        }
+        return result;
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/handler/TitleScoreHandler.java

@@ -0,0 +1,145 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.service.strategy.handler;
+
+import com.alibaba.fastjson.JSONArray;
+import com.huaban.analysis.jieba.JiebaSegmenter;
+import com.huaban.analysis.jieba.SegToken;
+import com.tzld.crawler.etl.model.dto.StrategyDataDto;
+import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.service.SlsService;
+import com.tzld.crawler.etl.service.strategy.StrategyAbstractHandler;
+import com.tzld.crawler.etl.util.FeishuUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-19 17:41.
+ */
+@Component
+public class TitleScoreHandler extends StrategyAbstractHandler {
+    private static final Logger log = LoggerFactory.getLogger(TitleScoreHandler.class);
+    private final SlsService slsService;
+
+    @Value("${feishu.appid:}")
+    private String feishuAppid;
+    @Value("${feishu.appsecret:}")
+    private String feishuAppsecret;
+    @Value(("${title.stop.word.feishu.sheetId:}"))
+    private String stopWordSheetId;
+    @Value(("${title.stop.word.feishu.sheetToken:}"))
+    private String stopWordSheetToken;
+    @Value(("${title.stop.word.feishu.range:}"))
+    private String stopWordSheetRange;
+    @Value(("${title.score.word.feishu.sheetId:}"))
+    private String scoreWordSheetId;
+    @Value(("${title.score.word.feishu.sheetToken:}"))
+    private String scoreWordSheetToken;
+    @Value(("${title.score.word.feishu.range:}"))
+    private String scoreWordSheetRange;
+
+    private Map<String, Double> scoreWordMap;
+    private List<String> stopWorlds;
+
+    public TitleScoreHandler(SlsService slsService) {
+        this.slsService = slsService;
+    }
+
+    @Override
+    public StrategyDataDto execute(CrawlerVideoVO param) {
+        double score = 0;
+        JiebaSegmenter segmenter = new JiebaSegmenter();
+        List<SegToken> process = segmenter.process(param.getVideoTitle(), JiebaSegmenter.SegMode.SEARCH);
+        for (SegToken segToken : process) {
+            String word = segToken.word;
+            if (stopWorlds.contains(word)) {
+                continue;
+            }
+            score += scoreWordMap.getOrDefault(word, 0.0);
+        }
+
+        if (score <= 0.3) {
+            log.warn("title score is less than 0.3 {}", param);
+            slsService.log("mode", param.getStrategy(), "crawler", param.getPlatform(), "message", "权重分:" + score + "<=0.3");
+            return null;
+        }
+
+        StrategyDataDto result = new StrategyDataDto();
+        BeanUtils.copyProperties(param, result);
+
+        result.setTitleScore(score);
+        return result;
+    }
+
+    @Override
+    public String eventTypes() {
+        return "titleScore";
+    }
+
+    @PostConstruct
+    public void init() {
+        ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+        executor.scheduleWithFixedDelay(() -> {
+            try {
+                JSONArray values = FeishuUtils.getValues(feishuAppid, feishuAppsecret, stopWordSheetToken, stopWordSheetId, stopWordSheetRange);
+                stopWorlds = new ArrayList<>();
+                for (int i = 0; i < values.size(); i++) {
+                    JSONArray row = values.getJSONArray(i);
+                    String val = row.getString(0);
+                    if (val != null) {
+                        stopWorlds.add(val);
+                    }
+                }
+                log.info("load stop word list size {}", stopWorlds.size());
+
+                values = FeishuUtils.getValues(feishuAppid, feishuAppsecret, scoreWordSheetToken, scoreWordSheetId, scoreWordSheetRange);
+                scoreWordMap = new HashMap<>();
+                for (int i = 0; i < values.size(); i++) {
+                    JSONArray row = values.getJSONArray(i);
+                    String val = row.getString(0);
+                    if (val != null) {
+                        scoreWordMap.put(val, row.getDouble(8));
+                    }
+                }
+                log.info("load score word map size {}", scoreWordMap.size());
+            } catch (Exception e) {
+                log.error("load stop/score word error.", e);
+            }
+        }, 0, 10, TimeUnit.MINUTES);
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/service/strategy/handler/VideoFilterHandler.java

@@ -0,0 +1,52 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.service.strategy.handler;
+
+import com.tzld.crawler.etl.model.dto.StrategyDataDto;
+import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.service.strategy.StrategyAbstractHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 15:19.
+ */
+@Component
+public class VideoFilterHandler extends StrategyAbstractHandler {
+    private static final Logger log = LoggerFactory.getLogger(VideoFilterHandler.class);
+
+    @Override
+    public StrategyDataDto execute(CrawlerVideoVO param) {
+        log.info("video filter {}", param);
+        return null;
+    }
+
+    @Override
+    public String eventTypes() {
+        return "videoFilter";
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/util/FeishuUtils.java

@@ -0,0 +1,75 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.util;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-19 14:55.
+ */
+public class FeishuUtils {
+    private static String fetchFeishuToken(String feishuAppid, String feishuAppsecret) {
+        String tokenResp = HttpUtil.post("https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal",
+                JsonUtil.obj2String(ImmutableMap.of("app_id", feishuAppid, "app_secret", feishuAppsecret)));
+        return JSONObject.parseObject(tokenResp).getString("tenant_access_token");
+    }
+
+    public static String insertRows(String appId, String appSecret, String sheetToken, String sheetId, int startIndex, int endIndex) {
+        return HttpUtil.post("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" + sheetToken + "/insert_dimension_range",
+                ImmutableMap.of("Authorization", String.format("Bearer %s", fetchFeishuToken(appId, appSecret))),
+                JsonUtil.obj2String(ImmutableMap.of("inheritStyle", "AFTER", "dimension",
+                        ImmutableMap.of("sheetId", sheetId, "majorDimension", "ROWS",
+                                "startIndex", startIndex, "endIndex", endIndex))));
+    }
+
+    public static String updateValues(String appId, String appSecret, String sheetToken, String range, List<List<Object>> values) {
+        return HttpUtil.post("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" + sheetToken + "/values_batch_update",
+                ImmutableMap.of("Authorization", String.format("Bearer %s", fetchFeishuToken(appId, appSecret))),
+                JsonUtil.obj2String(ImmutableMap.of("valueRanges",
+                        Lists.newArrayList(ImmutableMap.of("range", range, "values", values)))));
+    }
+
+    public static JSONArray getValues(String appId, String appSecret, String sheetToken, String sheetId, String range) {
+        try {
+            String dataResp = HttpUtil.get(
+                    String.format("https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/%s/values/%s!%s?&dateTimeRenderOption=FormattedString",
+                            sheetToken, sheetId, range),
+                    ImmutableMap.of("Authorization", String.format("Bearer %s", fetchFeishuToken(appId, appSecret))));
+            return JSONObject.parseObject(dataResp)
+                    .getJSONObject("data")
+                    .getJSONObject("valueRange")
+                    .getJSONArray("values");
+        } catch (Exception e) {
+            throw new RuntimeException("读取飞书文档数据失败.", e);
+        }
+    }
+
+}

etl-core/src/main/java/com/tzld/crawler/etl/util/FileUtils.java

@@ -0,0 +1,75 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.util;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 15:54.
+ */
+public class FileUtils {
+    private static final Logger log = LoggerFactory.getLogger(FileUtils.class);
+
+    public static void main(String[] args) {
+        String fileUrl = "https://v2.kwaicdn.com/upic/2023/06/05/13/BMjAyMzA2MDUxMzA1MDNfNDU4Mjk4ODc5XzEwNDgwODM0NzMyNF8xXzM=_b_B4c86f13e02feb5462f484f95626229f6.mp4?pkey=AAVsuVfgud8EyKY8wn5cED5iBHD_2CCfZLuih27xJbjMwhqxIU-IHLkRhoDV0RxWUBjRvYtWWswKPL_n6u3csHgYD-euPHV1phmxa0r3ndOom3mRtowdHKCs5C9mr-PlXT8&tag=1-1686310919-unknown-0-5o0hj64qdg-0ee9b724883fdad8&clientCacheKey=3xz7hyd9c8h4zwa_b.mp4&di=ab7f961c&bp=14944&tt=b&ss=vp";
+
+        String filePath = "/Users/ehlxr/Workspaces/tzld/crawler-etl/video1.mp4";
+        download(fileUrl, filePath);
+    }
+
+    public static boolean download(String fileUrl, String filePath) {
+        try {
+            log.info("begin download file url: {} local path: {}", fileUrl, filePath);
+            URL url = new URL(fileUrl);
+            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+            conn.setConnectTimeout(5000);
+            conn.setReadTimeout(5000);
+            log.info("File url: {} local path: {} size: {}MB", fileUrl, filePath, conn.getContentLength() / 1024 / 1024);
+
+            InputStream inputStream = conn.getInputStream();
+            FileOutputStream outputStream = new FileOutputStream(filePath);
+            byte[] buffer = new byte[4096];
+            int len;
+            while ((len = inputStream.read(buffer)) != -1) {
+                outputStream.write(buffer, 0, len);
+            }
+            inputStream.close();
+            outputStream.close();
+            log.info("downloaded successfully file url: {} local path: {} ", fileUrl, filePath);
+            return true;
+        } catch (Exception e) {
+            log.error("downloaded error file url: {} local path: {} ", fileUrl, filePath, e);
+            return false;
+        }
+    }
+}

etl-core/src/main/java/com/tzld/crawler/etl/util/HttpUtil.java

@@ -0,0 +1,181 @@
+package com.tzld.crawler.etl.util;
+
+import com.google.common.collect.Maps;
+import okhttp3.*;
+
+import java.io.File;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author ehlxr
+ * @since 2020/4/20.
+ */
+public class HttpUtil {
+    private static final OkHttpClient OK_HTTP_CLIENT = new OkHttpClient.Builder()
+            .connectTimeout(1, TimeUnit.MINUTES)
+            .readTimeout(1, TimeUnit.MINUTES)
+            .build();
+
+    public static String get(String url, Map<String, String> headers) {
+        String resp;
+        try {
+            Request request = new Request.Builder()
+                    .url(url)
+                    .headers(Headers.of(headers))
+                    .build();
+            Response response = OK_HTTP_CLIENT.newCall(request).execute();
+            if (response.isSuccessful()) {
+                resp = response.body() != null ? response.body().string() : "";
+            } else {
+                throw new RuntimeException("send get http request failed. Unexpected code: " + response);
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("send get http request failed.", e);
+        }
+
+        return resp;
+    }
+
+    public static String get(String url) {
+        return get(url, Maps.newHashMap());
+    }
+
+    public static String post(String url, Map<String, String> headers, String body) {
+        String resp;
+        try {
+            MediaType mediaType = MediaType.parse(headers.getOrDefault(HttpContentType.KEY.value(), HttpContentType.JSON.value()));
+            Request request = new Request.Builder()
+                    .url(url)
+                    .headers(Headers.of(headers))
+                    .post(RequestBody.create(mediaType, body))
+                    .build();
+
+            Response response = OK_HTTP_CLIENT.newCall(request).execute();
+            if (response.isSuccessful()) {
+                resp = response.body() != null ? response.body().string() : "";
+            } else {
+                throw new RuntimeException("send post http request failed. Unexpected code: " + response);
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("send post http request failed.", e);
+        }
+
+        return resp;
+    }
+
+    public static String postFormBody(String url, Map<String, String> headers, String body) {
+        String resp;
+        try {
+            MediaType mediaType = MediaType.parse(headers.getOrDefault(HttpContentType.KEY.value(), HttpContentType.FORMBODY.value()));
+            Request request = new Request.Builder()
+                    .url(url)
+                    .headers(Headers.of(headers))
+                    .post(RequestBody.create(mediaType, body))
+                    .build();
+
+            Response response = OK_HTTP_CLIENT.newCall(request).execute();
+            if (response.isSuccessful()) {
+                resp = response.body() != null ? response.body().string() : "";
+            } else {
+                throw new RuntimeException("send post http request failed. Unexpected code: " + response);
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("send post http request failed.", e);
+        }
+
+        return resp;
+    }
+
+    public static String post(String url, String body) {
+        return post(url, Maps.newHashMap(), body);
+    }
+
+    public static String formPost(String url, Map<String, Map<FormType, Object>> params) {
+        return formPost(url, params, Maps.newHashMap());
+    }
+
+    public static String formPost(String url, Map<String, Map<FormType, Object>> params, Map<String, String> header) {
+        String resp;
+        try {
+            MultipartBody.Builder builder = new MultipartBody.Builder().setType(MultipartBody.FORM);
+            params.forEach((key, value) -> value.forEach((k, v) -> {
+                if (k.equals(FormType.FILE)) {
+                    File file = (File) v;
+                    builder.addFormDataPart(
+                            key,
+                            file.getName(),
+                            RequestBody.create(MediaType.parse("application/octet-stream"), file));
+                } else if (k.equals(FormType.STRING)) {
+                    builder.addFormDataPart(key, String.valueOf(v));
+                }
+            }));
+
+            Request request = new Request.Builder()
+                    .url(url)
+                    .post(builder.build())
+                    .headers(Headers.of(header))
+                    .build();
+            Response response = OK_HTTP_CLIENT.newCall(request).execute();
+            if (response.isSuccessful()) {
+                resp = response.body() != null ? response.body().string() : "";
+            } else {
+                throw new RuntimeException("send multipart post http request failed. Unexpected code: " + response);
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("send multipart post http request failed.", e);
+        }
+
+        return resp;
+    }
+
+    public static String appendParam(Map<String, Object> param) {
+        StringBuilder sb = new StringBuilder();
+        param.forEach((k, v) -> sb.append(k).append("=").append(v).append("&"));
+
+        return sb.substring(0, sb.length() - 1);
+    }
+
+    public enum HttpContentType {
+        /**
+         * key
+         */
+        KEY("Content-Type"),
+        /**
+         * JSON
+         */
+        JSON("application/json; charset=utf-8"),
+        /**
+         * form data
+         */
+        FORMBODY("application/x-www-form-urlencoded");
+
+        private String v;
+
+        HttpContentType(String v) {
+            this.v = v;
+        }
+
+        public String value() {
+            return v;
+        }
+
+        public void setV(String v) {
+            this.v = v;
+        }
+    }
+
+    /**
+     * FORM 表单提交类型
+     *
+     * @author ehlxr
+     * @since 2020/4/27.
+     */
+    public enum FormType {
+        /**
+         * form 表单提交类型
+         */
+        FILE, STRING
+    }
+}
+

etl-core/src/main/java/com/tzld/crawler/etl/util/JsonUtil.java

@@ -0,0 +1,224 @@
+package com.tzld.crawler.etl.util;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.*;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+import com.tzld.crawler.etl.common.base.Constant;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * JSON 处理类
+ *
+ * @author ehlxr
+ * @since 2020/5/6.
+ */
+@SuppressWarnings("unchecked")
+public class JsonUtil {
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    static {
+        // 对象的所有字段全部列入
+        OBJECT_MAPPER.setSerializationInclusion(JsonInclude.Include.ALWAYS);
+        // 取消默认转换 timestamps 形式
+        OBJECT_MAPPER.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
+        // 忽略空 bean 转 JSON 的错误
+        OBJECT_MAPPER.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+        // 所有的日期格式都统一为以下的样式：yyyy-MM-dd HH:mm:ss
+        OBJECT_MAPPER.setDateFormat(new SimpleDateFormat(Constant.STANDARD_FORMAT));
+        // 忽略在 JSON 字符串中存在，但是在 java 对象中不存在对应属性的情况
+        OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    }
+
+    public static ObjectMapper om() {
+        return OBJECT_MAPPER;
+    }
+
+    /**
+     * 对象转为 JsonNode 实例
+     *
+     * @param obj 要转换的对象
+     * @param <T> 要转换的对象类型
+     * @return {@link JsonNode}实例
+     */
+    public static <T> JsonNode obj2JsonNode(T obj) {
+        try {
+            return OBJECT_MAPPER.readTree(obj2String(obj));
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 对象转为 JSON 字符串
+     *
+     * @param obj 要转换的对象
+     * @param <T> 要转换的对象类型
+     * @return JSON 字符串
+     */
+    public static <T> String obj2String(T obj) {
+        if (obj == null) {
+            return "";
+        }
+        try {
+            return obj instanceof String ? (String) obj : OBJECT_MAPPER.writeValueAsString(obj);
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 对象转为格式化的 JSON 字符串
+     *
+     * @param obj 要转换的对象
+     * @param <T> 要转换的对象类型
+     * @return 格式化的 JSON 字符串
+     */
+    public static <T> String obj2StringPretty(T obj) {
+        if (obj == null) {
+            return "";
+        }
+        try {
+            return obj instanceof String ? (String) obj : OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(obj);
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 字符串转换为自定义对象
+     *
+     * @param str   要转换的字符串
+     * @param clazz 自定义对象的 class 对象
+     * @param <T>   自定义对象类型
+     * @return 自定义对象
+     */
+    public static <T> T string2Obj(String str, Class<T> clazz) {
+        if (Strings.isNullOrEmpty(str) || clazz == null) {
+            throw new RuntimeException("json string to obj param should not empty");
+        }
+        try {
+            return clazz.equals(String.class) ? (T) str : OBJECT_MAPPER.readValue(str, clazz);
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 字符串转换为自定义对象
+     *
+     * @param str           要转换的字符串
+     * @param typeReference 集合对象 typeReference
+     * @param <T>           集合对象类型
+     * @return 自定义对象
+     */
+    public static <T> T string2Obj(String str, TypeReference<T> typeReference) {
+        if (Strings.isNullOrEmpty(str) || typeReference == null) {
+            throw new RuntimeException("json string to obj param should not empty");
+        }
+        try {
+            return typeReference.getType().equals(String.class) ? (T) str : OBJECT_MAPPER.readValue(str, typeReference);
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 字符串转换为自定义对象
+     *
+     * @param str             要转换的字符串
+     * @param collectionClazz 集合 class
+     * @param elementClazzes  集合对象 class
+     * @param <T>             集合对象类型
+     * @return 自定义对象
+     */
+    public static <T> T string2Obj(String str, Class<?> collectionClazz, Class<?>... elementClazzes) {
+        JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametricType(collectionClazz, elementClazzes);
+        try {
+            return OBJECT_MAPPER.readValue(str, javaType);
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static void main(String[] args) {
+        try {
+            User user1 = new User();
+            user1.setId(1);
+            user1.setEmail("xrv@live.com");
+
+            String userJsonstr = JsonUtil.obj2String(user1);
+            System.out.println(userJsonstr);
+
+            String userJsonPretty = JsonUtil.obj2StringPretty(user1);
+            System.out.println(userJsonPretty);
+
+            User user2 = JsonUtil.string2Obj(userJsonstr, User.class);
+            user2.setId(2);
+            user2.setEmail("ehlxr.me@gmail.com");
+
+            List<User> userList = new ArrayList<>();
+            userList.add(user1);
+            userList.add(user2);
+            String userListJson = JsonUtil.obj2String(userList);
+            System.out.println(userListJson);
+
+            List<User> userListBean = JsonUtil.string2Obj(userListJson, new TypeReference<List<User>>() {
+            });
+            if (userListBean != null) {
+                userListBean.forEach(user -> System.out.println(user.getId() + " : " + user.getEmail()));
+            }
+            List<User> userListBean2 = JsonUtil.string2Obj(userListJson, List.class, User.class);
+            if (userListBean2 != null) {
+                userListBean2.forEach(user -> System.out.println(user.getId() + " : " + user.getEmail()));
+            }
+
+
+            Map<String, String> body = ImmutableMap.of("mobile", "13211111222", "realName", "realName");
+            String obj2String = JsonUtil.obj2String(body);
+            System.out.println(obj2String);
+
+            Map<String, String> stringStringMap = JsonUtil.string2Obj(obj2String, new TypeReference<Map<String, String>>() {
+            });
+
+            stringStringMap.forEach((k, v) -> System.out.println(k + " : " + v));
+
+            JsonNode jsonNode = JsonUtil.obj2JsonNode(userList);
+            System.out.println(jsonNode.path(0));
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+    }
+
+    /**
+     * 测试类
+     */
+    static class User {
+        private Integer id;
+        private String email;
+
+        public Integer getId() {
+            return id;
+        }
+
+        public void setId(Integer id) {
+            this.id = id;
+        }
+
+        public String getEmail() {
+            return email;
+        }
+
+        public void setEmail(String email) {
+            this.email = email;
+        }
+    }
+}
+

etl-core/src/main/java/com/tzld/crawler/etl/util/MD5Util.java

@@ -0,0 +1,59 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright © 2023 xrv <xrv@live.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.tzld.crawler.etl.util;
+
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * @author ehlxr
+ * @since 2023-06-09 20:15.
+ */
+public class MD5Util {
+    public static String md5(String str) {
+        MessageDigest md;
+        try {
+            md = MessageDigest.getInstance("MD5");
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+        md.update(str.getBytes());
+        byte[] byteDigest = md.digest();
+        int i;
+        StringBuilder sb = new StringBuilder();
+        for (byte b : byteDigest) {
+            i = b;
+            if (i < 0) {
+                i += 256;
+            }
+            if (i < 16) {
+                sb.append("0");
+            }
+            sb.append(Integer.toHexString(i));
+        }
+        return sb.toString();
+    }
+}

etl-core/src/main/resources/mapper/CrawlerVideoMapper.xml

@@ -0,0 +1,544 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.tzld.crawler.etl.dao.mapper.CrawlerVideoMapper">
+    <resultMap id="BaseResultMap" type="com.tzld.crawler.etl.model.po.CrawlerVideo">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        <id column="id" jdbcType="BIGINT" property="id"/>
+        <result column="video_id" jdbcType="BIGINT" property="videoId"/>
+        <result column="user_id" jdbcType="BIGINT" property="userId"/>
+        <result column="out_user_id" jdbcType="VARCHAR" property="outUserId"/>
+        <result column="platform" jdbcType="VARCHAR" property="platform"/>
+        <result column="strategy" jdbcType="VARCHAR" property="strategy"/>
+        <result column="out_video_id" jdbcType="VARCHAR" property="outVideoId"/>
+        <result column="video_title" jdbcType="VARCHAR" property="videoTitle"/>
+        <result column="cover_url" jdbcType="VARCHAR" property="coverUrl"/>
+        <result column="video_url" jdbcType="VARCHAR" property="videoUrl"/>
+        <result column="duration" jdbcType="BIGINT" property="duration"/>
+        <result column="publish_time" jdbcType="VARCHAR" property="publishTime"/>
+        <result column="play_cnt" jdbcType="INTEGER" property="playCnt"/>
+        <result column="like_cnt" jdbcType="INTEGER" property="likeCnt"/>
+        <result column="share_cnt" jdbcType="INTEGER" property="shareCnt"/>
+        <result column="collection_cnt" jdbcType="INTEGER" property="collectionCnt"/>
+        <result column="comment_cnt" jdbcType="INTEGER" property="commentCnt"/>
+        <result column="crawler_rule" jdbcType="CHAR" property="crawlerRule"/>
+        <result column="width" jdbcType="INTEGER" property="width"/>
+        <result column="height" jdbcType="INTEGER" property="height"/>
+        <result column="create_time" jdbcType="TIMESTAMP" property="createTime"/>
+        <result column="update_time" jdbcType="TIMESTAMP" property="updateTime"/>
+    </resultMap>
+    <sql id="Example_Where_Clause">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        <where>
+            <foreach collection="oredCriteria" item="criteria" separator="or">
+                <if test="criteria.valid">
+                    <trim prefix="(" prefixOverrides="and" suffix=")">
+                        <foreach collection="criteria.criteria" item="criterion">
+                            <choose>
+                                <when test="criterion.noValue">
+                                    and ${criterion.condition}
+                                </when>
+                                <when test="criterion.singleValue">
+                                    and ${criterion.condition} #{criterion.value}
+                                </when>
+                                <when test="criterion.betweenValue">
+                                    and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
+                                </when>
+                                <when test="criterion.listValue">
+                                    and ${criterion.condition}
+                                    <foreach close=")" collection="criterion.value" item="listItem" open="("
+                                             separator=",">
+                                        #{listItem}
+                                    </foreach>
+                                </when>
+                            </choose>
+                        </foreach>
+                    </trim>
+                </if>
+            </foreach>
+        </where>
+    </sql>
+    <sql id="Update_By_Example_Where_Clause">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        <where>
+            <foreach collection="example.oredCriteria" item="criteria" separator="or">
+                <if test="criteria.valid">
+                    <trim prefix="(" prefixOverrides="and" suffix=")">
+                        <foreach collection="criteria.criteria" item="criterion">
+                            <choose>
+                                <when test="criterion.noValue">
+                                    and ${criterion.condition}
+                                </when>
+                                <when test="criterion.singleValue">
+                                    and ${criterion.condition} #{criterion.value}
+                                </when>
+                                <when test="criterion.betweenValue">
+                                    and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
+                                </when>
+                                <when test="criterion.listValue">
+                                    and ${criterion.condition}
+                                    <foreach close=")" collection="criterion.value" item="listItem" open="("
+                                             separator=",">
+                                        #{listItem}
+                                    </foreach>
+                                </when>
+                            </choose>
+                        </foreach>
+                    </trim>
+                </if>
+            </foreach>
+        </where>
+    </sql>
+    <sql id="Base_Column_List">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        id, video_id, user_id, out_user_id, platform, strategy, out_video_id, video_title,
+        cover_url, video_url, duration, publish_time, play_cnt, like_cnt, share_cnt, collection_cnt,
+        comment_cnt, crawler_rule, width, height, create_time, update_time
+    </sql>
+    <select id="selectByExample" parameterType="com.tzld.crawler.etl.model.po.CrawlerVideoExample"
+            resultMap="BaseResultMap">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        select
+        <if test="distinct">
+            distinct
+        </if>
+        <include refid="Base_Column_List"/>
+        from crawler_video
+        <if test="_parameter != null">
+            <include refid="Example_Where_Clause"/>
+        </if>
+        <if test="orderByClause != null">
+            order by ${orderByClause}
+        </if>
+    </select>
+    <select id="selectByPrimaryKey" parameterType="java.lang.Long" resultMap="BaseResultMap">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        select
+        <include refid="Base_Column_List"/>
+        from crawler_video
+        where id = #{id,jdbcType=BIGINT}
+    </select>
+    <delete id="deleteByPrimaryKey" parameterType="java.lang.Long">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        delete from crawler_video
+        where id = #{id,jdbcType=BIGINT}
+    </delete>
+    <delete id="deleteByExample" parameterType="com.tzld.crawler.etl.model.po.CrawlerVideoExample">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        delete from crawler_video
+        <if test="_parameter != null">
+            <include refid="Example_Where_Clause"/>
+        </if>
+    </delete>
+    <insert id="insert" parameterType="com.tzld.crawler.etl.model.po.CrawlerVideo">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        insert into crawler_video (id, video_id, user_id,
+        out_user_id, platform, strategy,
+        out_video_id, video_title, cover_url,
+        video_url, duration, publish_time,
+        play_cnt, like_cnt, share_cnt,
+        collection_cnt, comment_cnt, crawler_rule,
+        width, height, create_time,
+        update_time)
+        values (#{id,jdbcType=BIGINT}, #{videoId,jdbcType=BIGINT}, #{userId,jdbcType=BIGINT},
+        #{outUserId,jdbcType=VARCHAR}, #{platform,jdbcType=VARCHAR}, #{strategy,jdbcType=VARCHAR},
+        #{outVideoId,jdbcType=VARCHAR}, #{videoTitle,jdbcType=VARCHAR}, #{coverUrl,jdbcType=VARCHAR},
+        #{videoUrl,jdbcType=VARCHAR}, #{duration,jdbcType=BIGINT}, #{publishTime,jdbcType=VARCHAR},
+        #{playCnt,jdbcType=INTEGER}, #{likeCnt,jdbcType=INTEGER}, #{shareCnt,jdbcType=INTEGER},
+        #{collectionCnt,jdbcType=INTEGER}, #{commentCnt,jdbcType=INTEGER}, #{crawlerRule,jdbcType=CHAR},
+        #{width,jdbcType=INTEGER}, #{height,jdbcType=INTEGER}, #{createTime,jdbcType=TIMESTAMP},
+        #{updateTime,jdbcType=TIMESTAMP})
+    </insert>
+    <insert id="insertSelective" parameterType="com.tzld.crawler.etl.model.po.CrawlerVideo">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        insert into crawler_video
+        <trim prefix="(" suffix=")" suffixOverrides=",">
+            <if test="id != null">
+                id,
+            </if>
+            <if test="videoId != null">
+                video_id,
+            </if>
+            <if test="userId != null">
+                user_id,
+            </if>
+            <if test="outUserId != null">
+                out_user_id,
+            </if>
+            <if test="platform != null">
+                platform,
+            </if>
+            <if test="strategy != null">
+                strategy,
+            </if>
+            <if test="outVideoId != null">
+                out_video_id,
+            </if>
+            <if test="videoTitle != null">
+                video_title,
+            </if>
+            <if test="coverUrl != null">
+                cover_url,
+            </if>
+            <if test="videoUrl != null">
+                video_url,
+            </if>
+            <if test="duration != null">
+                duration,
+            </if>
+            <if test="publishTime != null">
+                publish_time,
+            </if>
+            <if test="playCnt != null">
+                play_cnt,
+            </if>
+            <if test="likeCnt != null">
+                like_cnt,
+            </if>
+            <if test="shareCnt != null">
+                share_cnt,
+            </if>
+            <if test="collectionCnt != null">
+                collection_cnt,
+            </if>
+            <if test="commentCnt != null">
+                comment_cnt,
+            </if>
+            <if test="crawlerRule != null">
+                crawler_rule,
+            </if>
+            <if test="width != null">
+                width,
+            </if>
+            <if test="height != null">
+                height,
+            </if>
+            <if test="createTime != null">
+                create_time,
+            </if>
+            <if test="updateTime != null">
+                update_time,
+            </if>
+        </trim>
+        <trim prefix="values (" suffix=")" suffixOverrides=",">
+            <if test="id != null">
+                #{id,jdbcType=BIGINT},
+            </if>
+            <if test="videoId != null">
+                #{videoId,jdbcType=BIGINT},
+            </if>
+            <if test="userId != null">
+                #{userId,jdbcType=BIGINT},
+            </if>
+            <if test="outUserId != null">
+                #{outUserId,jdbcType=VARCHAR},
+            </if>
+            <if test="platform != null">
+                #{platform,jdbcType=VARCHAR},
+            </if>
+            <if test="strategy != null">
+                #{strategy,jdbcType=VARCHAR},
+            </if>
+            <if test="outVideoId != null">
+                #{outVideoId,jdbcType=VARCHAR},
+            </if>
+            <if test="videoTitle != null">
+                #{videoTitle,jdbcType=VARCHAR},
+            </if>
+            <if test="coverUrl != null">
+                #{coverUrl,jdbcType=VARCHAR},
+            </if>
+            <if test="videoUrl != null">
+                #{videoUrl,jdbcType=VARCHAR},
+            </if>
+            <if test="duration != null">
+                #{duration,jdbcType=BIGINT},
+            </if>
+            <if test="publishTime != null">
+                #{publishTime,jdbcType=VARCHAR},
+            </if>
+            <if test="playCnt != null">
+                #{playCnt,jdbcType=INTEGER},
+            </if>
+            <if test="likeCnt != null">
+                #{likeCnt,jdbcType=INTEGER},
+            </if>
+            <if test="shareCnt != null">
+                #{shareCnt,jdbcType=INTEGER},
+            </if>
+            <if test="collectionCnt != null">
+                #{collectionCnt,jdbcType=INTEGER},
+            </if>
+            <if test="commentCnt != null">
+                #{commentCnt,jdbcType=INTEGER},
+            </if>
+            <if test="crawlerRule != null">
+                #{crawlerRule,jdbcType=CHAR},
+            </if>
+            <if test="width != null">
+                #{width,jdbcType=INTEGER},
+            </if>
+            <if test="height != null">
+                #{height,jdbcType=INTEGER},
+            </if>
+            <if test="createTime != null">
+                #{createTime,jdbcType=TIMESTAMP},
+            </if>
+            <if test="updateTime != null">
+                #{updateTime,jdbcType=TIMESTAMP},
+            </if>
+        </trim>
+    </insert>
+    <select id="countByExample" parameterType="com.tzld.crawler.etl.model.po.CrawlerVideoExample"
+            resultType="java.lang.Long">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        select count(*) from crawler_video
+        <if test="_parameter != null">
+            <include refid="Example_Where_Clause"/>
+        </if>
+    </select>
+    <update id="updateByExampleSelective" parameterType="map">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        update crawler_video
+        <set>
+            <if test="record.id != null">
+                id = #{record.id,jdbcType=BIGINT},
+            </if>
+            <if test="record.videoId != null">
+                video_id = #{record.videoId,jdbcType=BIGINT},
+            </if>
+            <if test="record.userId != null">
+                user_id = #{record.userId,jdbcType=BIGINT},
+            </if>
+            <if test="record.outUserId != null">
+                out_user_id = #{record.outUserId,jdbcType=VARCHAR},
+            </if>
+            <if test="record.platform != null">
+                platform = #{record.platform,jdbcType=VARCHAR},
+            </if>
+            <if test="record.strategy != null">
+                strategy = #{record.strategy,jdbcType=VARCHAR},
+            </if>
+            <if test="record.outVideoId != null">
+                out_video_id = #{record.outVideoId,jdbcType=VARCHAR},
+            </if>
+            <if test="record.videoTitle != null">
+                video_title = #{record.videoTitle,jdbcType=VARCHAR},
+            </if>
+            <if test="record.coverUrl != null">
+                cover_url = #{record.coverUrl,jdbcType=VARCHAR},
+            </if>
+            <if test="record.videoUrl != null">
+                video_url = #{record.videoUrl,jdbcType=VARCHAR},
+            </if>
+            <if test="record.duration != null">
+                duration = #{record.duration,jdbcType=BIGINT},
+            </if>
+            <if test="record.publishTime != null">
+                publish_time = #{record.publishTime,jdbcType=VARCHAR},
+            </if>
+            <if test="record.playCnt != null">
+                play_cnt = #{record.playCnt,jdbcType=INTEGER},
+            </if>
+            <if test="record.likeCnt != null">
+                like_cnt = #{record.likeCnt,jdbcType=INTEGER},
+            </if>
+            <if test="record.shareCnt != null">
+                share_cnt = #{record.shareCnt,jdbcType=INTEGER},
+            </if>
+            <if test="record.collectionCnt != null">
+                collection_cnt = #{record.collectionCnt,jdbcType=INTEGER},
+            </if>
+            <if test="record.commentCnt != null">
+                comment_cnt = #{record.commentCnt,jdbcType=INTEGER},
+            </if>
+            <if test="record.crawlerRule != null">
+                crawler_rule = #{record.crawlerRule,jdbcType=CHAR},
+            </if>
+            <if test="record.width != null">
+                width = #{record.width,jdbcType=INTEGER},
+            </if>
+            <if test="record.height != null">
+                height = #{record.height,jdbcType=INTEGER},
+            </if>
+            <if test="record.createTime != null">
+                create_time = #{record.createTime,jdbcType=TIMESTAMP},
+            </if>
+            <if test="record.updateTime != null">
+                update_time = #{record.updateTime,jdbcType=TIMESTAMP},
+            </if>
+        </set>
+        <if test="_parameter != null">
+            <include refid="Update_By_Example_Where_Clause"/>
+        </if>
+    </update>
+    <update id="updateByExample" parameterType="map">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        update crawler_video
+        set id = #{record.id,jdbcType=BIGINT},
+        video_id = #{record.videoId,jdbcType=BIGINT},
+        user_id = #{record.userId,jdbcType=BIGINT},
+        out_user_id = #{record.outUserId,jdbcType=VARCHAR},
+        platform = #{record.platform,jdbcType=VARCHAR},
+        strategy = #{record.strategy,jdbcType=VARCHAR},
+        out_video_id = #{record.outVideoId,jdbcType=VARCHAR},
+        video_title = #{record.videoTitle,jdbcType=VARCHAR},
+        cover_url = #{record.coverUrl,jdbcType=VARCHAR},
+        video_url = #{record.videoUrl,jdbcType=VARCHAR},
+        duration = #{record.duration,jdbcType=BIGINT},
+        publish_time = #{record.publishTime,jdbcType=VARCHAR},
+        play_cnt = #{record.playCnt,jdbcType=INTEGER},
+        like_cnt = #{record.likeCnt,jdbcType=INTEGER},
+        share_cnt = #{record.shareCnt,jdbcType=INTEGER},
+        collection_cnt = #{record.collectionCnt,jdbcType=INTEGER},
+        comment_cnt = #{record.commentCnt,jdbcType=INTEGER},
+        crawler_rule = #{record.crawlerRule,jdbcType=CHAR},
+        width = #{record.width,jdbcType=INTEGER},
+        height = #{record.height,jdbcType=INTEGER},
+        create_time = #{record.createTime,jdbcType=TIMESTAMP},
+        update_time = #{record.updateTime,jdbcType=TIMESTAMP}
+        <if test="_parameter != null">
+            <include refid="Update_By_Example_Where_Clause"/>
+        </if>
+    </update>
+    <update id="updateByPrimaryKeySelective" parameterType="com.tzld.crawler.etl.model.po.CrawlerVideo">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        update crawler_video
+        <set>
+            <if test="videoId != null">
+                video_id = #{videoId,jdbcType=BIGINT},
+            </if>
+            <if test="userId != null">
+                user_id = #{userId,jdbcType=BIGINT},
+            </if>
+            <if test="outUserId != null">
+                out_user_id = #{outUserId,jdbcType=VARCHAR},
+            </if>
+            <if test="platform != null">
+                platform = #{platform,jdbcType=VARCHAR},
+            </if>
+            <if test="strategy != null">
+                strategy = #{strategy,jdbcType=VARCHAR},
+            </if>
+            <if test="outVideoId != null">
+                out_video_id = #{outVideoId,jdbcType=VARCHAR},
+            </if>
+            <if test="videoTitle != null">
+                video_title = #{videoTitle,jdbcType=VARCHAR},
+            </if>
+            <if test="coverUrl != null">
+                cover_url = #{coverUrl,jdbcType=VARCHAR},
+            </if>
+            <if test="videoUrl != null">
+                video_url = #{videoUrl,jdbcType=VARCHAR},
+            </if>
+            <if test="duration != null">
+                duration = #{duration,jdbcType=BIGINT},
+            </if>
+            <if test="publishTime != null">
+                publish_time = #{publishTime,jdbcType=VARCHAR},
+            </if>
+            <if test="playCnt != null">
+                play_cnt = #{playCnt,jdbcType=INTEGER},
+            </if>
+            <if test="likeCnt != null">
+                like_cnt = #{likeCnt,jdbcType=INTEGER},
+            </if>
+            <if test="shareCnt != null">
+                share_cnt = #{shareCnt,jdbcType=INTEGER},
+            </if>
+            <if test="collectionCnt != null">
+                collection_cnt = #{collectionCnt,jdbcType=INTEGER},
+            </if>
+            <if test="commentCnt != null">
+                comment_cnt = #{commentCnt,jdbcType=INTEGER},
+            </if>
+            <if test="crawlerRule != null">
+                crawler_rule = #{crawlerRule,jdbcType=CHAR},
+            </if>
+            <if test="width != null">
+                width = #{width,jdbcType=INTEGER},
+            </if>
+            <if test="height != null">
+                height = #{height,jdbcType=INTEGER},
+            </if>
+            <if test="createTime != null">
+                create_time = #{createTime,jdbcType=TIMESTAMP},
+            </if>
+            <if test="updateTime != null">
+                update_time = #{updateTime,jdbcType=TIMESTAMP},
+            </if>
+        </set>
+        where id = #{id,jdbcType=BIGINT}
+    </update>
+    <update id="updateByPrimaryKey" parameterType="com.tzld.crawler.etl.model.po.CrawlerVideo">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+        -->
+        update crawler_video
+        set video_id = #{videoId,jdbcType=BIGINT},
+        user_id = #{userId,jdbcType=BIGINT},
+        out_user_id = #{outUserId,jdbcType=VARCHAR},
+        platform = #{platform,jdbcType=VARCHAR},
+        strategy = #{strategy,jdbcType=VARCHAR},
+        out_video_id = #{outVideoId,jdbcType=VARCHAR},
+        video_title = #{videoTitle,jdbcType=VARCHAR},
+        cover_url = #{coverUrl,jdbcType=VARCHAR},
+        video_url = #{videoUrl,jdbcType=VARCHAR},
+        duration = #{duration,jdbcType=BIGINT},
+        publish_time = #{publishTime,jdbcType=VARCHAR},
+        play_cnt = #{playCnt,jdbcType=INTEGER},
+        like_cnt = #{likeCnt,jdbcType=INTEGER},
+        share_cnt = #{shareCnt,jdbcType=INTEGER},
+        collection_cnt = #{collectionCnt,jdbcType=INTEGER},
+        comment_cnt = #{commentCnt,jdbcType=INTEGER},
+        crawler_rule = #{crawlerRule,jdbcType=CHAR},
+        width = #{width,jdbcType=INTEGER},
+        height = #{height,jdbcType=INTEGER},
+        create_time = #{createTime,jdbcType=TIMESTAMP},
+        update_time = #{updateTime,jdbcType=TIMESTAMP}
+        where id = #{id,jdbcType=BIGINT}
+    </update>
+</mapper>

etl-core/src/main/resources/mybatis-generator-config.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE generatorConfiguration
+        PUBLIC "-//mybatis.org//DTD MyBatis Generator Configuration 1.0//EN"
+        "http://mybatis.org/dtd/mybatis-generator-config_1_0.dtd">
+<!-- 配置生成器 -->
+<generatorConfiguration>
+    <context id="mysql" defaultModelType="flat">
+        <property name="autoDelimitKeywords" value="true"/>
+        <!-- 生成的Java文件的编码 -->
+        <property name="javaFileEncoding" value="UTF-8"/>
+        <!-- 格式化java代码 -->
+        <property name="javaFormatter" value="org.mybatis.generator.api.dom.DefaultJavaFormatter"/>
+        <!-- 格式化XML代码 -->
+        <property name="xmlFormatter" value="org.mybatis.generator.api.dom.DefaultXmlFormatter"/>
+        <!-- beginningDelimiter和endingDelimiter：指明数据库的用于标记数据库对象名的符号，比如ORACLE就是双引号，MYSQL默认是`反引号； -->
+        <property name="beginningDelimiter" value="`"/>
+        <property name="endingDelimiter" value="`"/>
+
+        <plugin type="org.mybatis.generator.plugins.ToStringPlugin"/>
+        <plugin type="org.mybatis.generator.plugins.UnmergeableXmlMappersPlugin"/>
+
+        <commentGenerator>
+            <property name="addRemarkComments" value="true"/>
+            <property name="suppressDate" value="true"/>
+            <!--<property name="suppressAllComments" value="true"/>-->
+        </commentGenerator>
+
+        <jdbcConnection driverClass="com.mysql.jdbc.Driver"
+                        connectionURL="jdbc:mysql://rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&amp;characterEncoding=utf-8&amp;zeroDateTimeBehavior=convertToNull&amp;useSSL=false"
+                        userId="crawler" password="crawler123456@">
+        </jdbcConnection>
+
+        <javaTypeResolver type="org.mybatis.generator.internal.types.JavaTypeResolverDefaultImpl">
+            <property name="forceBigDecimals" value="false"/>
+        </javaTypeResolver>
+
+        <javaModelGenerator targetPackage="com.tzld.crawler.etl.model.po" targetProject="src/main/java">
+            <property name="constructorBased" value="false"/>
+            <property name="enableSubPackages" value="true"/>
+            <property name="immutable" value="false"/>
+        </javaModelGenerator>
+
+        <sqlMapGenerator targetPackage="mapper" targetProject="src/main/resources">
+            <property name="enableSubPackages" value="true"/>
+        </sqlMapGenerator>
+
+        <javaClientGenerator targetPackage="com.tzld.crawler.etl.dao.mapper" type="XMLMAPPER"
+                             targetProject="src/main/java">
+            <property name="enableSubPackages" value="true"/>
+        </javaClientGenerator>
+
+        <table tableName="crawler_video" domainObjectName="CrawlerVideo" alias=""/>
+    </context>
+
+</generatorConfiguration>

etl-server/pom.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.tzld.piaoquan</groupId>
+        <artifactId>crawler-etl</artifactId>
+        <version>1.0.0</version>
+    </parent>
+    <artifactId>etl-server</artifactId>
+    <name>crawler-etl-server</name>
+    <description>crawler-etl-server for Spring Boot</description>
+
+    <properties>
+        <java.version>1.8</java.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.tzld.piaoquan</groupId>
+            <artifactId>etl-core</artifactId>
+            <version>1.0.0</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <!-- 固定包名 避免随着版本变动 -->
+        <finalName>crawler-etl-server</finalName>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <configuration>
+                    <mainClass>com.tzld.crawler.etl.EtlServerApplication</mainClass>
+                    <layout>ZIP</layout>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+
+    </build>
+
+</project>

etl-server/src/main/java/com/tzld/crawler/etl/EtlServerApplication.java

@@ -0,0 +1,21 @@
+package com.tzld.crawler.etl;
+
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cloud.openfeign.EnableFeignClients;
+
+/**
+ * 启动类
+ *
+ * @author supeng
+ */
+@EnableFeignClients
+@MapperScan("com.tzld.crawler.etl.dao.mapper")
+// @ServletComponentScan
+@SpringBootApplication
+public class EtlServerApplication {
+    public static void main(String[] args) {
+        SpringApplication.run(EtlServerApplication.class, args);
+    }
+}

etl-server/src/main/java/com/tzld/crawler/etl/controller/IndexController.java

@@ -0,0 +1,42 @@
+package com.tzld.crawler.etl.controller;
+
+import com.alibaba.fastjson.JSONObject;
+import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
+import com.tzld.crawler.etl.service.EtlService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+
+/**
+ * @author ehlxr
+ */
+@RestController
+@RequestMapping("/")
+public class IndexController {
+    private static final Logger LOGGER = LoggerFactory.getLogger(IndexController.class);
+    @Autowired
+    private EtlService etlService;
+
+    /**
+     * 探活
+     *
+     * @return
+     */
+    @GetMapping("/healthcheck")
+    public String healthcheck() {
+        LOGGER.info("I'm ok");
+        return "ok";
+    }
+
+    public static void main(String[] args) {
+        String s = "{\"user_id\": 6281907,\"out_user_id\": \"53322270\",\"platform\": \"xiaoniangao\",\"strategy\": \"author\",\"out_video_id\": \"5067863383\",\"video_title\": \"早上起床前念四句话，身体有可能变得更健康\uD83D\uDC8E\",\"cover_url\": \"https://cdn-xphoto2.xiaoniangao.cn/5067863386@690w_385h_0e_1pr%7C690x385-5rc_0r.jpg?OSSAccessKeyId=LTAI4G2W1FsgwzAWYpPoB3v6&Expires=1688140805&Signature=F%2BIn%2FdAfjM4UWnG%2F96qAUcHWhpI%3D\",\"video_url\": \"http://cdn-xalbum-baishan.xiaoniangao.cn/5067863383?Expires=1704038400&OSSAccessKeyId=LTAI5tB7cRkYiqHcTdkVprwb&Signature=kD2sBhazSa4L5q/XolJ8BkxW6SI%3D\",\"duration\": 40,\"publish_time\": \"2023-06-08 23:01:47\",\"play_cnt\": 602,\"like_cnt\": 0,\"share_cnt\": 0,\"collection_cnt\": 0,\"comment_cnt\": 0,\"crawler_rule\": {\"period\": {\"max\": 3,\"min\": 3},\"duration\": {\"max\": 999999999999999,\"min\": 40},\"play_cnt\": {\"max\": 999999999999999,\"min\": 500}},\"width\": 450,\"height\": 254}";
+        System.out.println(JSONObject.parseObject(s, CrawlerVideoVO.class));
+    }
+
+    @PostMapping("/etl")
+    public String deal(@RequestBody CrawlerVideoVO video) {
+        etlService.deal(video);
+        return "ok";
+    }
+}

etl-server/src/main/resources/application-dev.yml

@@ -0,0 +1,44 @@
+server:
+  port: 8080
+
+spring:
+  datasource:
+    url: jdbc:mysql://rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&useSSL=false
+    username: crawler
+    password: crawler123456@
+
+apollo:
+  meta: http://devapolloconfig-internal.piaoquantv.com
+
+aliyun:
+  log:
+    project: crawler-log-dev
+    logstore:
+      crawler: crawler-log-dev
+
+longvideo:
+  feign:
+    url: videotest-internal.yishihui.com
+
+logging:
+  level:
+    com:
+      tzld:
+        crawler:
+          etl: debug
+
+feign:
+  client:
+    config:
+      default:
+        # NONE 没有日志（默认）
+        # BASIC 只记录请求方法和 URL 以及响应状态码和执行时间
+        # HEADERS 记录基本信息以及请求和响应头
+        # FULL 记录请求和响应的头、正文和元数据
+        logger-level: FULL
+
+rocketmq:
+  crawler:
+    etl:
+      topic: topic_crawler_etl_dev
+      groupid: GID_CRAWLER_ETL_DEV

etl-server/src/main/resources/application-prod.yml

@@ -0,0 +1,27 @@
+server:
+  port: 8080
+
+spring:
+  datasource:
+    url: jdbc:mysql://rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&useSSL=false
+    username: crawler
+    password: crawler123456@
+
+apollo:
+  meta: http://apolloconfig-internal.piaoquantv.com
+
+aliyun:
+  log:
+    project: crawler-log-prod
+    logstore:
+      crawlaliyun.log.logstore.crawlerer: crawler-log-prod
+
+longvideo:
+  feign:
+    url: longvideoapi-internal.piaoquantv.com
+
+rocketmq:
+  crawler:
+    etl:
+      topic: topic_crawler_etl_prod
+      groupid: GID_CRAWLER_ETL_PROD

etl-server/src/main/resources/application.yml

@@ -0,0 +1,77 @@
+spring:
+  profiles:
+    active: dev
+  application:
+    name: crawler-etl
+
+  jackson:
+    default-property-inclusion: non_null
+
+  datasource:
+    driver-class-name: com.mysql.jdbc.Driver
+    type: com.zaxxer.hikari.HikariDataSource
+    hikari:
+      minimum-idle: 10
+      maximum-pool-size: 60
+      connection-test-query: SELECT 1
+
+server:
+  tomcat:
+    threads:
+      max: 1000
+    uri-encoding: UTF-8
+    accept-count: 1000
+    connection-timeout: 30000
+  servlet:
+    context-path: /crawler-etl
+    session:
+      timeout: 60
+
+pagehelper:
+  helper-dialect: mysql
+
+mybatis:
+  type-aliases-package: com.tzld.crawler.etl.model.po
+  mapper-locations: classpath:mapper/**/*.xml
+  configuration:
+    log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl
+
+logging:
+  file:
+    path: /datalog/weblog/${spring.application.name}/
+
+app:
+  id: ${spring.application.name}
+
+apollo:
+  bootstrap:
+    enabled: true
+    namespaces: application
+  cacheDir: /datalog/apollo-cache-dir
+
+feign:
+  client:
+    config:
+      default:
+        connectTimeout: 2000
+        readTimeout: 10000
+
+aliyun:
+  oss:
+    endpoint: oss-cn-hangzhou.aliyuncs.com
+    accessKeyId: LTAIP6x1l3DXfSxm
+    accessKeySecret: KbTaM9ars4OX3PMS6Xm7rtxGr1FLon
+
+  log:
+    endpoint: cn-hangzhou.log.aliyuncs.com
+    accessKeyId: LTAIP6x1l3DXfSxm
+    accessKeySecret: KbTaM9ars4OX3PMS6Xm7rtxGr1FLon
+    logstore:
+      info: info-log
+      error: error-log
+
+rocketmq:
+  httpEndpoint: http://1894469520484605.mqrest.cn-qingdao-public.aliyuncs.com
+  instanceId: MQ_INST_1894469520484605_BXhXuzkZ
+  accessKey: LTAI4G7puhXtLyHzHQpD6H7A
+  secretKey: nEbq3xWNQd1qLpdy2u71qFweHkZjSG

etl-server/src/main/resources/logback-spring.xml

@@ -0,0 +1,258 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 日志级别从低到高分为TRACE < DEBUG < INFO < WARN < ERROR < FATAL，如果设置为WARN，则低于WARN的信息都不会输出 -->
+<!-- scan:当此属性设置为true时，配置文件如果发生改变，将会被重新加载，默认值为true -->
+<!-- scanPeriod:设置监测配置文件是否有修改的时间间隔，如果没有给出时间单位，默认单位是毫秒。当scan为true时，此属性生效。默认的时间间隔为1分钟。 -->
+<!-- debug:当此属性设置为true时，将打印出logback内部日志信息，实时查看logback运行状态。默认值为false。 -->
+<configuration scan="true" scanPeriod="10 seconds">
+    <!--<include resource="org/springframework/boot/logging/logback/base.xml"/>-->
+
+    <contextName>logback</contextName>
+    <!-- name的值是变量的名称，value的值时变量定义的值。通过定义的值会被插入到logger上下文中。定义变量后，可以使“${}”来使用变量。 -->
+    <!-- <property name="LOG_PATH"  value="${logging.file.path}" />-->
+
+    <springProperty name="LOG_PATH" source="logging.file.path"/>
+    <springProperty name="ALIYUN_LOG_ENDPOINT" source="aliyun.log.endpoint"/>
+    <springProperty name="ALIYUN_LOG_ACCESSKEYID" source="aliyun.log.accessKeyId"/>
+    <springProperty name="ALIYUN_LOG_ACCESSKEYSECRET" source="aliyun.log.accessKeySecret"/>
+    <springProperty name="ALIYUN_LOG_PROJECT" source="aliyun.log.project"/>
+    <springProperty name="ALIYUN_LOG_LOGSTORE_INFO" source="aliyun.log.logstore.info"/>
+    <springProperty name="ALIYUN_LOG_LOGSTORE_ERROR" source="aliyun.log.logstore.error"/>
+
+    <!-- 彩色日志 -->
+    <!-- 彩色日志依赖的渲染类 -->
+    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
+    <conversionRule conversionWord="wex"
+                    converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
+    <conversionRule conversionWord="wEx"
+                    converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>
+    <!-- 彩色日志格式 -->
+    <property name="CONSOLE_LOG_PATTERN"
+              value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr([%X{logTraceId}]){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
+    <!--<property name="CONSOLE_LOG_PATTERN"-->
+    <!--          value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr([%X{√logTraceId}]){magenta} %clr(-&#45;&#45;){faint} %clr([%15.15t]){faint} %clr(at %class.%method){cyan} \\(%file:%line\\) %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>-->
+
+    <!--输出到控制台-->
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <!--此日志appender是为开发使用，只配置最底级别，控制台输出的日志级别是大于或等于此级别的日志信息-->
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>debug</level>
+        </filter>
+        <encoder>
+            <Pattern>${CONSOLE_LOG_PATTERN}</Pattern>
+            <!-- 设置字符集 -->
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <!--输出到文件-->
+    <!-- 时间滚动输出 level为 DEBUG 日志 -->
+    <appender name="DEBUG_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${LOG_PATH}/debug.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{logTraceId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!-- 日志归档 -->
+            <fileNamePattern>${LOG_PATH}/debug/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>15</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录debug级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>debug</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+    <!-- 时间滚动输出 level为 INFO 日志 -->
+    <appender name="INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${LOG_PATH}/info.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{logTraceId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!-- 每天日志归档路径以及格式 -->
+            <fileNamePattern>${LOG_PATH}/info/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>15</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录info级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>info</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+    <!-- 时间滚动输出 level为 WARN 日志 -->
+    <appender name="WARN_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${LOG_PATH}/warn.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{logTraceId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${LOG_PATH}/warn/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>15</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录warn级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>warn</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+    <!-- 时间滚动输出 level为 ERROR 日志 -->
+    <appender name="ERROR_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${LOG_PATH}/error.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{logTraceId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${LOG_PATH}/error/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>15</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录ERROR级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>ERROR</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+
+    <appender name="ALIYUN_LOG_INFO" class="com.aliyun.openservices.log.logback.LoghubAppender">
+        <endpoint>${ALIYUN_LOG_ENDPOINT}</endpoint>
+        <accessKeyId>${ALIYUN_LOG_ACCESSKEYID}</accessKeyId>
+        <accessKeySecret>${ALIYUN_LOG_ACCESSKEYSECRET}</accessKeySecret>
+        <project>${ALIYUN_LOG_PROJECT}</project>
+        <logStore>${ALIYUN_LOG_LOGSTORE_INFO}</logStore>
+
+        <totalSizeInBytes>104857600</totalSizeInBytes>
+        <maxBlockMs>0</maxBlockMs>
+        <ioThreadCount>8</ioThreadCount>
+        <batchSizeThresholdInBytes>524288</batchSizeThresholdInBytes>
+        <batchCountThreshold>4096</batchCountThreshold>
+        <lingerMs>2000</lingerMs>
+        <retries>10</retries>
+        <baseRetryBackoffMs>100</baseRetryBackoffMs>
+        <maxRetryBackoffMs>50000</maxRetryBackoffMs>
+
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{logTraceId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+
+        <timeFormat>yyyy-MM-dd'T'HH:mmZ</timeFormat>
+        <timeZone>Asia/Shanghai</timeZone>
+
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>INFO</level>
+        </filter>
+
+        <mdcFields>logTraceId</mdcFields>
+    </appender>
+
+    <appender name="ALIYUN_LOG_ERROR" class="com.aliyun.openservices.log.logback.LoghubAppender">
+        <endpoint>${ALIYUN_LOG_ENDPOINT}</endpoint>
+        <accessKeyId>${ALIYUN_LOG_ACCESSKEYID}</accessKeyId>
+        <accessKeySecret>${ALIYUN_LOG_ACCESSKEYSECRET}</accessKeySecret>
+        <project>${ALIYUN_LOG_PROJECT}</project>
+        <logStore>${ALIYUN_LOG_LOGSTORE_ERROR}</logStore>
+
+        <totalSizeInBytes>104857600</totalSizeInBytes>
+        <maxBlockMs>0</maxBlockMs>
+        <ioThreadCount>8</ioThreadCount>
+        <batchSizeThresholdInBytes>524288</batchSizeThresholdInBytes>
+        <batchCountThreshold>4096</batchCountThreshold>
+        <lingerMs>2000</lingerMs>
+        <retries>10</retries>
+        <baseRetryBackoffMs>100</baseRetryBackoffMs>
+        <maxRetryBackoffMs>50000</maxRetryBackoffMs>
+
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{logTraceId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+
+        <timeFormat>yyyy-MM-dd'T'HH:mmZ</timeFormat>
+        <timeZone>Asia/Shanghai</timeZone>
+
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>ERROR</level>
+        </filter>
+
+        <mdcFields>logTraceId</mdcFields>
+    </appender>
+
+    <!--
+        <logger>用来设置某一个包或者具体的某一个类的日志打印级别、以及指定<appender>。
+        <logger>仅有一个name属性，一个可选的level和一个可选的addtivity属性。
+        name:用来指定受此logger约束的某一个包或者具体的某一个类。
+        level:用来设置打印级别，大小写无关：TRACE, DEBUG, INFO, WARN, ERROR, ALL 和 OFF，
+              还有一个特俗值INHERITED或者同义词NULL，代表强制执行上级的级别。
+              如果未设置此属性，那么当前logger将会继承上级的级别。
+        addtivity:是否向上级logger传递打印信息。默认是true。
+    -->
+    <!--<logger name="org.springframework.web" level="info"/>-->
+    <!--<logger name="org.springframework.scheduling.annotation.ScheduledAnnotationBeanPostProcessor" level="INFO"/>-->
+
+    <!--
+        使用mybatis的时候，sql语句是debug下才会打印，而这里我们只配置了info，所以想要查看sql语句的话，有以下两种操作：
+        第一种把<root level="info">改成<root level="DEBUG">这样就会打印sql，不过这样日志那边会出现很多其他消息
+        第二种就是单独给dao下目录配置 <logger> 指定 level 属性为 debug，这样配置sql语句会打印，其他还是正常info级别：
+     -->
+
+    <!-- 可用来获取 StatusManager 中的状态 -->
+    <!--<statusListener class="ch.qos.logback.core.status.OnConsoleStatusListener"/>-->
+    <!-- 解决 aliyun loghub debug模式下循环发送的问题 -->
+    <logger name="org.apache.http.impl.conn.Wire" level="WARN"/>
+    <!--aliyun loghub 为了防止进程退出时，内存中的数据丢失，请加上此选项-->
+    <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+
+    <!--
+        root节点是必选节点，用来指定最基础的日志输出级别，只有一个level属性
+        level:用来设置打印级别，大小写无关：TRACE, DEBUG, INFO, WARN, ERROR, ALL 和 OFF，
+        不能设置为INHERITED或者同义词NULL。默认是DEBUG
+        可以包含零个或多个元素，标识这个appender将会添加到这个logger。
+    -->
+    <root level="info">
+        <appender-ref ref="CONSOLE"/>
+        <appender-ref ref="DEBUG_FILE"/>
+        <appender-ref ref="INFO_FILE"/>
+        <appender-ref ref="WARN_FILE"/>
+        <appender-ref ref="ERROR_FILE"/>
+        <appender-ref ref="ALIYUN_LOG_INFO"/>
+        <appender-ref ref="ALIYUN_LOG_ERROR"/>
+    </root>
+</configuration>

etl-server/src/test/java/com/tzld/crawler/etl/EtlServerApplicationTests.java

@@ -0,0 +1,70 @@
+package com.tzld.crawler.etl;
+
+import com.aliyun.mq.http.MQClient;
+import com.aliyun.mq.http.MQProducer;
+import com.aliyun.mq.http.model.TopicMessage;
+import com.google.common.collect.Lists;
+import com.huaban.analysis.jieba.JiebaSegmenter;
+import com.huaban.analysis.jieba.SegToken;
+import com.tzld.crawler.etl.mq.EtlMQConsumer;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.IntStream;
+
+@SpringBootTest
+class EtlServerApplicationTests {
+    @Value("${rocketmq.accessKey}")
+    private String accessKey;
+    @Value("${rocketmq.secretKey}")
+    private String secretKey;
+    @Value("${rocketmq.httpEndpoint}")
+    private String httpEndpoint;
+    @Value("${rocketmq.instanceId}")
+    private String instanceId;
+
+    /**
+     * 排除 EtlMQConsumer 的加载
+     */
+    @MockBean
+    private EtlMQConsumer etlMQConsumer;
+
+
+    @Test
+    void produceMsgTest() {
+        MQClient mqClient = new MQClient(httpEndpoint, accessKey, secretKey);
+        MQProducer producer = mqClient.getProducer(instanceId, "topic_crawler_etl_dev");
+        ArrayList<String> platforms = Lists.newArrayList("benshanzhufu", "kuaishou", "douyin", "xigua", "gongzhonghao", "xiaoniangao");
+
+        IntStream.range(0, 20).forEach(x -> {
+            try {
+                String s = "{\"user_id\": 6281907,\"out_user_id\": \"53322270\",\"platform\": \"" + platforms.get(x % 6) + "\",\"strategy\": \"author\"," +
+                        "\"out_video_id\": \"" + System.currentTimeMillis() + "\"," +
+                        "\"video_title\": \"" + x + "人最佳喧，睡眠时间⚡\",\"cover_url\": \"https://cdn-xphoto2.xiaoniangao.cn/5071472172@690w_385h_0e_1pr%7C690x385-5rc_0r.jpg?OSSAccessKeyId=LTAI4G2W1FsgwzAWYpPoB3v6&Expires=1688140805&Signature=BC%2F9U%2BTjPGVwaqTpStSO1kdYhQU%3D\"," +
+                        "\"video_url\": \"https://cdn-xalbum2.xiaoniangao.cn/64858d720000014567e794c9?Expires=1704038400&OSSAccessKeyId=LTAI5tB7cRkYiqHcTdkVprwb&Signature=j0WjPwp6wq1XnstwpXL6z7f9wRM%3D\"" +
+                        ",\"duration\": 40,\"publish_time\": \"2023-06-08 23:01:47\",\"play_cnt\": 602,\"like_cnt\": 0,\"share_cnt\": 0,\"collection_cnt\": 0,\"comment_cnt\": 0,\"crawler_rule\": {\"period\": { \"max\": 3, \"min\": 3 },\"duration\": { \"max\": 999999999999999, \"min\": 40 },\"play_cnt\": { \"max\": 999999999999999, \"min\": 500 }},\"width\": 450,\"height\": 254}";
+
+                TopicMessage pubMsg = new TopicMessage(s.getBytes());
+                producer.publishMessage(pubMsg);
+            } catch (Exception e) {
+                System.out.println(e);
+            }
+        });
+
+    }
+
+
+    @Test
+    public void testDemo() {
+        JiebaSegmenter segmenter = new JiebaSegmenter();
+        List<SegToken> process = segmenter.process("这是一个伸手不见五指的黑夜。我叫孙悟空，我爱北京，我爱Java和Rust还有Golang。", JiebaSegmenter.SegMode.SEARCH);
+        for (SegToken segToken : process) {
+            System.out.println(segToken.word);
+        }
+    }
+
+}

pom.xml

@@ -0,0 +1,153 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <packaging>pom</packaging>
+    <parent>
+        <groupId>com.tzld.commons</groupId>
+        <artifactId>supom</artifactId>
+        <version>1.0.9</version>
+    </parent>
+    <groupId>com.tzld.piaoquan</groupId>
+    <artifactId>crawler-etl</artifactId>
+    <version>1.0.0</version>
+    <name>crawler-etl</name>
+    <description>crawler-etl</description>
+
+    <modules>
+        <module>etl-core</module>
+        <module>etl-server</module>
+    </modules>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-commons</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-openfeign</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.junit.vintage</groupId>
+                    <artifactId>junit-vintage-engine</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.mybatis.spring.boot</groupId>
+            <artifactId>mybatis-spring-boot-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.mybatis.generator</groupId>
+            <artifactId>mybatis-generator-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.github.pagehelper</groupId>
+            <artifactId>pagehelper-spring-boot-starter</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-boot-starter</artifactId>
+            <version>3.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger2</artifactId>
+            <version>3.0.0</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger-ui</artifactId>
+            <version>3.0.0</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>fastjson</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.ctrip.framework.apollo</groupId>
+            <artifactId>apollo-client</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.tzld.commons</groupId>
+            <artifactId>aliyun-log-spring-boot-starter</artifactId>
+            <version>2.0.0</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>aliyun-log</artifactId>
+                    <groupId>com.aliyun.openservices</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>2.5.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.aliyun.openservices</groupId>
+            <artifactId>aliyun-log-logback-appender</artifactId>
+            <version>0.1.18</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-aop</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.github.ben-manes.caffeine</groupId>
+            <artifactId>caffeine</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.aliyun.mq</groupId>
+            <artifactId>mq-http-sdk</artifactId>
+            <version>1.0.3.2</version>
+            <classifier>jar-with-dependencies</classifier>
+        </dependency>
+        <dependency>
+            <groupId>commons-collections</groupId>
+            <artifactId>commons-collections</artifactId>
+            <version>3.2.2</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.tzld.commons</groupId>
+            <artifactId>aliyun-oss-spring-boot-starter</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>okhttp</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.huaban</groupId>
+            <artifactId>jieba-analysis</artifactId>
+            <version>1.0.2</version>
+        </dependency>
+
+    </dependencies>
+
+</project>