ArticleSensitive 迁移

long-article-recommend-service/pom.xml

@@ -112,6 +112,11 @@
             <artifactId>tencentcloud-sdk-java-tms</artifactId>
             <version>3.1.1010</version>
         </dependency>
+        <dependency>
+            <groupId>org.mybatis.spring.boot</groupId>
+            <artifactId>mybatis-spring-boot-starter</artifactId>
+        </dependency>
+
     </dependencies>
 
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/Application.java

@@ -1,6 +1,7 @@
 package com.tzld.longarticle.recommend.server;
 
 
+import org.mybatis.spring.boot.autoconfigure.MybatisAutoConfiguration;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
@@ -12,7 +13,8 @@ import org.springframework.context.annotation.EnableAspectJAutoProxy;
  * https://github.com/grpc-swagger/grpc-swagger/blob/master/README_CN.md
  */
 @SpringBootApplication(exclude = {
-        RedisReactiveAutoConfiguration.class
+        RedisReactiveAutoConfiguration.class,
+        MybatisAutoConfiguration.class
 })
 @ComponentScan({
         "com.tzld.longarticle.recommend.server.service",

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/config/PiaoquanCrawlerMysqlConfiguration.java

@@ -1,40 +0,0 @@
-package com.tzld.longarticle.recommend.server.config;
-
-import com.zaxxer.hikari.HikariDataSource;
-import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
-
-
-@Configuration(proxyBeanMethods = false)
-public class PiaoquanCrawlerMysqlConfiguration {
-
-    @Bean(name = "piaoquanCrawlerJdbcTemplate")
-    public NamedParameterJdbcTemplate piaoquanCrawlerJdbcTemplate() {
-        DataSourceProperties dataSourceProperties = new DataSourceProperties();
-        dataSourceProperties.setUsername("crawler");
-        dataSourceProperties.setPassword("crawler123456@");
-        dataSourceProperties.setUrl("jdbc:mysql://rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull");
-        dataSourceProperties.setDriverClassName("com.mysql.jdbc.Driver");
-        HikariDataSource dataSource = dataSourceProperties.initializeDataSourceBuilder().type(HikariDataSource.class).build();
-        dataSource.setMinimumIdle(5);
-        dataSource.setMaximumPoolSize(10);
-        dataSource.setConnectionTestQuery("select 1");
-        return new NamedParameterJdbcTemplate(dataSource);
-    }
-
-    @Bean(name = "programDataJdbcTemplate")
-    public NamedParameterJdbcTemplate programDataJdbcTemplate() {
-        DataSourceProperties dataSourceProperties = new DataSourceProperties();
-        dataSourceProperties.setUsername("wx2023_ad");
-        dataSourceProperties.setPassword("wx2023_adP@assword1234");
-        dataSourceProperties.setUrl("jdbc:mysql://rm-bp12k5fuh5zyx31d28o.mysql.rds.aliyuncs.com:3306/adplatform?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull");
-        dataSourceProperties.setDriverClassName("com.mysql.jdbc.Driver");
-        HikariDataSource dataSource = dataSourceProperties.initializeDataSourceBuilder().type(HikariDataSource.class).build();
-        dataSource.setMinimumIdle(5);
-        dataSource.setMaximumPoolSize(10);
-        dataSource.setConnectionTestQuery("select 1");
-        return new NamedParameterJdbcTemplate(dataSource);
-    }
-}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/config/db/AdplatformDBConfig.java

@@ -0,0 +1,65 @@
+package com.tzld.longarticle.recommend.server.config.db;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.jdbc.DataSourceBuilder;
+import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+import org.springframework.orm.jpa.JpaTransactionManager;
+import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
+
+import javax.persistence.EntityManagerFactory;
+import javax.sql.DataSource;
+import java.util.HashMap;
+import java.util.Map;
+
+@Configuration
+@EnableTransactionManagement
+@EnableJpaRepositories(
+        basePackages = "com.tzld.longarticle.recommend.server.repository.adplatform",
+        entityManagerFactoryRef = "adplatformEntityManagerFactory",
+        transactionManagerRef = "adplatformTransactionManager"
+)
+public class AdplatformDBConfig {
+
+    @Value("${spring.jpa.adplatform.hibernate.ddl-auto}")
+    private String ddlAuto;
+    @Value("${spring.jpa.adplatform.database}")
+    private String database;
+
+    public Map<String, Object> hibernateProperties() {
+        Map<String, Object> properties = new HashMap<>();
+        properties.put("hibernate.ddl-auto", ddlAuto);
+        properties.put("database", database);
+        return properties;
+    }
+
+    @Bean(name = "adplatformDataSource")
+    @ConfigurationProperties(prefix = "spring.datasource.adplatform")
+    public DataSource adplatformDataSource() {
+        return DataSourceBuilder.create().build();
+    }
+
+    @Bean(name = "adplatformEntityManagerFactory")
+    public LocalContainerEntityManagerFactoryBean adplatformEntityManagerFactory(EntityManagerFactoryBuilder builder,
+                                                                                @Qualifier("adplatformDataSource") DataSource dataSource) {
+        return builder
+                .dataSource(dataSource)
+                .packages("com.tzld.longarticle.recommend.server.repository.entity.adplatform") // 实体类包路径
+                .persistenceUnit("adplatform")
+                .properties(hibernateProperties())
+                .build();
+    }
+
+    @Bean(name = "adplatformTransactionManager")
+    public PlatformTransactionManager adplatformTransactionManager(
+            @Qualifier("adplatformEntityManagerFactory") EntityManagerFactory adplatformEntityManagerFactory) {
+        return new JpaTransactionManager(adplatformEntityManagerFactory);
+    }
+}
+

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/config/db/CrawlerDBConfig.java

@@ -0,0 +1,70 @@
+package com.tzld.longarticle.recommend.server.config.db;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.jdbc.DataSourceBuilder;
+import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+import org.springframework.orm.jpa.JpaTransactionManager;
+import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
+
+import javax.persistence.EntityManagerFactory;
+import javax.sql.DataSource;
+import java.util.HashMap;
+import java.util.Map;
+
+@Configuration
+@EnableTransactionManagement
+@EnableJpaRepositories(
+        basePackages = "com.tzld.longarticle.recommend.server.repository.crawler",
+        entityManagerFactoryRef = "crawlerEntityManagerFactory",
+        transactionManagerRef = "crawlerTransactionManager"
+)
+public class CrawlerDBConfig {
+
+
+    @Value("${spring.jpa.crawler.hibernate.ddl-auto}")
+    private String ddlAuto;
+    @Value("${spring.jpa.crawler.database}")
+    private String database;
+
+    public Map<String, Object> hibernateProperties() {
+        Map<String, Object> properties = new HashMap<>();
+        properties.put("hibernate.ddl-auto", ddlAuto);
+        properties.put("database", database);
+        return properties;
+    }
+
+    @Primary
+    @Bean(name = "crawlerDataSource")
+    @ConfigurationProperties(prefix = "spring.datasource.crawler")
+    public DataSource crawlerDataSource() {
+        return DataSourceBuilder.create().build();
+    }
+
+    @Primary
+    @Bean(name = "crawlerEntityManagerFactory")
+    public LocalContainerEntityManagerFactoryBean crawlerEntityManagerFactory(EntityManagerFactoryBuilder builder,
+                                                                              @Qualifier("crawlerDataSource") DataSource dataSource) {
+        return builder
+                .dataSource(dataSource)
+                .packages("com.tzld.longarticle.recommend.server.repository.entity.crawler") // 实体类包路径
+                .persistenceUnit("crawler")
+                .properties(hibernateProperties())
+                .build();
+    }
+
+    @Primary
+    @Bean(name = "crawlerTransactionManager")
+    public PlatformTransactionManager crawlerTransactionManager(
+            @Qualifier("crawlerEntityManagerFactory") EntityManagerFactory crawlerEntityManagerFactory) {
+        return new JpaTransactionManager(crawlerEntityManagerFactory);
+    }
+}
+

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/config/mybatis/AdplatformMybatisConfig.java

@@ -0,0 +1,35 @@
+package com.tzld.longarticle.recommend.server.config.mybatis;
+
+import org.apache.ibatis.session.SqlSessionFactory;
+import org.mybatis.spring.SqlSessionFactoryBean;
+import org.mybatis.spring.SqlSessionTemplate;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+
+import javax.sql.DataSource;
+
+@Configuration
+@MapperScan(basePackages = "com.tzld.longarticle.recommend.server.repository.mapper.adplatform",
+        sqlSessionFactoryRef = "adplatformSqlSessionFactory")
+public class AdplatformMybatisConfig {
+
+    @Bean(name = "adplatformSqlSessionFactory")
+    public SqlSessionFactory adplatformSqlSessionFactory(@Qualifier("adplatformDataSource") DataSource dataSource) throws Exception {
+        SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean();
+        sessionFactory.setDataSource(dataSource);
+        sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver()
+                .getResources("classpath*:mapper/**/*.xml"));
+        sessionFactory.setTypeAliasesPackage("com.tzld.longarticle.recommend.server");
+        sessionFactory.getObject().getConfiguration().setMapUnderscoreToCamelCase(true);
+        sessionFactory.getObject().getConfiguration().setUseGeneratedKeys(true);
+        return sessionFactory.getObject();
+    }
+
+    @Bean(name = "adplatformSqlSessionTemplate")
+    public SqlSessionTemplate adplatformSqlSessionTemplate(@Qualifier("adplatformSqlSessionFactory") SqlSessionFactory sqlSessionFactory) {
+        return new SqlSessionTemplate(sqlSessionFactory);
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/config/mybatis/CrawlerMybatisConfig.java

@@ -0,0 +1,38 @@
+package com.tzld.longarticle.recommend.server.config.mybatis;
+
+import org.apache.ibatis.session.SqlSessionFactory;
+import org.mybatis.spring.SqlSessionFactoryBean;
+import org.mybatis.spring.SqlSessionTemplate;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+
+import javax.sql.DataSource;
+
+@Configuration
+@MapperScan(basePackages = "com.tzld.longarticle.recommend.server.repository.mapper.crawler",
+        sqlSessionFactoryRef = "crawlerSqlSessionFactory")
+public class CrawlerMybatisConfig {
+
+    @Primary
+    @Bean(name = "crawlerSqlSessionFactory")
+    public SqlSessionFactory crawlerSqlSessionFactory(@Qualifier("crawlerDataSource") DataSource dataSource) throws Exception {
+        SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean();
+        sessionFactory.setDataSource(dataSource);
+        sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver()
+                .getResources("classpath*:mapper/**/*.xml"));
+        sessionFactory.setTypeAliasesPackage("com.tzld.longarticle.recommend.server");
+        sessionFactory.getObject().getConfiguration().setMapUnderscoreToCamelCase(true);
+        sessionFactory.getObject().getConfiguration().setUseGeneratedKeys(true);
+        return sessionFactory.getObject();
+    }
+
+    @Primary
+    @Bean(name = "crawlerSqlSessionTemplate")
+    public SqlSessionTemplate crawlerSqlSessionTemplate(@Qualifier("crawlerSqlSessionFactory") SqlSessionFactory sqlSessionFactory) {
+        return new SqlSessionTemplate(sqlSessionFactory);
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/remote/ArticleSensitive.java

@@ -6,5 +6,8 @@ import lombok.Data;
 public class ArticleSensitive {
     private String md5;
     private String title;
-    private Boolean sensitive;
+    private String sensitive;
+    private String label;
+    private String subLabel;
+    private String response;
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/ArticleListRemoteService.java

@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.tzld.longarticle.recommend.server.common.HttpPoolFactory;
 import com.tzld.longarticle.recommend.server.model.remote.Article;
+import com.tzld.longarticle.recommend.server.repository.mapper.crawler.CrawlerBaseMapper;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.http.HttpEntity;
@@ -14,9 +15,6 @@ import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.util.EntityUtils;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.jdbc.core.BeanPropertyRowMapper;
-import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
-import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
 import org.springframework.stereotype.Service;
 
 import java.nio.charset.StandardCharsets;
@@ -30,7 +28,7 @@ import java.util.Objects;
 public class ArticleListRemoteService {
 
     @Autowired
-    NamedParameterJdbcTemplate piaoquanCrawlerJdbcTemplate;
+    CrawlerBaseMapper crawlerBaseMapper;
 
     private final CloseableHttpClient client = HttpPoolFactory.aigcPool();
     private static final String articleListUrl = "http://61.48.133.26:8179/artlce_list";
@@ -94,14 +92,7 @@ public class ArticleListRemoteService {
 
 
     private List<Article> getArticleListByDB(String accountName, List<Integer> indexList) {
-        MapSqlParameterSource parameters = new MapSqlParameterSource();
-        parameters.addValue("accountName", accountName);
-        parameters.addValue("indexList", indexList);
-        List<Article> result = piaoquanCrawlerJdbcTemplate.query(
-                "select title from official_articles where accountName = :accountName and itemIndex in (:indexList) and type = 9",
-                parameters,
-                new BeanPropertyRowMapper<>(Article.class));
-        return result;
+        return crawlerBaseMapper.getOfficialArticlesTitle(accountName, indexList);
     }
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/ArticleSensitiveRemoteService.java

@@ -8,12 +8,11 @@ import com.tencentcloudapi.tms.v20201229.TmsClient;
 import com.tencentcloudapi.tms.v20201229.models.TextModerationRequest;
 import com.tencentcloudapi.tms.v20201229.models.TextModerationResponse;
 import com.tzld.longarticle.recommend.server.model.remote.ArticleSensitive;
+import com.tzld.longarticle.recommend.server.repository.mapper.crawler.CrawlerBaseMapper;
+import com.tzld.longarticle.recommend.server.util.Md5Util;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.tomcat.util.security.MD5Encoder;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
 import org.springframework.stereotype.Service;
 
 import java.util.Base64;
@@ -24,7 +23,7 @@ import java.util.Objects;
 public class ArticleSensitiveRemoteService {
 
     @Autowired
-    JdbcTemplate piaoquanCrawlerJdbcTemplate;
+    CrawlerBaseMapper crawlerBaseMapper;
 
     @Value("${tecent.cloud.secret.id:AKIDbLGZox42mZaloeIo16zeJrDRUelkbFfw}")
     private String tecentCloudSecretId;
@@ -34,10 +33,15 @@ public class ArticleSensitiveRemoteService {
     public Boolean articleSensitive(String title) {
         long start = System.currentTimeMillis();
         boolean isSensitive = false;
-        String md5 = MD5Encoder.encode(title.getBytes());
+        String md5 = Md5Util.encoderByMd5(title);
         ArticleSensitive articleSensitive = getArticleSensitive(md5);
         if (Objects.nonNull(articleSensitive)) {
-            return articleSensitive.getSensitive();
+            if ("Pass".equals(articleSensitive.getSensitive())
+                    || ("Polity".equals(articleSensitive.getLabel()) && "ForeignLeader".equals(articleSensitive.getSubLabel()))) {
+                return false;
+            } else {
+                return true;
+            }
         }
         String content = Base64.getEncoder().encodeToString(title.getBytes());
         try {
@@ -61,31 +65,24 @@ public class ArticleSensitiveRemoteService {
             log.info("articleSensitive title:{} response:{}", title, JSONObject.toJSONString(resp));
             if ("Pass".equals(resp.getSuggestion())
                     || ("Polity".equals(resp.getLabel()) && "ForeignLeader".equals(resp.getSubLabel()))) {
-                return isSensitive;
             } else {
                 isSensitive = true;
             }
+            insertArticleSensitive(md5, title, resp.getSuggestion(), resp.getLabel(), resp.getSubLabel(), JSONObject.toJSONString(resp));
         } catch (Exception e) {
             log.error("articleSensitive error", e);
         }
-        insertArticleSensitive(md5, title, isSensitive);
         log.info("articleSensitive耗时：{}", System.currentTimeMillis() - start);
         return isSensitive;
     }
 
-    private ArticleSensitive getArticleSensitive(String md5) {
-        MapSqlParameterSource parameters = new MapSqlParameterSource();
-        parameters.addValue("md5", md5);
-        ArticleSensitive result = piaoquanCrawlerJdbcTemplate.queryForObject(
-                "select md5, title, sensitiv from articles_sensitive where md5 = ?", ArticleSensitive.class, md5);
-        return result;
+    public ArticleSensitive getArticleSensitive(String md5) {
+        return crawlerBaseMapper.getArticleSensitiveByKey(md5);
     }
 
-    private void insertArticleSensitive(String md5, String title, boolean sensitive) {
+    public void insertArticleSensitive(String md5, String title, String sensitive, String label, String subLabel, String response) {
         // 插入
-        piaoquanCrawlerJdbcTemplate.update(
-                "insert into articles_sensitive(md5, title, sensitive) values (?,?,?)",
-                md5, title, sensitive);
+        crawlerBaseMapper.insertArticleSensitive(md5, title, sensitive, label, subLabel, response);
     }
 
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/ArticlePreDistributeAccountRepository.java → long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/ArticlePreDistributeAccountRepository.java

@@ -1,10 +1,9 @@
-package com.tzld.longarticle.recommend.server.repository;
+package com.tzld.longarticle.recommend.server.repository.crawler;
 
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.ArticlePreDistributeAccount;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
-import java.util.Optional;
-
 @Repository
 public interface ArticlePreDistributeAccountRepository extends JpaRepository<ArticlePreDistributeAccount,
         ArticlePreDistributeAccount.PK> {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/ArticlePreDistributeAccount.java → long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/entity/crawler/ArticlePreDistributeAccount.java

@@ -1,4 +1,4 @@
-package com.tzld.longarticle.recommend.server.repository;
+package com.tzld.longarticle.recommend.server.repository.entity.crawler;
 
 import lombok.Data;
 
@@ -23,8 +23,10 @@ public class ArticlePreDistributeAccount implements Serializable {
 
     @Data
     public static class PK implements Serializable {
+
+        @Column(name = "gh_id")
         private String ghId;
-        // YYYY-mm-dd
+        @Column(name = "date")
         private String date;
 
         public PK() {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/mapper/adplatform/AdplatformBaseMapper.java

@@ -0,0 +1,14 @@
+package com.tzld.longarticle.recommend.server.repository.mapper.adplatform;
+
+import com.tzld.longarticle.recommend.server.service.score.ChangwenArticleDTO;
+import com.tzld.longarticle.recommend.server.service.score.ChangwenArticleDatastatDTO;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+public interface AdplatformBaseMapper {
+
+    List<ChangwenArticleDTO> getChangwenArticles(@Param("channelContentIds") List<String> channelContentIds);
+
+    List<ChangwenArticleDatastatDTO> getChangwenArticleDatastats(@Param("channelContentIds") List<String> channelContentIds);
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/mapper/crawler/CrawlerBaseMapper.java

@@ -0,0 +1,24 @@
+package com.tzld.longarticle.recommend.server.repository.mapper.crawler;
+
+import com.tzld.longarticle.recommend.server.model.remote.Article;
+import com.tzld.longarticle.recommend.server.model.remote.ArticleSensitive;
+import com.tzld.longarticle.recommend.server.service.recall.ContentCategory;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+public interface CrawlerBaseMapper {
+
+    ArticleSensitive getArticleSensitiveByKey(@Param("md5") String md5);
+
+    int insertArticleSensitive(@Param("md5") String md5,
+                               @Param("title") String title,
+                               @Param("sensitive") String sensitive,
+                               @Param("label") String label,
+                               @Param("subLabel") String subLabel,
+                               @Param("response") String response);
+
+    List<Article> getOfficialArticlesTitle(@Param("accountName") String accountName, @Param("indexList") List<Integer> indexList);
+
+    List<ContentCategory> getContentCategory(@Param("channelContentIds") List<String> channelContentIds);
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/strategy/AccountPreDistributeStrategy.java

@@ -2,8 +2,8 @@ package com.tzld.longarticle.recommend.server.service.filter.strategy;
 
 import com.google.common.reflect.TypeToken;
 import com.tzld.longarticle.recommend.server.model.Content;
-import com.tzld.longarticle.recommend.server.repository.ArticlePreDistributeAccount;
-import com.tzld.longarticle.recommend.server.repository.ArticlePreDistributeAccountRepository;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.ArticlePreDistributeAccount;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticlePreDistributeAccountRepository;
 import com.tzld.longarticle.recommend.server.service.AccountContentPoolConfigService;
 import com.tzld.longarticle.recommend.server.service.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.filter.FilterResult;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recall/strategy/DefaultRecallStrategy.java

@@ -2,18 +2,15 @@ package com.tzld.longarticle.recommend.server.service.recall.strategy;
 
 import com.tzld.longarticle.recommend.server.model.Content;
 import com.tzld.longarticle.recommend.server.remote.AIGCRemoteService;
+import com.tzld.longarticle.recommend.server.repository.mapper.crawler.CrawlerBaseMapper;
 import com.tzld.longarticle.recommend.server.service.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.filter.FilterService;
 import com.tzld.longarticle.recommend.server.service.recall.*;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.jdbc.core.BeanPropertyRowMapper;
-import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
-import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
 import org.springframework.stereotype.Component;
 
-import javax.annotation.Resource;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -29,8 +26,8 @@ public class DefaultRecallStrategy implements RecallStrategy {
     private FilterService filterService;
     @Autowired
     private AIGCRemoteService aigcRemoteService;
-    @Resource(name="piaoquanCrawlerJdbcTemplate")
-    NamedParameterJdbcTemplate piaoquanCrawlerJdbcTemplate;
+    @Autowired
+    CrawlerBaseMapper crawlerBaseMapper;
 
     @Override
     public RecallResult.RecallData recall(RecallParam param) {
@@ -66,14 +63,7 @@ public class DefaultRecallStrategy implements RecallStrategy {
         if (CollectionUtils.isEmpty(channelContentIds)) {
             return new ArrayList<>();
         }
-        MapSqlParameterSource parameters = new MapSqlParameterSource();
-        parameters.addValue("channelContentIds", channelContentIds);
-        List<ContentCategory> result = piaoquanCrawlerJdbcTemplate.query(
-                "select distinct content_channel_id, category from cold_start_article_pool where content_channel_id " +
-                        "in (:channelContentIds)",
-                parameters,
-                new BeanPropertyRowMapper<>(ContentCategory.class));
-        return result;
+        return crawlerBaseMapper.getContentCategory(channelContentIds);
     }
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/score/strategy/ViewMultiplierStrategy.java

@@ -3,17 +3,15 @@ package com.tzld.longarticle.recommend.server.service.score.strategy;
 import cn.hutool.core.io.resource.ResourceUtil;
 import com.alibaba.fastjson.JSONObject;
 import com.tzld.longarticle.recommend.server.model.Content;
+import com.tzld.longarticle.recommend.server.repository.mapper.adplatform.AdplatformBaseMapper;
 import com.tzld.longarticle.recommend.server.service.score.*;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import com.tzld.longarticle.recommend.server.util.NormalizationUtils;
-import org.springframework.jdbc.core.BeanPropertyRowMapper;
-import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
-import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
 import javax.annotation.PostConstruct;
-import javax.annotation.Resource;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -23,8 +21,9 @@ import java.util.stream.Collectors;
 @Component
 public class ViewMultiplierStrategy implements ScoreStrategy {
 
-    @Resource(name = "programDataJdbcTemplate")
-    NamedParameterJdbcTemplate programDataJdbcTemplate;
+    @Autowired
+    AdplatformBaseMapper adplatformBaseMapper;
+
     private JSONObject jsonObject;
 
     @PostConstruct
@@ -63,31 +62,25 @@ public class ViewMultiplierStrategy implements ScoreStrategy {
     }
 
     private Map<String, ChangwenArticleDTO> getContentIndex(List<String> channelContentIds) {
-        MapSqlParameterSource parameters = new MapSqlParameterSource();
-        parameters.addValue("channelContentIds", channelContentIds);
-        List<ChangwenArticleDTO> result = programDataJdbcTemplate.query(
-                "select id, account_id, item_index from changwen_article where id in (:channelContentIds) ",
-                parameters,
-                new BeanPropertyRowMapper<>(ChangwenArticleDTO.class));
+        if (CollectionUtils.isEmpty(channelContentIds)) {
+            return new HashMap<>();
+        }
+        List<ChangwenArticleDTO> result = adplatformBaseMapper.getChangwenArticles(channelContentIds);
         if (CollectionUtils.isEmpty(result)) {
             return new HashMap<>();
         }
-        Map<String, ChangwenArticleDTO> map = result.stream().collect(Collectors.toMap(ChangwenArticleDTO::getId, o -> o));
-        return map;
+        return result.stream().collect(Collectors.toMap(ChangwenArticleDTO::getId, o -> o));
     }
 
     private Map<String, ChangwenArticleDatastatDTO> getContentViewCount(List<String> channelContentIds) {
-        MapSqlParameterSource parameters = new MapSqlParameterSource();
-        parameters.addValue("channelContentIds", channelContentIds);
-        List<ChangwenArticleDatastatDTO> result = programDataJdbcTemplate.query(
-                "select article_id, read_count from changwen_article_datastat where article_id in (:channelContentIds) ",
-                parameters,
-                new BeanPropertyRowMapper<>(ChangwenArticleDatastatDTO.class));
+        if (CollectionUtils.isEmpty(channelContentIds)) {
+            return new HashMap<>();
+        }
+        List<ChangwenArticleDatastatDTO> result = adplatformBaseMapper.getChangwenArticleDatastats(channelContentIds);
         if (CollectionUtils.isEmpty(result)) {
             return new HashMap<>();
         }
-        Map<String, ChangwenArticleDatastatDTO> map = result.stream().collect(Collectors.toMap(ChangwenArticleDatastatDTO::getArticleId, o -> o));
-        return map;
+        return result.stream().collect(Collectors.toMap(ChangwenArticleDatastatDTO::getArticleId, o -> o));
     }
 
     private double getAvgReadCount(String ghId, Integer index) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/util/Md5Util.java

@@ -0,0 +1,72 @@
+package com.tzld.longarticle.recommend.server.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+
+public class Md5Util {
+	static char hexDigits[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };  
+	
+	public static final String encoderByMd5(String msg) {
+		try {
+			byte[] strTemp = msg.getBytes();
+            //如果输入“SHA”，就是实现SHA加密。
+			MessageDigest mdTemp = MessageDigest.getInstance("MD5"); 
+			mdTemp.update(strTemp);
+			byte[] md = mdTemp.digest();
+			int j = md.length;
+			char str[] = new char[j * 2];
+			int k = 0;
+			for (int i = 0; i < j; i++) {
+				byte byte0 = md[i];
+				str[k++] = hexDigits[byte0 >>> 4 & 0xf];
+				str[k++] = hexDigits[byte0 & 0xf];
+			}
+			return new String(str);
+		} catch (Exception e) {
+			return null;
+		}
+	}
+
+	public static final String encoderFileByMd5(InputStream is){
+        try {  
+            MessageDigest md = MessageDigest.getInstance("MD5");  
+            byte[] buffer = new byte[2048];  
+            int length = -1;  
+            while ((length = is.read(buffer)) != -1) {  
+                md.update(buffer, 0, length);  
+            }  
+            byte[] b = md.digest();  
+            return byteToHexString(b);  
+        } catch (Exception e) {  
+            e.printStackTrace();  
+            return null;  
+        } finally {  
+            try {  
+                is.close();  
+            } catch (IOException e) {  
+                e.printStackTrace();  
+            }  
+        }  
+	}
+	
+	private static String byteToHexString(byte[] tmp) {  
+        String s;  
+        // 用字节表示就是 16 个字节  
+        // 每个字节用 16 进制表示的话，使用两个字符，所以表示成 16 进制需要 32 个字符  
+        // 比如一个字节为01011011，用十六进制字符来表示就是“5b”  
+        char str[] = new char[16 * 2];  
+        int k = 0; // 表示转换结果中对应的字符位置  
+        for (int i = 0; i < 16; i++) { // 从第一个字节开始，对 MD5 的每一个字节转换成 16 进制字符的转换  
+            byte byte0 = tmp[i]; // 取第 i 个字节  
+            str[k++] = hexDigits[byte0 >>> 4 & 0xf]; // 取字节中高 4 位的数字转换, >>> 为逻辑右移，将符号位一起右移  
+            str[k++] = hexDigits[byte0 & 0xf]; // 取字节中低 4 位的数字转换  
+        }  
+        s = new String(str); // 换后的结果转换为字符串  
+        return s;  
+    }  
+	
+	public static void main(String[] args) {
+		System.out.println(encoderByMd5("test1002"));
+	}
+}

long-article-recommend-service/src/main/resources/application-dev.yml

@@ -14,20 +14,37 @@ spring:
 #        max-idle: 8
 #        min-idle: 0
   datasource:
-    url: jdbc:mysql://rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
-    username: crawler
-    password: crawler123456@
-    driver-class-name: com.mysql.jdbc.Driver
-    hikari:
-      connection-timeout: 30000
-      minimum-idle: 5
-      maximum-pool-size: 10
-      auto-commit: true
-      idle-timeout: 30000
+    crawler:
+      jdbc-url: jdbc:mysql://rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+      username: crawler
+      password: crawler123456@
+      driver-class-name: com.mysql.jdbc.Driver
+      hikari:
+        connection-timeout: 30000
+        minimum-idle: 5
+        maximum-pool-size: 10
+        auto-commit: true
+        idle-timeout: 30000
+    adplatform:
+      jdbc-url: jdbc:mysql://rm-bp12k5fuh5zyx31d28o.mysql.rds.aliyuncs.com:3306/adplatform?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+      username: wx2023_ad
+      password: wx2023_adP@assword1234
+      driver-class-name: com.mysql.jdbc.Driver
+      hikari:
+        connection-timeout: 30000
+        minimum-idle: 5
+        maximum-pool-size: 10
+        auto-commit: true
+        idle-timeout: 30000
   jpa:
-    hibernate:
-      ddl-auto: validate
-    database: mysql
+    crawler:
+      hibernate:
+        ddl-auto: validate
+      database: mysql
+    adplatform:
+      hibernate:
+        ddl-auto: validate
+      database: mysql
 
 apollo:
   meta: http://devapolloconfig-internal.piaoquantv.com

long-article-recommend-service/src/main/resources/application-pre.yml

@@ -84,20 +84,38 @@ spring:
       minSize: 5
       maxSize: 100
   datasource:
-    url: jdbc:mysql://rm-bp1jjv3jv98133plv285-vpc-rw.mysql.rds.aliyuncs.com:3306/longvideo?useSSL=true&useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
-    username: wx2016_longvideo
-    password: wx2016_longvideoP@assword1234
-    driver-class-name: com.mysql.jdbc.Driver
-    hikari:
-      connection-timeout: 30000
-      minimum-idle: 5
-      maximum-pool-size: 10
-      auto-commit: true
-      idle-timeout: 30000
+    crawler:
+      jdbc-url: jdbc:mysql://rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+      username: crawler
+      password: crawler123456@
+      driver-class-name: com.mysql.jdbc.Driver
+      hikari:
+        connection-timeout: 30000
+        minimum-idle: 5
+        maximum-pool-size: 10
+        auto-commit: true
+        idle-timeout: 30000
+    adplatform:
+      jdbc-url: jdbc:mysql://rm-bp12k5fuh5zyx31d28o.mysql.rds.aliyuncs.com:3306/adplatform?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+      username: wx2023_ad
+      password: wx2023_adP@assword1234
+      driver-class-name: com.mysql.jdbc.Driver
+      hikari:
+        connection-timeout: 30000
+        minimum-idle: 5
+        maximum-pool-size: 10
+        auto-commit: true
+        idle-timeout: 30000
   jpa:
-    hibernate:
-      ddl-auto: validate
-    database: mysql
+    crawler:
+      hibernate:
+        ddl-auto: validate
+      database: mysql
+    adplatform:
+      hibernate:
+        ddl-auto: validate
+      database: mysql
+
 xxl:
   job:
     admin:

long-article-recommend-service/src/main/resources/application-prod.yml

@@ -3,20 +3,37 @@ server:
 
 spring:
   datasource:
-    url: jdbc:mysql://rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
-    username: crawler
-    password: crawler123456@
-    driver-class-name: com.mysql.jdbc.Driver
-    hikari:
-      connection-timeout: 30000
-      minimum-idle: 5
-      maximum-pool-size: 10
-      auto-commit: true
-      idle-timeout: 30000
+    crawler:
+      jdbc-url: jdbc:mysql://rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+      username: crawler
+      password: crawler123456@
+      driver-class-name: com.mysql.jdbc.Driver
+      hikari:
+        connection-timeout: 30000
+        minimum-idle: 5
+        maximum-pool-size: 10
+        auto-commit: true
+        idle-timeout: 30000
+    adplatform:
+      jdbc-url: jdbc:mysql://rm-bp12k5fuh5zyx31d28o.mysql.rds.aliyuncs.com:3306/adplatform?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+      username: wx2023_ad
+      password: wx2023_adP@assword1234
+      driver-class-name: com.mysql.jdbc.Driver
+      hikari:
+        connection-timeout: 30000
+        minimum-idle: 5
+        maximum-pool-size: 10
+        auto-commit: true
+        idle-timeout: 30000
   jpa:
-    hibernate:
-      ddl-auto: validate
-    database: mysql
+    crawler:
+      hibernate:
+        ddl-auto: validate
+      database: mysql
+    adplatform:
+      hibernate:
+        ddl-auto: validate
+      database: mysql
 
 apollo:
   meta: http://apolloconfig-internal.piaoquantv.com

long-article-recommend-service/src/main/resources/application-test.yml

@@ -74,20 +74,38 @@ spring:
       minSize: 5
       maxSize: 10
   datasource:
-    url: jdbc:mysql://rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com:3306/longvideo?useSSL=true&useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
-    username: wx2016_longvideo
-    password: wx2016_longvideoP@assword1234
-    driver-class-name: com.mysql.jdbc.Driver
-    hikari:
-      connection-timeout: 30000
-      minimum-idle: 5
-      maximum-pool-size: 10
-      auto-commit: true
-      idle-timeout: 30000
+    crawler:
+      jdbc-url: jdbc:mysql://rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+      username: crawler
+      password: crawler123456@
+      driver-class-name: com.mysql.jdbc.Driver
+      hikari:
+        connection-timeout: 30000
+        minimum-idle: 5
+        maximum-pool-size: 10
+        auto-commit: true
+        idle-timeout: 30000
+    adplatform:
+      jdbc-url: jdbc:mysql://rm-bp12k5fuh5zyx31d28o.mysql.rds.aliyuncs.com:3306/adplatform?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
+      username: wx2023_ad
+      password: wx2023_adP@assword1234
+      driver-class-name: com.mysql.jdbc.Driver
+      hikari:
+        connection-timeout: 30000
+        minimum-idle: 5
+        maximum-pool-size: 10
+        auto-commit: true
+        idle-timeout: 30000
   jpa:
-    hibernate:
-      ddl-auto: validate
-    database: mysql
+    crawler:
+      hibernate:
+        ddl-auto: validate
+      database: mysql
+    adplatform:
+      hibernate:
+        ddl-auto: validate
+      database: mysql
+
 xxl:
   job:
     admin:

long-article-recommend-service/src/main/resources/mapper/adplatform/AdplatformBaseMapper.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.tzld.longarticle.recommend.server.repository.mapper.adplatform.AdplatformBaseMapper">
+
+    <select id="getChangwenArticles"
+            resultType="com.tzld.longarticle.recommend.server.service.score.ChangwenArticleDTO">
+        select id, account_id, item_index from changwen_article where id in
+        <foreach collection="channelContentIds" item="item" separator="," open="(" close=")">
+            #{item}
+        </foreach>
+    </select>
+
+    <select id="getChangwenArticleDatastats"
+            resultType="com.tzld.longarticle.recommend.server.service.score.ChangwenArticleDatastatDTO">
+        select article_id, read_count from changwen_article_datastat where article_id in
+        <foreach collection="channelContentIds" item="item" separator="," open="(" close=")">
+            #{item}
+        </foreach>
+    </select>
+
+
+</mapper>

long-article-recommend-service/src/main/resources/mapper/crawler/CrawlerBaseMapper.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.tzld.longarticle.recommend.server.repository.mapper.crawler.CrawlerBaseMapper">
+
+    <insert id="insertArticleSensitive">
+        insert into article_sensitive(md5, title, `sensitive`, label, sub_label, response)
+        values (#{md5}, #{title}, #{sensitive}, #{label}, #{subLabel}, #{response})
+    </insert>
+
+
+    <select id="getArticleSensitiveByKey"
+            resultType="com.tzld.longarticle.recommend.server.model.remote.ArticleSensitive">
+        select md5, title, `sensitive`, label, sub_label, response from article_sensitive where md5 = #{md5}
+    </select>
+
+    <select id="getOfficialArticlesTitle" resultType="com.tzld.longarticle.recommend.server.model.remote.Article">
+        select title
+        from official_articles
+        where accountName = #{accountName}
+        and itemIndex in
+        <foreach collection="indexList" item="item" separator="," open="(" close=")">
+            #{item}
+        </foreach>
+        and type = 9
+    </select>
+
+    <select id="getContentCategory"
+            resultType="com.tzld.longarticle.recommend.server.service.recall.ContentCategory">
+        select distinct content_channel_id, category
+        from cold_start_article_pool
+        where content_channel_id in
+        <foreach collection="channelContentIds" item="item" separator="," open="(" close=")">
+            #{item}
+        </foreach>
+    </select>
+
+
+</mapper>

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/FilterStrategyTest.java

@@ -1,19 +1,28 @@
 package com.tzld.longarticle.recommend.server;
 
 import com.alibaba.fastjson.JSONObject;
+import com.tencentcloudapi.tms.v20201229.models.TextModerationResponse;
 import com.tzld.longarticle.recommend.server.model.Content;
 import com.tzld.longarticle.recommend.server.remote.AIGCRemoteService;
+import com.tzld.longarticle.recommend.server.remote.ArticleSensitiveRemoteService;
 import com.tzld.longarticle.recommend.server.service.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.filter.FilterStrategy;
 import com.tzld.longarticle.recommend.server.service.recall.RecallParam;
+import com.tzld.longarticle.recommend.server.util.Md5Util;
+import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Test;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.dao.DuplicateKeyException;
 
 import javax.annotation.Resource;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.util.List;
 
 @SpringBootTest(classes = Application.class)
+@Slf4j
 public class FilterStrategyTest {
 
     @Resource
@@ -22,6 +31,8 @@ public class FilterStrategyTest {
     FilterStrategy historyTitleStrategy;
     @Resource
     AIGCRemoteService aigcRemoteService;
+    @Resource
+    ArticleSensitiveRemoteService articleSensitiveRemoteService;
 //    @Test
 //    public void badStrategyTest() {
 //        FilterParam param = new FilterParam();
@@ -52,4 +63,33 @@ public class FilterStrategyTest {
         FilterResult result = historyTitleStrategy.filter(filterParam);
         System.out.println(JSONObject.toJSONString(result));
     }
+
+    @Test
+    public void dealHistoryArticleSensitiveJSON() {
+
+        String folderPath = "/Users/wangyunpeng/Downloads/moderation.tar/moderation";
+
+        File folder = new File(folderPath);
+        File[] listOfFiles = folder.listFiles((dir, name) -> name.endsWith(".json"));
+
+        if (listOfFiles != null) {
+            for (File file : listOfFiles) {
+                try {
+                    String content = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
+                    JSONObject jsonObject = JSONObject.parseObject(content);
+                    TextModerationResponse data = jsonObject.getObject("data", TextModerationResponse.class);
+                    String title = jsonObject.getString("text");
+                    String md5 = Md5Util.encoderByMd5(title);
+                    articleSensitiveRemoteService.insertArticleSensitive(md5, title, data.getSuggestion(),
+                            data.getLabel(), data.getSubLabel(), JSONObject.toJSONString(data));
+                    log.info("readFile fileName:{} content:{}", file.getName(), jsonObject.toJSONString());
+                } catch (DuplicateKeyException ignore) {
+                } catch (Exception e) {
+                    log.error("readFileError fileName:{}",file.getName(), e);
+                }
+            }
+        } else {
+            log.error("No JSON files found in the folder.");
+        }
+    }
 }