增加长文数据库 导出写入到长文库

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/config/db/LongArticleDBConfig.java

@@ -0,0 +1,65 @@
+package com.tzld.longarticle.recommend.server.config.db;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.jdbc.DataSourceBuilder;
+import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+import org.springframework.orm.jpa.JpaTransactionManager;
+import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
+
+import javax.persistence.EntityManagerFactory;
+import javax.sql.DataSource;
+import java.util.HashMap;
+import java.util.Map;
+
+@Configuration
+@EnableTransactionManagement
+@EnableJpaRepositories(
+        basePackages = "com.tzld.longarticle.recommend.server.repository.longArticle",
+        entityManagerFactoryRef = "longArticleEntityManagerFactory",
+        transactionManagerRef = "longArticleTransactionManager"
+)
+public class LongArticleDBConfig {
+
+    @Value("${spring.jpa.longarticle.hibernate.ddl-auto}")
+    private String ddlAuto;
+    @Value("${spring.jpa.longarticle.database}")
+    private String database;
+
+    public Map<String, Object> hibernateProperties() {
+        Map<String, Object> properties = new HashMap<>();
+        properties.put("hibernate.ddl-auto", ddlAuto);
+        properties.put("database", database);
+        return properties;
+    }
+
+    @Bean(name = "longArticleDataSource")
+    @ConfigurationProperties(prefix = "spring.datasource.longarticle")
+    public DataSource longArticleDataSource() {
+        return DataSourceBuilder.create().build();
+    }
+
+    @Bean(name = "longArticleEntityManagerFactory")
+    public LocalContainerEntityManagerFactoryBean longArticleEntityManagerFactory(EntityManagerFactoryBuilder builder,
+                                                                           @Qualifier("longArticleDataSource") DataSource dataSource) {
+        return builder
+                .dataSource(dataSource)
+                .packages("com.tzld.longarticle.recommend.server.repository.entity.longArticle") // 实体类包路径
+                .persistenceUnit("longarticle")
+                .properties(hibernateProperties())
+                .build();
+    }
+
+    @Bean(name = "longArticleTransactionManager")
+    public PlatformTransactionManager longArticleTransactionManager(
+            @Qualifier("longArticleEntityManagerFactory") EntityManagerFactory longArticleEntityManagerFactory) {
+        return new JpaTransactionManager(longArticleEntityManagerFactory);
+    }
+}
+

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/config/mybatis/LongArticleMybatisConfig.java

@@ -0,0 +1,35 @@
+package com.tzld.longarticle.recommend.server.config.mybatis;
+
+import org.apache.ibatis.session.SqlSessionFactory;
+import org.mybatis.spring.SqlSessionFactoryBean;
+import org.mybatis.spring.SqlSessionTemplate;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+
+import javax.sql.DataSource;
+
+@Configuration
+@MapperScan(basePackages = "com.tzld.longarticle.recommend.server.repository.mapper.longArticle",
+        sqlSessionFactoryRef = "longArticleSqlSessionFactory")
+public class LongArticleMybatisConfig {
+
+    @Bean(name = "longArticleSqlSessionFactory")
+    public SqlSessionFactory longArticleSqlSessionFactory(@Qualifier("longArticleDataSource") DataSource dataSource) throws Exception {
+        SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean();
+        sessionFactory.setDataSource(dataSource);
+        sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver()
+                .getResources("classpath*:mapper/**/*.xml"));
+        sessionFactory.setTypeAliasesPackage("com.tzld.longarticle.recommend.server");
+        sessionFactory.getObject().getConfiguration().setMapUnderscoreToCamelCase(true);
+        sessionFactory.getObject().getConfiguration().setUseGeneratedKeys(true);
+        return sessionFactory.getObject();
+    }
+
+    @Bean(name = "longArticleSqlSessionTemplate")
+    public SqlSessionTemplate longArticleSqlSessionTemplate(@Qualifier("longArticleSqlSessionFactory") SqlSessionFactory sqlSessionFactory) {
+        return new SqlSessionTemplate(sqlSessionFactory);
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/entity/crawler/DatastatSortStrategy.java → long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/entity/longArticle/DatastatSortStrategy.java

@@ -1,4 +1,4 @@
-package com.tzld.longarticle.recommend.server.repository.entity.crawler;
+package com.tzld.longarticle.recommend.server.repository.entity.longArticle;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -13,7 +13,7 @@ import java.io.Serializable;
 @Entity
 @Table(name = "datastat_sort_strategy")
 @IdClass(DatastatSortStrategy.PK.class)
-public class DatastatSortStrategy  implements Serializable {
+public class DatastatSortStrategy implements Serializable {
 
     @Id
     private String dateStr;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/DatastatSortStrategyRepository.java → long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/DatastatSortStrategyRepository.java

@@ -1,6 +1,6 @@
-package com.tzld.longarticle.recommend.server.repository.crawler;
+package com.tzld.longarticle.recommend.server.repository.longArticle;
 
-import com.tzld.longarticle.recommend.server.repository.entity.crawler.DatastatSortStrategy;
+import com.tzld.longarticle.recommend.server.repository.entity.longArticle.DatastatSortStrategy;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/mapper/crawler/CrawlerBaseMapper.java

@@ -1,16 +1,11 @@
 package com.tzld.longarticle.recommend.server.repository.mapper.crawler;
 
 import com.tzld.longarticle.recommend.server.repository.entity.crawler.AccountCorrelation;
-import com.tzld.longarticle.recommend.server.repository.entity.crawler.DatastatSortStrategy;
 
 import java.util.List;
 
 public interface CrawlerBaseMapper {
 
-    void deleteByDateStrGreaterThanEqual(String dateStr);
-
-    void batchInsertDatastatSortStrategy(List<DatastatSortStrategy> list);
-
     void batchInsertAccountCorrelation(List<AccountCorrelation> list);
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/mapper/longArticle/LongArticleBaseMapper.java

@@ -0,0 +1,15 @@
+package com.tzld.longarticle.recommend.server.repository.mapper.longArticle;
+
+import com.tzld.longarticle.recommend.server.repository.entity.longArticle.DatastatSortStrategy;
+import org.apache.ibatis.annotations.Mapper;
+
+import java.util.List;
+
+@Mapper
+public interface LongArticleBaseMapper {
+
+    void deleteByDateStrGreaterThanEqual(String dateStr);
+
+    void batchInsertDatastatSortStrategy(List<DatastatSortStrategy> list);
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/DataDashboardService.java

@@ -16,9 +16,13 @@ import com.tzld.longarticle.recommend.server.repository.crawler.ArticleDetailInf
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.PublishSortLogRepository;
 import com.tzld.longarticle.recommend.server.repository.entity.aigc.*;
-import com.tzld.longarticle.recommend.server.repository.entity.crawler.*;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.AccountAvgInfo;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.Article;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.ArticleDetailInfo;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.PublishSortLog;
+import com.tzld.longarticle.recommend.server.repository.entity.longArticle.DatastatSortStrategy;
 import com.tzld.longarticle.recommend.server.repository.mapper.aigc.AigcBaseMapper;
-import com.tzld.longarticle.recommend.server.repository.mapper.crawler.CrawlerBaseMapper;
+import com.tzld.longarticle.recommend.server.repository.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
 import com.tzld.longarticle.recommend.server.util.MapBuilder;
 import com.tzld.longarticle.recommend.server.util.feishu.FeiShu;
@@ -52,7 +56,7 @@ public class DataDashboardService {
     @Autowired
     private PublishSortLogRepository publishSortLogRepository;
     @Autowired
-    private CrawlerBaseMapper crawlerBaseMapper;
+    private LongArticleBaseMapper longArticleBaseMapper;
     @Autowired
     private AigcBaseMapper aigcBaseMapper;
     @Autowired
@@ -407,7 +411,7 @@ public class DataDashboardService {
         result.sort(Comparator.comparing(NewSortStrategyExport::getDateStr).reversed()
                 .thenComparing(NewSortStrategyExport::getGhId).thenComparing(NewSortStrategyExport::getPosition));
         if (CollectionUtils.isNotEmpty(result)) {
-            crawlerBaseMapper.deleteByDateStrGreaterThanEqual(dateStr);
+            longArticleBaseMapper.deleteByDateStrGreaterThanEqual(dateStr);
             List<DatastatSortStrategy> saveList = new ArrayList<>();
             for (NewSortStrategyExport newSortStrategyExport : result) {
                 DatastatSortStrategy item = new DatastatSortStrategy();
@@ -415,7 +419,7 @@ public class DataDashboardService {
                 saveList.add(item);
             }
             for (List<DatastatSortStrategy> saveListPartition : Lists.partition(saveList, 1000)) {
-                crawlerBaseMapper.batchInsertDatastatSortStrategy(saveListPartition);
+                longArticleBaseMapper.batchInsertDatastatSortStrategy(saveListPartition);
             }
         }
         return result;

long-article-recommend-service/src/main/resources/application-dev.yml

@@ -51,6 +51,17 @@ spring:
         maximum-pool-size: 10
         auto-commit: true
         idle-timeout: 30000
+    longarticle:
+      jdbc-url: jdbc:mysql://rm-bp14529nwwcw75yr1ko.mysql.rds.aliyuncs.com:3306/long_articles?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&useSSL=false
+      username: changwen_admin
+      password: changwen@123456
+      driver-class-name: com.mysql.jdbc.Driver
+      hikari:
+        connection-timeout: 30000
+        minimum-idle: 5
+        maximum-pool-size: 10
+        auto-commit: true
+        idle-timeout: 30000
   jpa:
     crawler:
       hibernate:
@@ -66,6 +77,10 @@ spring:
       hibernate:
         ddl-auto: validate
       database: mysql
+    longarticle:
+      hibernate:
+        ddl-auto: validate
+      database: mysql
 
 apollo:
   meta: http://devapolloconfig-internal.piaoquantv.com

long-article-recommend-service/src/main/resources/application-prod.yml

@@ -36,6 +36,17 @@ spring:
         maximum-pool-size: 10
         auto-commit: true
         idle-timeout: 30000
+    longarticle:
+      jdbc-url: jdbc:mysql://rm-bp14529nwwcw75yr1ko.mysql.rds.aliyuncs.com:3306/long_articles?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&useSSL=false
+      username: changwen_admin
+      password: changwen@123456
+      driver-class-name: com.mysql.jdbc.Driver
+      hikari:
+        connection-timeout: 30000
+        minimum-idle: 5
+        maximum-pool-size: 10
+        auto-commit: true
+        idle-timeout: 30000
   jpa:
     crawler:
       hibernate:
@@ -49,6 +60,10 @@ spring:
       hibernate:
         ddl-auto: validate
       database: mysql
+    longarticle:
+      hibernate:
+        ddl-auto: validate
+      database: mysql
 
 apollo:
   meta: http://apolloconfig-internal.piaoquantv.com

long-article-recommend-service/src/main/resources/mapper/crawler/CrawlerBaseMapper.xml

@@ -2,35 +2,6 @@
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="com.tzld.longarticle.recommend.server.repository.mapper.crawler.CrawlerBaseMapper">
 
-    <delete id="deleteByDateStrGreaterThanEqual">
-        delete from datastat_sort_strategy where date_str >= #{dateStr}
-    </delete>
-
-    <insert id="batchInsertDatastatSortStrategy">
-        INSERT INTO datastat_sort_strategy
-        (date_str, publish_time, account_mode, account_source, account_type, account_status, account_name, strategy,
-        fans, view_count, avg_view_count, first_view_count, first_avg_view_count, first_level, fission0, fission1,
-        fission2, read_rate, read_fans_rate, first_read_rate, fission0_first_rate, fission1_fission0_rate,
-        fission0_read_avg_rate, position, gh_id, title, link, wx_sn, fission0_read_avg_100_rate,
-        fission0_read_avg_500_rate, fission0_read_avg_1000_rate, crawler_plan_name, crawler_plan_tag,
-        produce_plan_name, produce_plan_tag, publish_plan_name, publish_mini_program_insert_strategy,
-        publish_mini_program_insert_use_type, publish_mini_program_num)
-        VALUES
-        <foreach collection="list" item="item" separator=",">
-            (#{item.dateStr}, #{item.publishTime}, #{item.accountMode}, #{item.accountSource}, #{item.accountType},
-            #{item.accountStatus}, #{item.accountName}, #{item.strategy}, #{item.fans}, #{item.viewCount},
-            #{item.avgViewCount}, #{item.firstViewCount}, #{item.firstAvgViewCount}, #{item.firstLevel},
-            #{item.fission0}, #{item.fission1}, #{item.fission2}, #{item.readRate}, #{item.readFansRate},
-            #{item.firstReadRate}, #{item.fission0FirstRate}, #{item.fission1Fission0Rate},
-            #{item.fission0ReadAvgRate}, #{item.position}, #{item.ghId}, #{item.title}, #{item.link},
-            #{item.wxSn}, #{item.fission0ReadAvg100Rate}, #{item.fission0ReadAvg500Rate},
-            #{item.fission0ReadAvg1000Rate}, #{item.crawlerPlanName}, #{item.crawlerPlanTag},
-            #{item.producePlanName}, #{item.producePlanTag}, #{item.publishPlanName},
-            #{item.publishMiniProgramInsertStrategy}, #{item.publishMiniProgramInsertUseType},
-            #{item.publishMiniProgramNum})
-        </foreach>
-    </insert>
-
     <insert id="batchInsertAccountCorrelation">
         INSERT INTO account_correlation (date_str, gh_id, account_name, rel_gh_id, rel_account_name, status, correlation)
         VALUES

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.tzld.longarticle.recommend.server.repository.mapper.longArticle.LongArticleBaseMapper">
+
+    <delete id="deleteByDateStrGreaterThanEqual">
+        delete from datastat_sort_strategy where date_str >= #{dateStr}
+    </delete>
+
+    <insert id="batchInsertDatastatSortStrategy">
+        INSERT INTO datastat_sort_strategy
+        (date_str, publish_time, account_mode, account_source, account_type, account_status, account_name, strategy,
+        fans, view_count, avg_view_count, first_view_count, first_avg_view_count, first_level, fission0, fission1,
+        fission2, read_rate, read_fans_rate, first_read_rate, fission0_first_rate, fission1_fission0_rate,
+        fission0_read_avg_rate, position, gh_id, title, link, wx_sn, fission0_read_avg_100_rate,
+        fission0_read_avg_500_rate, fission0_read_avg_1000_rate, crawler_plan_name, crawler_plan_tag,
+        produce_plan_name, produce_plan_tag, publish_plan_name, publish_mini_program_insert_strategy,
+        publish_mini_program_insert_use_type, publish_mini_program_num)
+        VALUES
+        <foreach collection="list" item="item" separator=",">
+            (#{item.dateStr}, #{item.publishTime}, #{item.accountMode}, #{item.accountSource}, #{item.accountType},
+            #{item.accountStatus}, #{item.accountName}, #{item.strategy}, #{item.fans}, #{item.viewCount},
+            #{item.avgViewCount}, #{item.firstViewCount}, #{item.firstAvgViewCount}, #{item.firstLevel},
+            #{item.fission0}, #{item.fission1}, #{item.fission2}, #{item.readRate}, #{item.readFansRate},
+            #{item.firstReadRate}, #{item.fission0FirstRate}, #{item.fission1Fission0Rate},
+            #{item.fission0ReadAvgRate}, #{item.position}, #{item.ghId}, #{item.title}, #{item.link},
+            #{item.wxSn}, #{item.fission0ReadAvg100Rate}, #{item.fission0ReadAvg500Rate},
+            #{item.fission0ReadAvg1000Rate}, #{item.crawlerPlanName}, #{item.crawlerPlanTag},
+            #{item.producePlanName}, #{item.producePlanTag}, #{item.publishPlanName},
+            #{item.publishMiniProgramInsertStrategy}, #{item.publishMiniProgramInsertUseType},
+            #{item.publishMiniProgramNum})
+        </foreach>
+    </insert>
+
+</mapper>