fuwuhao

long-article-recommend-service/pom.xml

@@ -116,6 +116,12 @@
             <groupId>org.mybatis.spring.boot</groupId>
             <artifactId>mybatis-spring-boot-starter</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.mybatis.generator</groupId>
+            <artifactId>mybatis-generator-core</artifactId>
+            <version>1.4.2</version>
+        </dependency>
+
 
     </dependencies>
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/MBG.java

@@ -0,0 +1,28 @@
+package com.tzld.longarticle.recommend.server;
+
+import org.mybatis.generator.api.MyBatisGenerator;
+import org.mybatis.generator.config.Configuration;
+import org.mybatis.generator.config.xml.ConfigurationParser;
+import org.mybatis.generator.internal.DefaultShellCallback;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author dyp
+ */
+public class MBG {
+    public static void main(String[] args) throws Exception {
+        List<String> warnings = new ArrayList<String>();
+        boolean overwrite = true;
+        File configFile = new File("generatorConfig.xml");
+        ConfigurationParser cp = new ConfigurationParser(warnings);
+        Configuration config = cp.parseConfiguration(configFile);
+        DefaultShellCallback callback = new DefaultShellCallback(overwrite);
+        MyBatisGenerator myBatisGenerator = new MyBatisGenerator(config, callback, warnings);
+        myBatisGenerator.generate(null);
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/WxAccessTokenRemoteService.java

@@ -44,11 +44,11 @@ public class WxAccessTokenRemoteService {
     private Map<String, Map<String, String>> gzhConfig;
 
     public String getAccessToken(String gzhId) {
-        return cache.getUnchecked(gzhId);
+        return "83_UKkuWGS175MX05L95rdaJ_8qyH_HsOs78UxXjQFeiFoifuHt35wbiXreE6E37tPbxm_ACeqySrft02uML4K0SAog5mrkbG1xwIeFoV0zKax8pTsb3ZSewBJXjct4jgTn76PS8ZJOtJ2soAyUSHZdAFDNCU";
     }
 
     private String loadAccessToken(String gzhId) {
-        return "83_FQXl8oow9m0xNoEmb5dcYmdeas3nsJzNQP9ie_zBUqa8oILqdq0xK-_VUYEb27QqcSMSDTU8a28TfOTBTfDoZGkMEo43Aq-2gAq4qWVOcGzEQbCuEJ29bjgKpQycMLJB_fGMGAiujjD-TKfQLUXdALDOXA";
+        return "83_UKkuWGS175MX05L95rdaJ_8qyH_HsOs78UxXjQFeiFoifuHt35wbiXreE6E37tPbxm_ACeqySrft02uML4K0SAog5mrkbG1xwIeFoV0zKax8pTsb3ZSewBJXjct4jgTn76PS8ZJOtJ2soAyUSHZdAFDNCU";
     }
 
 //    private String loadAccessToken(String gzhId) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/ArticleUserGroupRepository.java

@@ -1,9 +1,10 @@
 package com.tzld.longarticle.recommend.server.repository.crawler;
 
-import com.tzld.longarticle.recommend.server.repository.entity.crawler.ArticlePreDistributeAccount;
 import com.tzld.longarticle.recommend.server.repository.entity.crawler.ArticleUserGroup;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
 
 import java.util.List;
@@ -18,4 +19,8 @@ public interface ArticleUserGroupRepository extends JpaRepository<ArticleUserGro
     @Query("select distinct openId from ArticleUserGroup where gzhId = :gzhId" +
             " and userGroupId = :userGroupId and isDelete = 0")
     List<String> findAllOpenId(String gzhId, int userGroupId);
+
+    @Modifying
+    @Query("UPDATE ArticleUserGroup a SET a.userGroupId = a.id % :groupNum WHERE a.gzhId = :gzhId")
+    void updateUserGroupIdWithMod(@Param("gzhId") String gzhId, @Param("groupNum") int groupNum);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/entity/crawler/ArticleUserGroup.java

@@ -12,7 +12,8 @@ import javax.persistence.*;
 @Table(name = "article_user_group")
 public class ArticleUserGroup {
     @Id
-    private long id;
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
     @Column(name = "gzh_id")
     private String gzhId;
     @Column(name = "user_group_id")
@@ -21,4 +22,4 @@ public class ArticleUserGroup {
     private String openId;
     @Column(name = "is_delete")
     private int isDelete;
-}
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/UserManagementService.java

@@ -14,11 +14,11 @@ import org.dom4j.Element;
 import org.dom4j.io.SAXReader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+import java.util.*;
 
 /**
  * @author dyp
@@ -31,6 +31,9 @@ public class UserManagementService {
     @Autowired
     private ArticleUserGroupRepository articleUserGroupRepository;
 
+    @Autowired
+    private UserManagementTransactionalService userManagementTransactionalService;
+
     public void addGZH(String gzhId, int groupNum) {
 
         // syncAllUserOnce(gzhId, groupNum);
@@ -58,7 +61,9 @@ public class UserManagementService {
                 ug.setUserGroupId(RandomUtils.nextInt(groupNum));
                 return ug;
             });
-            articleUserGroupRepository.saveAll(userGroups);
+
+            //articleUserGroupRepository.saveAll(userGroups);
+            userManagementTransactionalService.insertAll(userGroups);
         }
 
         while (StringUtils.isNotBlank(nextOpenId)) {
@@ -77,14 +82,17 @@ public class UserManagementService {
                     ArticleUserGroup ug = new ArticleUserGroup();
                     ug.setGzhId(gzhId);
                     ug.setOpenId(s);
-                    ug.setUserGroupId(RandomUtils.nextInt(groupNum));
+                    // ug.setUserGroupId(RandomUtils.nextInt(groupNum));
                     return ug;
                 });
-                articleUserGroupRepository.saveAll(userGroups);
+                userManagementTransactionalService.insertAll(userGroups);
+                // articleUserGroupRepository.saveAll(userGroups);
             }
         }
 
         log.info("end sync user gzhId={}", gzhId);
+        log.info("start group gzhId={}", gzhId);
+
 
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/UserManagementTransactionalService.java

@@ -0,0 +1,45 @@
+package com.tzld.longarticle.recommend.server.service;
+
+import com.tzld.longarticle.recommend.server.model.WxUserGetResponse;
+import com.tzld.longarticle.recommend.server.remote.WxUserManagementRemoteService;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleUserGroupRepository;
+import com.tzld.longarticle.recommend.server.repository.entity.crawler.ArticleUserGroup;
+import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.math.RandomUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.dom4j.Document;
+import org.dom4j.Element;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+import java.util.*;
+
+/**
+ * @author dyp
+ */
+@Slf4j
+@Service
+public class UserManagementTransactionalService {
+
+    @PersistenceContext
+    private EntityManager entityManager;
+
+    @Transactional
+    public void insertAll(Iterable<ArticleUserGroup> entities) {
+        Iterator<ArticleUserGroup> iterator = entities.iterator();
+
+        while (iterator.hasNext()) {
+            ArticleUserGroup next = iterator.next();
+            entityManager.persist(next);
+        }
+        entityManager.flush();
+        entityManager.clear();
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/UserManagementController.java

@@ -26,7 +26,7 @@ public class UserManagementController {
 
 
     @RequestMapping("/addGZH")
-    public String homepageRecommend(@RequestParam(name = "gzhId") String gzhId,
+    public String addGZH(@RequestParam(name = "gzhId") String gzhId,
                                     @RequestParam(name = "groupNum") int groupNum) {
         userManagementService.addGZH(gzhId, groupNum);
         return "success";

long-article-recommend-service/src/main/resources/application-dev.yml

@@ -51,6 +51,8 @@ spring:
     crawler:
       hibernate:
         ddl-auto: validate
+        jdbc:
+          batch_size: 2000
       database: mysql
     adplatform:
       hibernate:

long-article-recommend-service/src/main/resources/generatorConfig.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE generatorConfiguration
+        PUBLIC "-//mybatis.org//DTD MyBatis Generator Configuration 1.0//EN"
+        "http://mybatis.org/dtd/mybatis-generator-config_1_0.dtd">
+
+<generatorConfiguration>
+    <context id="simple" targetRuntime="MyBatis3">
+        <jdbcConnection driverClass="com.mysql.jdbc.Driver"
+                        connectionURL="jdbc:mysql://rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com:3306/piaoquan-crawler?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&useSSL=false"
+                        userId="crawler"
+                        password="crawler123456@">
+        </jdbcConnection>
+
+        <javaTypeResolver >
+            <property name="forceBigDecimals" value="false" />
+        </javaTypeResolver>
+
+        <javaModelGenerator targetPackage="model" targetProject="./target">
+            <property name="enableSubPackages" value="true" />
+            <property name="trimStrings" value="true" />
+        </javaModelGenerator>
+
+        <sqlMapGenerator targetPackage="xml"  targetProject="./src/resources/mapper">
+            <property name="enableSubPackages" value="true" />
+        </sqlMapGenerator>
+
+        <javaClientGenerator type="XMLMAPPER" targetPackage="mapper"  targetProject="./target">
+            <property name="enableSubPackages" value="true" />
+        </javaClientGenerator>
+
+        <table schema="mybatis" tableName="article_user_group" >
+            <generatedKey column="id" sqlStatement="mysql" identity="true" />
+        </table>
+
+    </context>
+</generatorConfiguration>

long-article-recommend-service/src/main/resources/mapper/crawler/CrawlerBaseMapper.xml

@@ -3,5 +3,4 @@
 <mapper namespace="com.tzld.longarticle.recommend.server.repository.mapper.crawler.CrawlerBaseMapper">
 
 
-
 </mapper>