Merge branch 'wyp/0603-fwhUserGroup' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/crawler/ArticleUserGroupMapper.java

@@ -2,12 +2,11 @@ package com.tzld.longarticle.recommend.server.mapper.crawler;
 
 import com.tzld.longarticle.recommend.server.repository.model.ArticleUserGroup;
 import com.tzld.longarticle.recommend.server.repository.model.ArticleUserGroupExample;
-
-import java.util.List;
-
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
 
+import java.util.List;
+
 @Mapper
 public interface ArticleUserGroupMapper {
     /**
@@ -98,12 +97,4 @@ public interface ArticleUserGroupMapper {
      */
     int updateByPrimaryKey(ArticleUserGroup row);
 
-    int insertBatch(List<ArticleUserGroup> list);
-
-    ArticleUserGroup selectByGzhIdAndOpenId(String gzhId, String openId);
-
-    List<Integer> selectUserGroupIdByGzhId(String gzhId);
-
-    List<String> selectOpenIdByGzhIdAndUserGroupId(String gzhId, Integer userGroupId);
-
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/crawler/ext/ArticleUserGroupMapperExt.java

@@ -0,0 +1,23 @@
+package com.tzld.longarticle.recommend.server.mapper.crawler.ext;
+
+import com.tzld.longarticle.recommend.server.model.dto.UserGroupCountDTO;
+import com.tzld.longarticle.recommend.server.repository.model.ArticleUserGroup;
+import org.apache.ibatis.annotations.Mapper;
+
+import java.util.List;
+
+@Mapper
+public interface ArticleUserGroupMapperExt {
+
+    int insertBatch(List<ArticleUserGroup> list);
+
+    ArticleUserGroup selectByGzhIdAndOpenId(String gzhId, String openId);
+
+    List<UserGroupCountDTO> selectUserGroupIdByGzhId(String gzhId);
+
+    List<String> selectOpenIdByGzhIdAndUserGroupId(String gzhId, Integer userGroupId);
+
+    List<ArticleUserGroup> selectByGzhIdAndPage(String gzhId, int offset, long pageSize);
+
+    void updateUserGroup(List<Long> ids, int userGroupId);
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/UserGroupCountDTO.java

@@ -0,0 +1,9 @@
+package com.tzld.longarticle.recommend.server.model.dto;
+
+import lombok.Data;
+
+@Data
+public class UserGroupCountDTO {
+    private Integer userGroupId;
+    private Integer count;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/UserManagementService.java

@@ -1,11 +1,14 @@
 package com.tzld.longarticle.recommend.server.service;
 
-import com.tzld.longarticle.recommend.server.model.vo.WxUserGetResponse;
-import com.tzld.longarticle.recommend.server.remote.WxUserManagementRemoteService;
 import com.tzld.longarticle.recommend.server.mapper.crawler.ArticleGzhDeveloperMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.ArticleUserGroupMapper;
+import com.tzld.longarticle.recommend.server.mapper.crawler.ext.ArticleUserGroupMapperExt;
+import com.tzld.longarticle.recommend.server.model.dto.UserGroupCountDTO;
+import com.tzld.longarticle.recommend.server.model.vo.WxUserGetResponse;
+import com.tzld.longarticle.recommend.server.remote.WxUserManagementRemoteService;
 import com.tzld.longarticle.recommend.server.repository.model.ArticleGzhDeveloper;
 import com.tzld.longarticle.recommend.server.repository.model.ArticleUserGroup;
+import com.tzld.longarticle.recommend.server.repository.model.ArticleUserGroupExample;
 import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
@@ -15,10 +18,14 @@ import org.dom4j.Document;
 import org.dom4j.Element;
 import org.dom4j.io.SAXReader;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.StringReader;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * @author dyp
@@ -32,9 +39,18 @@ public class UserManagementService {
     @Autowired
     private ArticleUserGroupMapper articleUserGroupMapper;
 
+    @Autowired
+    private ArticleUserGroupMapperExt articleUserGroupMapperExt;
+
     @Autowired
     private ArticleGzhDeveloperMapper articleGzhDeveloperMapper;
 
+    @Value("${gzh.user.group.size:8}")
+    private Integer gzhUserGroupSize;
+
+    @Value("${gzh.user.group.per.count:10000}")
+    private Integer gzhUserGroupPerCount;
+
     public void addGZH(String gzhId, String appId, int groupNum) {
 
         // syncAllUserOnce(gzhId, groupNum);
@@ -65,7 +81,7 @@ public class UserManagementService {
             });
 
             //articleUserGroupRepository.saveAll(userGroups);
-            articleUserGroupMapper.insertBatch(userGroups);
+            articleUserGroupMapperExt.insertBatch(userGroups);
         }
 
         while (StringUtils.isNotBlank(nextOpenId)) {
@@ -86,10 +102,9 @@ public class UserManagementService {
                     ug.setOpenId(s);
                     ug.setUserGroupId(RandomUtils.nextInt(groupNum));
                     ug.setIsDelete(0);
-                    ug.setUserGroupId(RandomUtils.nextInt(groupNum));
                     return ug;
                 });
-                articleUserGroupMapper.insertBatch(userGroups);
+                articleUserGroupMapperExt.insertBatch(userGroups);
                 // articleUserGroupRepository.saveAll(userGroups);
             }
         }
@@ -101,7 +116,29 @@ public class UserManagementService {
         obj.setGzhId(gzhId);
         articleGzhDeveloperMapper.insert(obj);
 
+        resetUserGroup(gzhId);
+    }
+
+    private void resetUserGroup(String gzhId) {
+        Long count = getGzhUserCount(gzhId);
+        long pageSize = gzhUserGroupPerCount;
+        int pageNum = 1;
+        if (count < ((long) gzhUserGroupSize * gzhUserGroupPerCount)) {
+            pageSize = (count / gzhUserGroupSize) + 1;
+        }
+        int groupId = 0;
+        while (true) {
+            int offset = (pageNum - 1) * (int) pageSize;
+            List<ArticleUserGroup> userGroups = articleUserGroupMapperExt.selectByGzhIdAndPage(gzhId, offset, pageSize);
+            if (CollectionUtils.isEmpty(userGroups)) {
+                break;
+            }
 
+            pageNum++;
+            List<Long> ids = CommonCollectionUtils.toList(userGroups, ArticleUserGroup::getId);
+            articleUserGroupMapperExt.updateUserGroup(ids, groupId);
+            groupId++;
+        }
     }
 
 //    private void syncAllUserOnce(String gzhId, int groupNum) {
@@ -182,13 +219,14 @@ public class UserManagementService {
 
     private void handleSubscribe(String gzhId, String openId) {
 
-        ArticleUserGroup group = articleUserGroupMapper.selectByGzhIdAndOpenId(gzhId, openId);
+        ArticleUserGroup group = articleUserGroupMapperExt.selectByGzhIdAndOpenId(gzhId, openId);
+        Long count = getGzhUserCount(gzhId);
         if (group == null) {
             // insert
             // 查库，知道所有分组
-            List<Integer> ugIds = articleUserGroupMapper.selectUserGroupIdByGzhId(gzhId);
-            // 随机选一个组
-            int ugId = ugIds.get(RandomUtils.nextInt(ugIds.size()));
+            List<UserGroupCountDTO> userGroupCountDTOList = articleUserGroupMapperExt.selectUserGroupIdByGzhId(gzhId);
+            // 选择分组
+            int ugId = getUserGroupId(userGroupCountDTOList, count);
             group = new ArticleUserGroup();
             group.setGzhId(gzhId);
             group.setOpenId(openId);
@@ -203,8 +241,32 @@ public class UserManagementService {
 
     }
 
+    private Long getGzhUserCount(String gzhId) {
+        ArticleUserGroupExample countExample = new ArticleUserGroupExample();
+        countExample.createCriteria().andGzhIdEqualTo(gzhId);
+        return articleUserGroupMapper.countByExample(countExample);
+    }
+
+    private Integer getUserGroupId(List<UserGroupCountDTO> userGroupCountDTOList, long count) {
+        if (count >= ((long) gzhUserGroupSize * gzhUserGroupPerCount)
+                || userGroupCountDTOList.size() > gzhUserGroupSize) {
+            UserGroupCountDTO max = userGroupCountDTOList.get(userGroupCountDTOList.size() - 1);
+            if (max.getCount() < gzhUserGroupPerCount) {
+                return max.getUserGroupId();
+            } else {
+                return max.getUserGroupId() + 1;
+            }
+        } else {
+            int ugId = RandomUtils.nextInt(userGroupCountDTOList.size());
+            while (userGroupCountDTOList.get(ugId).getCount() >= gzhUserGroupPerCount) {
+                ugId = RandomUtils.nextInt(userGroupCountDTOList.size());
+            }
+            return ugId;
+        }
+    }
+
     private void handleUnsubscribe(String gzhId, String openId) {
-        ArticleUserGroup group = articleUserGroupMapper.selectByGzhIdAndOpenId(gzhId, openId);
+        ArticleUserGroup group = articleUserGroupMapperExt.selectByGzhIdAndOpenId(gzhId, openId);
         if (group != null) {
             group.setIsDelete(1);
             articleUserGroupMapper.updateByPrimaryKey(group);
@@ -212,6 +274,6 @@ public class UserManagementService {
     }
 
     public List<String> listByUserGroupId(String gzhId, int userGroupId) {
-        return articleUserGroupMapper.selectOpenIdByGzhIdAndUserGroupId(gzhId, userGroupId);
+        return articleUserGroupMapperExt.selectOpenIdByGzhIdAndUserGroupId(gzhId, userGroupId);
     }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/RecommendService.java

@@ -7,8 +7,9 @@ import com.tzld.longarticle.recommend.server.common.constant.SceneConstants;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PushTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
-import com.tzld.longarticle.recommend.server.mapper.crawler.ArticleUserGroupMapper;
+import com.tzld.longarticle.recommend.server.mapper.crawler.ext.ArticleUserGroupMapperExt;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
+import com.tzld.longarticle.recommend.server.model.dto.UserGroupCountDTO;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishPlan;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.PublishSortLog;
@@ -64,7 +65,7 @@ public class RecommendService {
     @Autowired
     AccountAvgInfoRepository accountAvgInfoRepository;
     @Autowired
-    private ArticleUserGroupMapper articleUserGroupMapper;
+    private ArticleUserGroupMapperExt articleUserGroupMapperExt;
     @Autowired
     private ArticleRepository articleRepository;
     @Autowired
@@ -139,8 +140,15 @@ public class RecommendService {
         }
         // 有账号策略，设置为账号策略
         String strategyConfig = accountStrategyConfigMap.get(request.getAccountName());
-        if (StringUtils.hasText(strategyConfig) && !request.isParamStrategy()) {
-            param.setStrategy(strategyConfig);
+        if (!request.isParamStrategy()) {
+            if (StringUtils.hasText(strategyConfig)) {
+                param.setStrategy(strategyConfig);
+            } else {
+                String defaultConfig = accountStrategyConfigMap.get("default");
+                if (StringUtils.hasText(defaultConfig)) {
+                    param.setStrategy(defaultConfig);
+                }
+            }
         }
         // 历史群发头条小于10条，且开启配置，则走历史表现随机策略
         if (historyCount < 10 && accountHisJumpStrategyList.contains(request.getGhId())) {
@@ -152,9 +160,10 @@ public class RecommendService {
                 param.setStrategy(RankStrategyEnum.LATE_STRATEGY.getStrategy());
             }
         }
-        // 服务号计划走服务号策略
+        // 服务号计划走服务号随机策略
         PublishPlan publishPlan = publishPlanRepository.getById(param.getPlanId());
-        if (publishPlan.getName().contains("服务号")) {
+        if (publishPlan.getName().contains("服务号")
+                && !Objects.equals(request.getPushType(), PushTypeEnum.BATCH_GROUP_PUBLISH.getVal())) {
             param.setStrategy(RankStrategyEnum.FWH_STRATEGY.getStrategy());
         }
     }
@@ -241,14 +250,14 @@ public class RecommendService {
 
         switch (abCode) {
             case FWH_COLD_START:
-                List<Integer> userGroupIds = articleUserGroupMapper.selectUserGroupIdByGzhId(request.getGhId());
+                List<UserGroupCountDTO> userGroupIds = articleUserGroupMapperExt.selectUserGroupIdByGzhId(request.getGhId());
                 LocalDate today = LocalDate.now(); // 获取当前日期
                 int dayOfYear = today.getDayOfYear();
                 int tailNum = dayOfYear % 8;
                 List<Integer> needPushUserGroupIds = new ArrayList<>();
-                for (int userGroupId : userGroupIds) {
-                    if (userGroupId % 8 == tailNum) {
-                        needPushUserGroupIds.add(userGroupId);
+                for (UserGroupCountDTO userGroup : userGroupIds) {
+                    if (userGroup.getUserGroupId() % 8 == tailNum) {
+                        needPushUserGroupIds.add(userGroup.getUserGroupId());
                     }
                 }
                 param.setUserGroupIds(needPushUserGroupIds);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -14,6 +14,7 @@ import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishPlanInputS
 import com.tzld.longarticle.recommend.server.common.enums.longArticle.ArticleVideoAuditStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.*;
 import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
+import com.tzld.longarticle.recommend.server.mapper.crawler.ArticleGzhDeveloperMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.ArticleMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
@@ -39,6 +40,8 @@ import com.tzld.longarticle.recommend.server.repository.crawler.AccountCorrelati
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleDetailInfoRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.*;
+import com.tzld.longarticle.recommend.server.repository.model.ArticleGzhDeveloper;
+import com.tzld.longarticle.recommend.server.repository.model.ArticleGzhDeveloperExample;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountIndexAvgViewCountService;
 import com.tzld.longarticle.recommend.server.service.recommend.recall.strategy.DefaultRecallStrategy;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreStrategy;
@@ -118,6 +121,8 @@ public class RecallService implements ApplicationContextAware {
     ProducePlanExeRecordRepository producePlanExeRecordRepository;
     @Autowired
     AccountRepository accountRepository;
+    @Autowired
+    ArticleGzhDeveloperMapper articleGzhDeveloperMapper;
 
     private final Map<String, RecallStrategy> strategyMap = new HashMap<>();
     private ApplicationContext applicationContext;
@@ -503,6 +508,9 @@ public class RecallService implements ApplicationContextAware {
         List<Account> accountList = accountRepository.getByStatus(1);
         Map<String, Account> accountMap = accountList.stream()
                 .collect(Collectors.toMap(Account::getGhId, Function.identity()));
+        // 获取分组发送服务号
+        List<ArticleGzhDeveloper> articleGzhDeveloperList = articleGzhDeveloperMapper.selectByExample(new ArticleGzhDeveloperExample());
+        List<String> fwhAccountGhIds = articleGzhDeveloperList.stream().map(ArticleGzhDeveloper::getGzhId).collect(Collectors.toList());
 
         for (TitleHisCacheParam cacheParam : paramList) {
             Content res = new Content();
@@ -571,6 +579,9 @@ public class RecallService implements ApplicationContextAware {
                 if (ScoreStrategy.hisContentLateFilter(hisArticle.getPublishTimestamp())) {
                     continue;
                 }
+                if (fwhAccountGhIds.contains(hisArticle.getGhId())) {
+                    continue;
+                }
                 // 历史表现 文章品类如果与历史发布账号负相关 则过滤，不计算该历史发布表现
                 JSONObject categoryWeightMap = accountCategoryMap.get(hisArticle.getGhId());
                 if (Objects.nonNull(categoryWeightMap) && CollectionUtils.isNotEmpty(cacheParam.getCategory())) {

long-article-recommend-service/src/main/resources/mapper/crawler/ArticleUserGroupMapper.xml

@@ -285,35 +285,4 @@
     where id = #{id,jdbcType=BIGINT}
   </update>
 
-  <insert id="insertBatch" parameterType="list">
-    INSERT INTO article_user_group (gzh_id, user_group_id, open_id, is_delete)
-    VALUES
-    <foreach collection="list" item="item" separator=",">
-      (#{item.gzhId}, #{item.userGroupId}, #{item.openId}, #{item.isDelete})
-    </foreach>
-  </insert>
-
-  <select id="selectByGzhIdAndOpenId" resultMap="BaseResultMap">
-    select
-    <include refid="Base_Column_List" />
-    from article_user_group
-    where gzh_id = #{gzhId}
-    and open_id = #{openId}
-  </select>
-
-  <select id="selectUserGroupIdByGzhId" parameterType="java.lang.String" resultType="integer">
-    select
-    distinct user_group_id
-    from article_user_group
-    where gzh_id = #{gzhId}
-  </select>
-
-  <select id="selectOpenIdByGzhIdAndUserGroupId" resultType="string">
-    select
-      distinct open_id
-    from article_user_group
-    where gzh_id = #{gzhId}
-    and user_group_id = #{userGroupId}
-    and is_delete = 0
-  </select>
 </mapper>

long-article-recommend-service/src/main/resources/mapper/crawler/ext/ArticleUserGroupMapperExt.xml

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.tzld.longarticle.recommend.server.mapper.crawler.ext.ArticleUserGroupMapperExt">
+    <resultMap id="BaseResultMap" type="com.tzld.longarticle.recommend.server.repository.model.ArticleUserGroup">
+        <!--
+          WARNING - @mbg.generated
+          This element is automatically generated by MyBatis Generator, do not modify.
+          This element was generated on Thu Aug 22 20:53:08 CST 2024.
+        -->
+        <id column="id" jdbcType="BIGINT" property="id"/>
+        <result column="gzh_id" jdbcType="VARCHAR" property="gzhId"/>
+        <result column="user_group_id" jdbcType="INTEGER" property="userGroupId"/>
+        <result column="open_id" jdbcType="VARCHAR" property="openId"/>
+        <result column="is_delete" jdbcType="INTEGER" property="isDelete"/>
+    </resultMap>
+
+
+    <insert id="insertBatch" parameterType="list">
+        INSERT INTO article_user_group (gzh_id, user_group_id, open_id, is_delete)
+        VALUES
+        <foreach collection="list" item="item" separator=",">
+            (#{item.gzhId}, #{item.userGroupId}, #{item.openId}, #{item.isDelete})
+        </foreach>
+    </insert>
+
+    <select id="selectByGzhIdAndOpenId" resultMap="BaseResultMap">
+        select id, gzh_id, user_group_id, open_id, is_delete
+        from article_user_group
+        where gzh_id = #{gzhId}
+          and open_id = #{openId}
+    </select>
+
+    <select id="selectUserGroupIdByGzhId" parameterType="java.lang.String"
+            resultType="com.tzld.longarticle.recommend.server.model.dto.UserGroupCountDTO">
+        select user_group_id, count(1) as count
+        from article_user_group
+        where gzh_id = #{gzhId}
+        group by user_group_id
+        order by user_group_id
+    </select>
+
+    <select id="selectOpenIdByGzhIdAndUserGroupId" resultType="string">
+        select distinct open_id
+        from article_user_group
+        where gzh_id = #{gzhId}
+          and user_group_id = #{userGroupId}
+          and is_delete = 0
+    </select>
+
+    <select id="selectByGzhIdAndPage"
+            resultType="com.tzld.longarticle.recommend.server.repository.model.ArticleUserGroup">
+        select *
+        from article_user_group
+        where gzh_id = #{gzhId}
+        order by id
+        limit #{offset}, #{pageSize}
+    </select>
+
+    <update id="updateUserGroup">
+        update article_user_group
+        set user_group_id = #{userGroupId}
+        where id in
+        <foreach collection="ids" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+    </update>
+
+</mapper>