Ver código fonte

Merge branch 'master' into dev-xym-update-3rd

xueyiming 7 meses atrás
pai
commit
a01d7533f3

+ 1 - 0
long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/RankStrategyEnum.java

@@ -20,6 +20,7 @@ public enum RankStrategyEnum {
     ArticleRankV13("ArticleRankV13", "ArticleRankV13", "rankV13Strategy"),
     ArticleRankV14("ArticleRankV14", "ArticleRankV14", "rankV14Strategy"),
 
+    HIS_JUMP_STRATEGY("ArticleRankHisJump", "历史表现跳过相似度策略", "hisJumpRankStrategy"),
     INFINITE_STRATEGY("ArticleRankInfinite", "无限发表", "infiniteRankStrategy"),
     LATE_STRATEGY("ArticleRankLate", "晚间策略", "lateRankStrategy"),
     RANDOM_STRATEGY("ArticleRankRandom", "随机策略", "randomRankStrategy"),

+ 2 - 0
long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/param/PushMessageParam.java

@@ -1,8 +1,10 @@
 package com.tzld.longarticle.recommend.server.model.param;
 
 import lombok.Data;
+import lombok.ToString;
 
 @Data
+@ToString
 public class PushMessageParam {
 
     private String accessToken;

+ 1 - 0
long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/crawler/ArticleRepository.java

@@ -33,4 +33,5 @@ public interface ArticleRepository extends JpaRepository<Article, String> {
 
     Article getByWxSn(String wxSn);
 
+    int countByGhIdAndTypeAndItemIndex(String ghId, String val, Integer itemIndex);
 }

+ 37 - 6
long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/XxlJobService.java

@@ -40,6 +40,9 @@ import com.tzld.longarticle.recommend.server.util.LarkRobotUtil;
 import com.tzld.longarticle.recommend.server.util.feishu.FeishuMessageSender;
 import com.xxl.job.core.biz.model.ReturnT;
 import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.dao.DuplicateKeyException;
@@ -119,6 +122,22 @@ public class XxlJobService {
         return ReturnT.SUCCESS;
     }
 
+    @Getter
+    @Setter
+    @Accessors(chain = true)
+    public static class PlanErrorMsg {
+        private String date;
+        List<PlanErrorMsgDetail> errorMsgDetails;
+    }
+
+    @Getter
+    @Setter
+    @Accessors(chain = true)
+    public static class PlanErrorMsgDetail {
+        private String accountName;
+        private String errorMsg;
+    }
+
     private void sendFeishuPublishPlanNotPushWarn(NotPublishPlan publishPlan) {
         Long todayStart = DateUtils.getTodayStart();
         List<PublishAccount> publishAccounts = aigcBaseMapper.getPublishAccounts(publishPlan.getPlanId(), todayStart);
@@ -128,11 +147,23 @@ public class XxlJobService {
         } else {
             return;
         }
-        if (StringUtils.hasText(publishPlan.getErrorMsg()) && (
-                publishPlan.getErrorMsg().contains("45028")
-                        || publishPlan.getErrorMsg().contains("48004")
-                        || publishPlan.getErrorMsg().contains("50002")
-        )) {
+        PlanErrorMsg planErrorMsg = JSONObject.parseObject(publishPlan.getErrorMsg(), PlanErrorMsg.class);
+        if (planErrorMsg == null) {
+            return;
+        } else if (CollectionUtil.isEmpty(planErrorMsg.getErrorMsgDetails())) {
+            return;
+        }
+        Iterator<PlanErrorMsgDetail> iterator = planErrorMsg.getErrorMsgDetails().iterator();
+        while (iterator.hasNext()) {
+            PlanErrorMsgDetail errorMsgDetail = iterator.next();
+            if (errorMsgDetail.getErrorMsg().contains("45028")
+                    || errorMsgDetail.getErrorMsg().contains("48004")
+                    || errorMsgDetail.getErrorMsg().contains("50002")) {
+                iterator.remove();
+                accountNames.remove(errorMsgDetail.getAccountName());
+            }
+        }
+        if (CollectionUtil.isEmpty(planErrorMsg.getErrorMsgDetails())) {
             return;
         }
         FeishuMessageSender.sendWebHookMessage(FeishuRobotIdEnum.RECOMMEND.getRobotId(),
@@ -142,7 +173,7 @@ public class XxlJobService {
                         + "预计数量: " + publishPlan.getExpectCount() + "\n"
                         + "实际数量: " + publishPlan.getPublishCount() + "\n"
                         + "发送失败账号: " + JSONObject.toJSONString(accountNames) + "\n"
-                        + "发布计划失败原因: " + publishPlan.getErrorMsg() + "\n"
+                        + "发布计划失败原因: " + JSONObject.toJSONString(planErrorMsg) + "\n"
                         + "发布时间: " + publishPlan.getPublishPushTime() + "\n"
                         + "<at user_id=\"all\">所有人</at> ");
     }

+ 33 - 3
long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/exterior/impl/ThirdPartyServiceImpl.java

@@ -3,10 +3,14 @@ package com.tzld.longarticle.recommend.server.service.exterior.impl;
 import com.alibaba.fastjson.JSON;
 import com.aliyun.odps.data.Record;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.constant.TimeConstant;
+import com.tzld.longarticle.recommend.server.common.enums.GhTypeEnum;
 import com.tzld.longarticle.recommend.server.common.enums.SecretEnum;
+import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.cgi.ReplyStrategyServiceEnum;
 import com.tzld.longarticle.recommend.server.common.response.CommonResponse;
 import com.tzld.longarticle.recommend.server.common.response.ExceptionCodeEnum;
+import com.tzld.longarticle.recommend.server.mapper.growth.GhDetailMapper;
 import com.tzld.longarticle.recommend.server.model.bo.MiniData;
 import com.tzld.longarticle.recommend.server.model.cgi.BucketDataParam;
 import com.tzld.longarticle.recommend.server.model.cgi.GroupData;
@@ -15,10 +19,13 @@ import com.tzld.longarticle.recommend.server.model.cgi.ReplyBucketData;
 import com.tzld.longarticle.recommend.server.model.param.PushMessageParam;
 import com.tzld.longarticle.recommend.server.model.vo.PushMessageVo;
 import com.tzld.longarticle.recommend.server.model.vo.ReportUvVo;
+import com.tzld.longarticle.recommend.server.repository.model.GhDetail;
+import com.tzld.longarticle.recommend.server.repository.model.GhDetailExample;
 import com.tzld.longarticle.recommend.server.service.exterior.AccessTokenService;
 import com.tzld.longarticle.recommend.server.service.exterior.ThirdPartyService;
 import com.tzld.longarticle.recommend.server.service.strategy.reply.ReplyStrategyService;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
+import com.tzld.longarticle.recommend.server.util.LarkRobotUtil;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -33,6 +40,7 @@ import javax.annotation.PostConstruct;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 @Slf4j
 @Service
@@ -47,6 +55,9 @@ public class ThirdPartyServiceImpl implements ThirdPartyService {
     @Autowired
     private ODPSManager odpsManager;
 
+    @Autowired
+    private GhDetailMapper ghDetailMapper;
+
     @ApolloJsonValue("${canViewReportDate:2024-10-31}")
     private String canViewReportDate;
 
@@ -70,7 +81,23 @@ public class ThirdPartyServiceImpl implements ThirdPartyService {
         if (secretEnum == null) {
             return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "获取secret失败");
         }
-        String channel = secretEnum.channel;
+        log.info("getPushMessage param={} secretEnum desc={}", param, secretEnum.desc);
+        GhDetailExample example = new GhDetailExample();
+        example.createCriteria().andTypeEqualTo(GhTypeEnum.THIRD_PARTY_GH.type).andGhIdEqualTo(param.getGhId());
+        List<GhDetail> ghDetails = ghDetailMapper.selectByExample(example);
+        if (CollectionUtils.isEmpty(ghDetails)) {
+            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "ghId不存在,请联系管理员配置");
+        }
+        String channel = ghDetails.get(0).getChannel();
+        if (channel == null) {
+            LarkRobotUtil.sendMessage("channel不存在,请查看详情 ghId=", param.getGhId());
+            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "ghId异常,请联系管理员检查");
+        }
+        if (!Objects.equals(secretEnum.channel, channel)) {
+            LarkRobotUtil.sendMessage(String.format("channel异常 secretEnum.channel=%s ghDetail.channel=%s ghId=%s",
+                    secretEnum.channel, channel, param.getGhId()));
+            return CommonResponse.create(ExceptionCodeEnum.PARAM_ERROR, "ghId异常,请联系管理员检查");
+        }
         List<PushMessageVo> pushMessageVoList = new ArrayList<>();
         ReplyBucketData replyBucketData = getPushMessageData(param, channel);
         log.info("replyBucketData={}", JSON.toJSONString(replyBucketData));
@@ -116,9 +143,12 @@ public class ThirdPartyServiceImpl implements ThirdPartyService {
         if (secretEnum == SecretEnum.SECRET_ENUM_2) {
             return CommonResponse.create(500, "数据不存在");
         }
-        String channel = secretEnum.channel;
+
+        //10点后可查询前一天数据
+        long nowTimestamp = System.currentTimeMillis() / 1000;
+        long limitTime = nowTimestamp - 34L * TimeConstant.HOUR;
         long targetTime = DateUtils.dateStrToTimestamp(date, "yyyy-MM-dd");
-        long limitTime = DateUtils.dateStrToTimestamp(canViewReportDate, "yyyy-MM-dd");
+        String channel = secretEnum.channel;
         if (targetTime > limitTime) {
             return CommonResponse.create(500, "数据不存在");
         }

+ 2 - 0
long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleService.java

@@ -465,6 +465,7 @@ public class ArticleService {
                 "名人八卦\n" +
                 "政治新闻\n" +
                 "军事新闻\n" +
+                "影视解读\n" +
                 "为了更好地完成任务,可参考下列对文章标题的分类:\n" +
                 "{" +
                 "\"大舅病了,我取了三万元送过去,病房门口听到舅妈的话我改了主意\": \"家长里短\",\n" +
@@ -477,6 +478,7 @@ public class ArticleService {
                 "\"中国最美的女将军:上世纪曾家喻户晓,如今仍然健在\": \"历史人物\",\n" +
                 "\"北大才女蒙曼48岁仍未婚,被问最想嫁给谁,一个名字让全场笑喷\": \"名人八卦\",\n" +
                 "\"广东一老人去世,家人把老人的旧床垫扔了,环卫工人看到后,竟发现里面藏了15万元现金!家人傻眼了\": \"奇闻趣事\",\n" +
+                "\"《孤舟》结局:要不是海沫2年不生娃,周知非至死不知,顾易中要3000元的报销费的真实目的!\": \"影视解读\"\n" +
                 "}" +
                 "最后输出结果请用JSON格式输出,key为title,value为类目,仅输出JSON,不要markdown格式,不要任何其他内容," +
                 "并且内容可以被 fastJSON 的JSONObject.parseObject转换为JSON对象\n" +

+ 14 - 0
long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/RecommendService.java

@@ -18,6 +18,7 @@ import com.tzld.longarticle.recommend.server.model.vo.ArticleSortResponseDataIte
 import com.tzld.longarticle.recommend.server.model.vo.RecommendResponse;
 import com.tzld.longarticle.recommend.server.model.vo.RecommendWithUserGroupResponse;
 import com.tzld.longarticle.recommend.server.repository.crawler.AccountAvgInfoRepository;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.PublishContentSortLogRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.PublishSortLogRepository;
 import com.tzld.longarticle.recommend.server.common.constant.SceneConstants;
@@ -71,9 +72,13 @@ public class RecommendService {
     private ArticleUserGroupMapper articleUserGroupMapper;
     @Autowired
     private AigcBaseMapper aigcBaseMapper;
+    @Autowired
+    private ArticleRepository articleRepository;
 
     @ApolloJsonValue("${accountStrategyConfig:{}}")
     private Map<String, String> accountStrategyConfigMap;
+    @ApolloJsonValue("${accountHisJumpStrategyConfig:[]]}")
+    private List<String> accountHisJumpStrategyList;
     @Value("${spring.profiles.active}")
     private String env;
 
@@ -95,16 +100,25 @@ public class RecommendService {
     }
 
     private void setStrategy(RecommendRequest request, RecommendParam param) {
+        // 无限发表,设置为无限发表策略
         if (Objects.equals(request.getPushType(), PushTypeEnum.AUTO_PUBLISH.getVal())
                 || Objects.equals(request.getPushType(), PushTypeEnum.ROBOPOST.getVal())) {
             param.setStrategy(RankStrategyEnum.INFINITE_STRATEGY.getStrategy());
             param.setType(ArticleTypeEnum.WUXIANLIU.getVal());
             return;
         }
+        // 有账号策略,设置为账号策略
         String strategyConfig = accountStrategyConfigMap.get(request.getAccountName());
         if (StringUtils.hasText(strategyConfig) && !request.isParamStrategy()) {
             param.setStrategy(strategyConfig);
         }
+        // 历史群发头条小于10条,且开启配置,则走历史表现随机策略
+        int historyCount = articleRepository.countByGhIdAndTypeAndItemIndex(request.getGhId(),
+                ArticleTypeEnum.QUNFA.getVal(), 1);
+        if (historyCount < 10 && accountHisJumpStrategyList.contains(request.getGhId())) {
+            param.setStrategy(RankStrategyEnum.HIS_JUMP_STRATEGY.getStrategy());
+        }
+        // 凌晨19点之后,设置为晚上策略
         if (DateUtils.getCurrentHour() >= 19 && !request.isParamStrategy()) {
             if (Objects.equals(request.getPushType(), PushTypeEnum.AUTO_GROUP_PUBLISH.getVal())) {
                 param.setStrategy(RankStrategyEnum.LATE_STRATEGY.getStrategy());

+ 124 - 0
long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/HisJumpRankStrategy.java

@@ -0,0 +1,124 @@
+package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
+
+import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
+import com.tzld.longarticle.recommend.server.model.dto.Content;
+import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
+import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
+import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
+import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
+import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
+import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
+import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+
+@Service
+@Slf4j
+public class HisJumpRankStrategy implements RankStrategy {
+
+    @Autowired
+    private ScoreService scoreService;
+    @Autowired
+    private AccountContentPoolConfigService accountContentPoolConfigService;
+    @Autowired
+    private ArticleRepository articleRepository;
+    @Autowired
+    private StrategyIndexScoreWeightService weightService;
+
+    public RankResult rank(RankParam param) {
+        List<Content> result = new ArrayList<>();
+
+        ScoreResult scoreResult = scoreService.score(RankStrategy.convertToScoreParam(param));
+
+        Map<String, Map<String, Double>> scoreMap = scoreResult.getScoreMap();
+        String[] contentPools = accountContentPoolConfigService.getContentPools(param.getAccountName());
+        Map<Integer, AccountIndexReplacePoolConfig> indexReplacePoolConfigMap = accountContentPoolConfigService.getContentReplacePools(param.getAccountName());
+
+        List<RankItem> items = CommonCollectionUtils.toList(param.getContents(), c -> {
+            RankItem item = new RankItem();
+            item.setContent(c);
+            c.setScoreMap(scoreMap.get(c.getId()));
+            item.setScoreMap(scoreMap.get(c.getId()));
+            double score;
+            int index = weightService.getIndex(item.getContent().getContentPoolType(), contentPools);
+            if (contentPools[0].equals(item.getContent().getContentPoolType())
+                    || contentPools[1].equals(item.getContent().getContentPoolType())) {
+                score = item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value())
+                        + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value());
+                if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
+                    score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
+                            * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                            ScoreStrategyEnum.VIEW_COUNT_RATE.value());
+                }
+            } else {
+                score = item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
+                        + item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value())
+                        + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value())
+                        + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
+            }
+            c.setScore(score);
+            item.setScore(score);
+            return item;
+        });
+
+        // 1 排序
+        Collections.sort(items, (o1, o2) -> -Double.compare(o1.getScore(), o2.getScore()));
+        // 2 相似去重
+        List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
+
+        // 3 文章按照内容池分组
+        Map<String, List<Content>> contentMap = new HashMap<>();
+        for (Content c : contents) {
+            List<Content> data = contentMap.computeIfAbsent(c.getContentPoolType(), k -> new ArrayList<>());
+            data.add(c);
+        }
+        // 4 选文章
+        String[] publishPool = Arrays.copyOf(contentPools, contentPools.length);
+
+        // 头
+        List<Content> pool1 = contentMap.get(contentPools[0]);
+        RankService.printSortLog(RankStrategyEnum.HIS_JUMP_STRATEGY.getStrategy(), param.getAccountName(), pool1);
+        if (CollectionUtils.isNotEmpty(pool1)) {
+            int i = RandomUtils.nextInt(0, Math.min(pool1.size(), 20));
+            result.add(pool1.get(i));
+        } else {
+            RankStrategy.sendFeishuFirstPoolEmpty(param, contentPools[0]);
+            return new RankResult(result);
+        }
+        // 次
+        List<Content> pool2 = contentMap.get(contentPools[1]);
+        if (CollectionUtils.isNotEmpty(pool2)) {
+            int i = RandomUtils.nextInt(0, Math.min(pool2.size(), 20));
+            result.add(pool2.get(i));
+        } else {
+            // 替补 根据设置替补内容池查找内容尽心替补
+            AccountIndexReplacePoolConfig replacePoolConfig = indexReplacePoolConfigMap.get(2);
+            if (Objects.nonNull(replacePoolConfig)) {
+                List<Content> pool2Replace = contentMap.get(replacePoolConfig.getContentPool());
+                if (CollectionUtils.isNotEmpty(pool2Replace)) {
+                    publishPool[1] = replacePoolConfig.getContentPool();
+                    int i = RandomUtils.nextInt(0, Math.min(pool2Replace.size(), 20));
+                    result.add(pool2Replace.get(i));
+                }
+            }
+        }
+
+        // 3-8
+        List<Content> pool = contentMap.get(contentPools[2]);
+        if (CollectionUtils.isNotEmpty(pool) && param.getSize() > result.size()) {
+            result.addAll(pool.subList(0, Math.min(pool.size(), param.getSize() - result.size())));
+        }
+
+        RankStrategy.deduplication(result, contentMap, publishPool);
+
+        return new RankResult(result);
+    }
+
+}

+ 3 - 1
long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/ScoreService.java

@@ -95,7 +95,8 @@ public class ScoreService implements ApplicationContextAware {
             return strategies;
         }
         strategies.add(strategyMap.get(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value()));
-        if (!similarityStopStrategies.contains(param.getStrategy())) {
+        if (!similarityStopStrategies.contains(param.getStrategy())
+                && !StringUtils.equals(param.getStrategy(), RankStrategyEnum.HIS_JUMP_STRATEGY.getStrategy())) {
             strategies.add(strategyMap.get(ScoreStrategyEnum.SIMILARITY.value()));
         }
         if (StringUtils.equals(param.getStrategy(), RankStrategyEnum.LATE_STRATEGY.getStrategy())
@@ -109,6 +110,7 @@ public class ScoreService implements ApplicationContextAware {
         strategies.add(strategyMap.get(ScoreStrategyEnum.VIEW_COUNT.value()));
         if (StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV3.getStrategy())
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV4.getStrategy())
+                || StringUtils.equals(param.getStrategy(), RankStrategyEnum.HIS_JUMP_STRATEGY.getStrategy())
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV5.getStrategy())
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV7.getStrategy())
                 || StringUtils.equals(param.getStrategy(), RankStrategyEnum.ArticleRankV8.getStrategy())