Merge branch 'wyp/1107-category' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/StatusEnum.java

@@ -5,8 +5,8 @@ import lombok.Getter;
 @Getter
 public enum StatusEnum {
 
-    FAIL(0, "fail"),
-    SUCCESS(1, "success"),
+    ZERO(0, "0"),
+    ONE(1, "1"),
     ;
 
     private int code;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/enums/recommend/ArticleCategoryStatusEnum.java

@@ -0,0 +1,31 @@
+package com.tzld.longarticle.recommend.server.common.enums.recommend;
+
+import lombok.Getter;
+
+@Getter
+public enum ArticleCategoryStatusEnum {
+
+    WAITING(0, "待分类"),
+    PROCESS(1, "执行中"),
+    SUCCESS(2, "成功"),
+    FAIL(3, "失败"),
+    ;
+
+    //状态（0-待分类 1-执行中 2-成功 3-失败）
+    private int code;
+    private String msg;
+
+    ArticleCategoryStatusEnum(int code, String msg) {
+        this.code = code;
+        this.msg = msg;
+    }
+
+    public static ArticleCategoryStatusEnum getByCode(int code) {
+        for (ArticleCategoryStatusEnum statusEnum : ArticleCategoryStatusEnum.values()) {
+            if (statusEnum.getCode() == code) {
+                return statusEnum;
+            }
+        }
+        return null;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/aigc/AigcBaseMapper.java

@@ -38,5 +38,9 @@ public interface AigcBaseMapper {
 
     CrawlerContent getCrawlerContentByChannelContentId(String channelContentId);
 
+    List<CrawlerContent> getCrawlerContentByChannelContentIdIn(List<String> channelContentIds);
+
     List<ProduceContentDTO> getProduceContentByPlanId(String planId);
+
+    List<ProducePlanExeRecord> getAllByProducePlanId(List<String> producePlanIds);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/mapper/longArticle/LongArticleBaseMapper.java

@@ -1,5 +1,6 @@
 package com.tzld.longarticle.recommend.server.mapper.longArticle;
 
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatScore;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticlePoolPromotionSource;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.DatastatSortStrategy;
@@ -21,4 +22,6 @@ public interface LongArticleBaseMapper {
     void updateRootProduceContentLevel(String rootProduceContentId, String level);
 
     void batchInsertDatastatScore(List<DatastatScore> list);
+
+    void batchInsertArticleCategory(List<ArticleCategory> list);
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/CrawlerContent.java

@@ -5,6 +5,7 @@ import lombok.Data;
 @Data
 public class CrawlerContent {
     private String channelContentId;
+    private String crawlerPlanId;
     private String ghId;
     private String title;
     private Long publishTimestamp;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/kimi/KimiOfficialApiResponse.java

@@ -0,0 +1,39 @@
+package com.tzld.longarticle.recommend.server.model.dto.kimi;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class KimiOfficialApiResponse {
+
+    private String id;
+    private String object;
+    private long created;
+    private String model;
+    private List<Choice> choices;
+    private Usage usage;
+
+
+    @Data
+    public static class Choice {
+        private long index;
+        private Message message;
+        private String finishReason;
+    }
+
+
+    @Data
+    public static class Message {
+        private String role;
+        private String content;
+    }
+
+    @Data
+    public static class Usage {
+        private long promptTokens;
+        private long completionTokens;
+        private long totalTokens;
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/dto/kimi/KimiResult.java

@@ -0,0 +1,11 @@
+package com.tzld.longarticle.recommend.server.model.dto.kimi;
+
+import lombok.Data;
+
+@Data
+public class KimiResult {
+    private boolean success;
+    private KimiOfficialApiResponse response;
+    private String failReason;
+    private String responseStr;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/AccountCategory.java

@@ -0,0 +1,48 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.*;
+import java.io.Serializable;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "account_category")
+@IdClass(AccountCategory.PK.class)
+public class AccountCategory {
+
+    @Id
+    private String dt;
+    @Id
+    private String ghId;
+
+    @Column(name = "category_map")
+    private String categoryMap;
+
+    @Column(name = "status")
+    private Integer status;
+
+    @Column(name = "create_timestamp")
+    private Long createTimestamp;
+
+    @Column(name = "update_timestamp")
+    private Long updateTimestamp;
+
+
+    @Data
+    public static class PK implements Serializable {
+
+        @Column(name = "dt")
+        private String dt;
+        @Column(name = "gh_id")
+        private String ghId;
+
+        public PK() {
+        }
+
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleCategory.java

@@ -0,0 +1,56 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "article_category")
+public class ArticleCategory {
+
+    @Id
+    @Column(name = "produce_content_id")
+    private String produceContentId;
+
+    @Column(name = "channel_content_id")
+    private String channelContentId;
+
+    @Column(name = "crawler_plan_id")
+    private String crawlerPlanId;
+
+    @Column(name = "title")
+    private String title;
+
+    @Column(name = "title_md5")
+    private String titleMd5;
+
+    @Column(name = "category")
+    private String category;
+
+    @Column(name = "kimi_result")
+    private String kimiResult;
+
+    @Column(name = "status")
+    private Integer status;
+
+    @Column(name = "fail_reason")
+    private String failReason;
+
+    @Column(name = "retry_times")
+    private Integer retryTimes;
+
+    @Column(name = "create_timestamp")
+    private Long createTimestamp;
+
+    @Column(name = "update_timestamp")
+    private Long updateTimestamp;
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/entity/longArticle/ArticleCrawlerPlan.java

@@ -0,0 +1,34 @@
+package com.tzld.longarticle.recommend.server.model.entity.longArticle;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Entity
+@Table(name = "article_crawler_plan")
+public class ArticleCrawlerPlan {
+
+    @Id
+    @Column(name = "crawler_plan_id")
+    private String crawlerPlanId;
+
+    @Column(name = "name")
+    private String name;
+
+    @Column(name = "status")
+    private Integer status;
+
+    @Column(name = "create_timestamp")
+    private Long createTimestamp;
+
+    @Column(name = "update_timestamp")
+    private Long updateTimestamp;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/vo/ProduceContentCrawlerVO.java

@@ -0,0 +1,10 @@
+package com.tzld.longarticle.recommend.server.model.vo;
+
+import lombok.Data;
+
+@Data
+public class ProduceContentCrawlerVO {
+    private String produceContentId;
+    private String channelContentId;
+    private String title;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/CrawlerContentByPlanService.java

@@ -0,0 +1,62 @@
+package com.tzld.longarticle.recommend.server.remote;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.tzld.longarticle.recommend.server.common.HttpPoolFactory;
+import com.tzld.longarticle.recommend.server.model.vo.ProduceContentCrawlerVO;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+import org.springframework.stereotype.Service;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+@Service
+@Slf4j
+public class CrawlerContentByPlanService {
+    private final CloseableHttpClient client = HttpPoolFactory.aigcPool();
+    private static final String url = "http://aigc-api.cybertogether.net/aigc/produce/content/getContentByCrawlerPlan";
+
+
+    public List<ProduceContentCrawlerVO> getCrawlerContentByPlan(String crawlerPlanId, List<String> producePlanIds) {
+        long start = System.currentTimeMillis();
+        List<ProduceContentCrawlerVO> result = new ArrayList<>();
+        JSONObject bodyParam = new JSONObject();
+        JSONObject bodyParamParams = new JSONObject();
+        bodyParamParams.put("crawlerPlanId", crawlerPlanId);
+        bodyParamParams.put("producePlanIds", producePlanIds);
+        bodyParam.put("params", bodyParamParams);
+        try {
+            HttpPost httpPost = new HttpPost(url);
+            StringEntity stringEntity = new StringEntity(bodyParam.toJSONString(), StandardCharsets.UTF_8);
+            httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
+            httpPost.setEntity(stringEntity);
+            CloseableHttpResponse response = client.execute(httpPost);
+            StatusLine statusLine = response.getStatusLine();
+            if (statusLine.getStatusCode() == 200) {
+                HttpEntity responseEntity = response.getEntity();
+                if (Objects.nonNull(responseEntity)) {
+                    String responseBody = EntityUtils.toString(responseEntity, "UTF-8");
+                    JSONObject jsonObject = JSONObject.parseObject(responseBody);
+                    if (jsonObject.getInteger("code") == 0) {
+                        JSONArray data = jsonObject.getJSONArray("data");
+                        result.addAll(JSONArray.parseArray(data.toJSONString(), ProduceContentCrawlerVO.class));
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("getCrawlerContentByPlan error", e);
+        }
+        log.info("getCrawlerContentByPlan耗时：{}", System.currentTimeMillis() - start);
+        return result;
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/KimiApiService.java

@@ -0,0 +1,89 @@
+package com.tzld.longarticle.recommend.server.remote;
+
+import cn.hutool.core.collection.CollectionUtil;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.tzld.longarticle.recommend.server.model.dto.kimi.KimiOfficialApiResponse;
+import com.tzld.longarticle.recommend.server.model.dto.kimi.KimiResult;
+import com.tzld.longarticle.recommend.server.util.MapBuilder;
+import lombok.extern.slf4j.Slf4j;
+import okhttp3.*;
+import org.apache.http.util.TextUtils;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+@Service
+@Slf4j
+public class KimiApiService {
+
+    private OkHttpClient client;
+
+    @PostConstruct
+    public void init() {
+        client = new OkHttpClient().newBuilder()
+                .connectTimeout(15, TimeUnit.MINUTES)
+                .readTimeout(15, TimeUnit.MINUTES)
+                .writeTimeout(15, TimeUnit.MINUTES)
+                .build();
+
+    }
+
+
+    public KimiResult requestOfficialApi(String prompt, String model, Double temperature) {
+        KimiResult result = new KimiResult();
+        result.setSuccess(false);
+        if (TextUtils.isBlank(prompt) || TextUtils.isBlank(prompt.trim())) {
+            result.setFailReason("prompt is empty");
+            return result;
+        }
+
+        try {
+            JSONArray jsonArray = new JSONArray();
+            JSONObject message = new JSONObject();
+            message.put("role", "user");
+            message.put("content", prompt);
+            jsonArray.add(message);
+
+            Map<Object, Object> bodyParam = MapBuilder
+                    .builder()
+                    .put("model", Optional.ofNullable(model).orElse("moonshot-v1-auto"))
+                    .put("temperature", Optional.ofNullable(temperature).orElse(0.3))
+                    .put("messages", jsonArray)
+                    .build();
+
+            MediaType mediaType = MediaType.parse("application/json");
+            RequestBody body = RequestBody.create(mediaType, JSONObject.toJSONString(bodyParam));
+            Request request = new Request.Builder()
+                    .url("https://api.moonshot.cn/v1/chat/completions")
+                    .method("POST", body)
+                    .addHeader("Content-Type", "application/json")
+                    .addHeader("Authorization", "Bearer sk-5DqYCa88kche6nwIWjLE1p4oMm8nXrR9kQMKbBolNAWERu7q")
+                    .build();
+            Response response = client.newCall(request).execute();
+
+            String responseContent = response.body().string();
+            result.setResponseStr(responseContent);
+            log.info("kimi api responseContent = {}", responseContent);
+            if (response.isSuccessful()) {
+                KimiOfficialApiResponse obj = JSONObject.parseObject(responseContent, KimiOfficialApiResponse.class);
+                if (CollectionUtil.isNotEmpty(obj.getChoices())) {
+                    result.setSuccess(true);
+                    result.setResponse(obj);
+                } else {
+                    result.setFailReason("response empty");
+                }
+            } else {
+                JSONObject json = JSONObject.parseObject(responseContent);
+                result.setFailReason("request error code:" + response.code() + " message:" + json.getString("error"));
+            }
+        } catch (Exception e) {
+            log.error("kimi official api fail: " + e.getMessage());
+            result.setFailReason(e.getMessage());
+        }
+        return result;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/AccountCategoryRepository.java

@@ -0,0 +1,12 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.AccountCategory;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface AccountCategoryRepository extends JpaRepository<AccountCategory, AccountCategory.PK> {
+
+    AccountCategory getByGhIdAndStatus(String ghId, Integer status);
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/ArticleCategoryRepository.java

@@ -0,0 +1,19 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+@Repository
+public interface ArticleCategoryRepository extends JpaRepository<ArticleCategory, String> {
+
+    List<ArticleCategory> getByProduceContentIdIn(List<String> produceContentIds);
+
+    List<ArticleCategory> getByStatus(Integer status);
+
+    List<ArticleCategory> getAllByChannelContentIdIn(List<String> channelContentIds);
+
+    List<ArticleCategory> getByStatusAndRetryTimesLessThan(Integer status, Integer retryTimes);
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/ArticleCrawlerPlanRepository.java

@@ -0,0 +1,14 @@
+package com.tzld.longarticle.recommend.server.repository.longArticle;
+
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCrawlerPlan;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+@Repository
+public interface ArticleCrawlerPlanRepository extends JpaRepository<ArticleCrawlerPlan, String> {
+
+    List<ArticleCrawlerPlan> getByStatus(Integer status);
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/ArticlePoolPromotionSourceRepository.java

@@ -11,6 +11,8 @@ public interface ArticlePoolPromotionSourceRepository extends JpaRepository<Arti
 
     ArticlePoolPromotionSource getByChannelContentId(String channelContentId);
 
+    List<ArticlePoolPromotionSource> getByChannelContentIdIn(List<String> channelContentIds);
+
     List<ArticlePoolPromotionSource> getByChannelContentIdInAndStatusAndDeleted(List<String> channelContentIds, Integer status, Integer deleted);
 
     List<ArticlePoolPromotionSource> getByStatusAndDeleted(Integer status, Integer deleted);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/XxlJobService.java

@@ -391,4 +391,16 @@ public class XxlJobService {
         articleService.articlePromotionTraceability(channelContentId);
         return ReturnT.SUCCESS;
     }
+
+    @XxlJob("articleCategoryJob")
+    public ReturnT<String> articleCategoryJob(String param) {
+        articleService.articleCategory();
+        return ReturnT.SUCCESS;
+    }
+
+    @XxlJob("articleCategoryJobRetry")
+    public ReturnT<String> articleCategoryJobRetry(String param) {
+        articleService.articleCategoryJobRetry();
+        return ReturnT.SUCCESS;
+    }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleService.java

@@ -1,28 +1,41 @@
 package com.tzld.longarticle.recommend.server.service.recommend;
 
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.collect.Lists;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.tzld.longarticle.recommend.server.common.CommonThreadPoolExecutor;
 import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.aigc.PublishContentTypeEnum;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleCategoryStatusEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticlePoolPromotionSourceStatusEnum;
 import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.crawler.CrawlerBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
 import com.tzld.longarticle.recommend.server.model.dto.CrawlerContent;
+import com.tzld.longarticle.recommend.server.model.dto.kimi.KimiResult;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishAccount;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishContent;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.PublishContentOutput;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCrawlerPlan;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticlePoolPromotionSource;
 import com.tzld.longarticle.recommend.server.model.param.ArticleFindSourceParam;
+import com.tzld.longarticle.recommend.server.model.vo.ProduceContentCrawlerVO;
 import com.tzld.longarticle.recommend.server.model.vo.RootPublishContentVO;
+import com.tzld.longarticle.recommend.server.remote.CrawlerContentByPlanService;
+import com.tzld.longarticle.recommend.server.remote.KimiApiService;
 import com.tzld.longarticle.recommend.server.repository.aigc.PublishAccountRepository;
 import com.tzld.longarticle.recommend.server.repository.aigc.PublishContentOutputRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.AccountAvgInfoRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
+import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleCategoryRepository;
+import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleCrawlerPlanRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.ArticlePoolPromotionSourceRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountIndexAvgViewCountService;
 import com.tzld.longarticle.recommend.server.util.DateUtils;
+import com.tzld.longarticle.recommend.server.util.Md5Util;
 import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
@@ -31,10 +44,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
 import java.util.concurrent.*;
 import java.util.stream.Collectors;
 
@@ -63,6 +73,19 @@ public class ArticleService {
     LongArticleBaseMapper longArticleBaseMapper;
     @Autowired
     ArticlePoolPromotionSourceRepository articlePoolPromotionSourceRepository;
+    @Autowired
+    ArticleCrawlerPlanRepository articleCrawlerPlanRepository;
+    @Autowired
+    ArticleCategoryRepository articleCategoryRepository;
+    @Autowired
+    CrawlerContentByPlanService crawlerContentByPlanService;
+    @Autowired
+    KimiApiService kimiApiService;
+
+    @ApolloJsonValue("${cold.pool.produce.planId:[\"20240802021606053813696\", \"20240802080355355308981\",\n" +
+            "\"20240805154433785506170\", \"20240805154359027876170\", \"20241024100016206421084\", " +
+            "\"20241030070010871546586\"]}")
+    private static List<String> producePlanIds;
 
     private final static ExecutorService pool = new CommonThreadPoolExecutor(
             32,
@@ -262,7 +285,7 @@ public class ArticleService {
             // 溯源
             Article article = articleRepository.getByWxSn(task.getWxSn());
             if (Objects.isNull(article)) {
-                task.setDeleted(StatusEnum.SUCCESS.getCode());
+                task.setDeleted(StatusEnum.ONE.getCode());
                 articlePoolPromotionSourceRepository.save(task);
                 continue;
             }
@@ -274,7 +297,7 @@ public class ArticleService {
             List<PublishContent> publishContentList = aigcBaseMapper.getNearestPublishContent(publishAccount.getId(), publishTimestamp, 100);
             PublishContent publishContent = findPublishContent(publishContentList, task.getTitle(), publishTimestamp);
             if (Objects.isNull(publishContent)) {
-                task.setDeleted(StatusEnum.SUCCESS.getCode());
+                task.setDeleted(StatusEnum.ONE.getCode());
                 articlePoolPromotionSourceRepository.save(task);
                 continue;
             }
@@ -309,9 +332,145 @@ public class ArticleService {
                 }
                 longArticleBaseMapper.updateRootProduceContentLevel(task.getRootProduceContentId(), task.getLevel());
             } else {
-                task.setDeleted(StatusEnum.SUCCESS.getCode());
+                task.setDeleted(StatusEnum.ONE.getCode());
                 articlePoolPromotionSourceRepository.save(task);
             }
         }
     }
+
+
+    public void articleCategory() {
+        List<ArticleCrawlerPlan> articleCrawlerPlanList = articleCrawlerPlanRepository.getByStatus(StatusEnum.ZERO.getCode());
+        for (ArticleCrawlerPlan crawlerPlan : articleCrawlerPlanList) {
+            List<ProduceContentCrawlerVO> list = crawlerContentByPlanService.getCrawlerContentByPlan(crawlerPlan.getCrawlerPlanId(), producePlanIds);
+            List<String> produceContentIds = list.stream().map(ProduceContentCrawlerVO::getProduceContentId).collect(Collectors.toList());
+            List<ArticleCategory> exists = articleCategoryRepository.getByProduceContentIdIn(produceContentIds);
+            List<String> existsIds = exists.stream().map(ArticleCategory::getProduceContentId).collect(Collectors.toList());
+            list = list.stream().filter(o -> !existsIds.contains(o.getProduceContentId())).collect(Collectors.toList());
+            long now = System.currentTimeMillis();
+            List<ArticleCategory> saveList = new ArrayList<>();
+            for (ProduceContentCrawlerVO vo : list) {
+                ArticleCategory item = new ArticleCategory();
+                item.setCrawlerPlanId(crawlerPlan.getCrawlerPlanId());
+                item.setChannelContentId(vo.getChannelContentId());
+                item.setProduceContentId(vo.getProduceContentId());
+                item.setTitle(vo.getTitle());
+                item.setTitleMd5(Md5Util.encoderByMd5(vo.getTitle()));
+                item.setCreateTimestamp(now);
+                saveList.add(item);
+            }
+            if (CollectionUtils.isNotEmpty(saveList)) {
+                longArticleBaseMapper.batchInsertArticleCategory(saveList);
+            }
+            // 抓取计划超过5天设置为已处理
+            String dateStr = crawlerPlan.getCrawlerPlanId().substring(0, 8);
+            if (DateUtils.dateStrToTimestamp(dateStr, "yyyyMMdd") < now - 86400 * 5) {
+                crawlerPlan.setStatus(StatusEnum.ONE.getCode());
+                crawlerPlan.setUpdateTimestamp(now);
+                articleCrawlerPlanRepository.save(crawlerPlan);
+            }
+        }
+        List<ArticleCategory> dealList = articleCategoryRepository.getByStatus(ArticleCategoryStatusEnum.WAITING.getCode());
+        List<List<ArticleCategory>> partitionList = Lists.partition(dealList, 20);
+        for (List<ArticleCategory> partition : partitionList) {
+            List<String> partitionTitles = partition.stream().map(ArticleCategory::getTitle).distinct().collect(Collectors.toList());
+            String prompt = buildKimiPrompt(partitionTitles);
+            KimiResult kimiResult = kimiApiService.requestOfficialApi(prompt, null, null);
+            long now = System.currentTimeMillis();
+            JSONObject obj = null;
+            if (kimiResult.isSuccess()) {
+                try {
+                    obj = JSONObject.parseObject(kimiResult.getResponse().getChoices().get(0).getMessage().getContent());
+                } catch (Exception e) {
+                    log.error(kimiResult.getResponse().getChoices().get(0).getMessage().getContent());
+                }
+            }
+            for (ArticleCategory articleCategory : partition) {
+                articleCategory.setKimiResult(kimiResult.getResponseStr());
+                articleCategory.setUpdateTimestamp(now);
+                if (kimiResult.isSuccess() && Objects.nonNull(obj) && obj.containsKey(articleCategory.getTitle())) {
+                    articleCategory.setCategory(obj.getString(articleCategory.getTitle()));
+                    articleCategory.setStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
+                } else {
+                    articleCategory.setStatus(ArticleCategoryStatusEnum.FAIL.getCode());
+                    articleCategory.setFailReason(kimiResult.getFailReason());
+                }
+                articleCategoryRepository.save(articleCategory);
+            }
+        }
+
+    }
+
+    private String buildKimiPrompt(List<String> titleList) {
+        StringBuilder prompt = new StringBuilder(
+                "请帮我完成以下任务：输入为文章的标题，根据标题判断其内容所属的类目，输出为文章标题及其对应的类目。\n" +
+                "类目需从以下范围内选择：\n" +
+                "奇闻趣事\n" +
+                "历史人物\n" +
+                "家长里短\n" +
+                "温情故事\n" +
+                "健康养生\n" +
+                "生活知识\n" +
+                "名人八卦\n" +
+                "政治新闻\n" +
+                "军事新闻\n" +
+                "为了更好地完成任务，可参考下列对文章标题的分类：\n" +
+                "{" +
+                "\"大舅病了，我取了三万元送过去，病房门口听到舅妈的话我改了主意\": \"家长里短\",\n" +
+                "\"能活到90岁的老人，基本上在70岁的时候，就不再做这些事了！\": \"健康养生\",\n" +
+                "\"去医院看望病人时，切忌带这4样东西，再亲近也不行，这是做人的根本\": \"生活知识\",\n" +
+                "\"上海一女子去饭店吃生煎包，戳了个洞想凉一凉，往里一看，瞬间惊呆了\": \"奇闻趣事\",\n" +
+                "\"卫生间放一把食盐，一年能省下好几百，涨新知识\": \"生活知识\",\n" +
+                "\"中国有一古寺，庙不大，却有武警24小时站岗，到底有何“过人”之处\": \"奇闻趣事\",\n" +
+                "\"1974年，苏联外长故意拿邓小平身高“取笑”，邓小平一句话轻松反击\": \"历史人物\",\n" +
+                "\"中国最美的女将军：上世纪曾家喻户晓，如今仍然健在\": \"历史人物\",\n" +
+                "\"北大才女蒙曼48岁仍未婚，被问最想嫁给谁，一个名字让全场笑喷\": \"名人八卦\",\n" +
+                "\"广东一老人去世，家人把老人的旧床垫扔了，环卫工人看到后，竟发现里面藏了15万元现金！家人傻眼了\": \"奇闻趣事\",\n" +
+                "}" +
+                "最后输出结果请用JSON格式输出，key为title，value为类目，仅输出JSON，不要markdown格式，不要任何其他内容，" +
+                "并且内容可以被 fastJSON 的JSONObject.parseObject转换为JSON对象\n" +
+                "当标题的开头或结尾为以下字符时“”“，则在标题的开头或结尾增加\" " +
+                "输出结果格式如下：\n" +
+                "{" +
+                "\"浙江老人用“假钱”吃霸王餐9年，离世后，老板却崩溃大哭：“每天都在等他！”\": \"奇闻趣事\"," +
+                "\"“最美婴儿”迅速走红，像在娘胎里整过容，网友：看到第一眼就想抱回家\": \"奇闻趣事\"" +
+                "}" +
+                "以下是需要分析的文章标题列表，每一行是一个标题：\n");
+        for (String title : titleList) {
+            prompt.append(title).append("\n");
+        }
+        return prompt.toString();
+    }
+
+    public void articleCategoryJobRetry() {
+        List<ArticleCategory> dealList = articleCategoryRepository.getByStatusAndRetryTimesLessThan(ArticleCategoryStatusEnum.FAIL.getCode(), 3);
+        for (ArticleCategory articleCategory : dealList) {
+            List<String> partitionTitles = Collections.singletonList(articleCategory.getTitle());
+            String prompt = buildKimiPrompt(partitionTitles);
+            KimiResult kimiResult = kimiApiService.requestOfficialApi(prompt, null, null);
+            long now = System.currentTimeMillis();
+            JSONObject obj = null;
+            if (kimiResult.isSuccess()) {
+                try {
+                    obj = JSONObject.parseObject(kimiResult.getResponse().getChoices().get(0).getMessage().getContent());
+                } catch (Exception e) {
+                    log.error(kimiResult.getResponse().getChoices().get(0).getMessage().getContent());
+                }
+            }
+            articleCategory.setKimiResult(kimiResult.getResponseStr());
+            articleCategory.setUpdateTimestamp(now);
+            articleCategory.setRetryTimes(articleCategory.getRetryTimes() + 1);
+            if (kimiResult.isSuccess() && Objects.nonNull(obj)) {
+                List<String> keys = new ArrayList<>(obj.keySet());
+                String category = obj.getString(keys.get(0));
+                articleCategory.setCategory(category);
+                articleCategory.setStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
+                articleCategory.setFailReason(null);
+            } else {
+                articleCategory.setStatus(ArticleCategoryStatusEnum.FAIL.getCode());
+                articleCategory.setFailReason(kimiResult.getFailReason());
+            }
+            articleCategoryRepository.save(articleCategory);
+        }
+    }
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/InfiniteRankStrategy.java

@@ -3,6 +3,7 @@ package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
+import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
@@ -27,6 +28,8 @@ public class InfiniteRankStrategy implements RankStrategy {
     private ScoreService scoreService;
     @Autowired
     private AccountContentPoolConfigService accountContentPoolConfigService;
+    @Autowired
+    private StrategyIndexScoreWeightService weightService;
 
     public RankResult rank(RankParam param) {
         List<Content> result = new ArrayList<>();
@@ -44,9 +47,12 @@ public class InfiniteRankStrategy implements RankStrategy {
             c.setScoreMap(scoreMap.get(c.getId()));
             item.setScoreMap(scoreMap.get(c.getId()));
             double score = 0.0;
+            int index = weightService.getIndex(item.getContent().getContentPoolType(), contentPools);
             if (contentPools[2].equals(item.getContent().getContentPoolType())) {
                 score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
                         + item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value())
                         + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value())

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/LateRankStrategy.java

@@ -3,6 +3,7 @@ package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
+import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
@@ -27,6 +28,8 @@ public class LateRankStrategy implements RankStrategy {
     private ScoreService scoreService;
     @Autowired
     private AccountContentPoolConfigService accountContentPoolConfigService;
+    @Autowired
+    private StrategyIndexScoreWeightService weightService;
 
     public RankResult rank(RankParam param) {
         List<Content> result = new ArrayList<>();
@@ -44,9 +47,12 @@ public class LateRankStrategy implements RankStrategy {
             c.setScoreMap(scoreMap.get(c.getId()));
             item.setScoreMap(scoreMap.get(c.getId()));
             double score = 0.0;
+            int index = weightService.getIndex(item.getContent().getContentPoolType(), contentPools);
             if (contentPools[2].equals(item.getContent().getContentPoolType())) {
                 score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
                         + item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value())
                         + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value())

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV10Strategy.java

@@ -6,6 +6,7 @@ import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
+import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
 import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
@@ -35,6 +36,8 @@ public class RankV10Strategy implements RankStrategy {
     private AccountContentPoolConfigService accountContentPoolConfigService;
     @Autowired
     private ArticleRepository articleRepository;
+    @Autowired
+    private StrategyIndexScoreWeightService weightService;
 
     public RankResult rank(RankParam param) {
         List<Content> result = new ArrayList<>();
@@ -51,11 +54,14 @@ public class RankV10Strategy implements RankStrategy {
             c.setScoreMap(scoreMap.get(c.getId()));
             item.setScoreMap(scoreMap.get(c.getId()));
             double score;
+            int index = weightService.getIndex(item.getContent().getContentPoolType(), contentPools);
             if (contentPools[0].equals(item.getContent().getContentPoolType())) {
                 score = item.getScore(ScoreStrategyEnum.HIS_FISSION_AVG_READ_SUM_RATE.value());
             } else if (contentPools[1].equals(item.getContent().getContentPoolType())) {
                 score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
                 if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
                     score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value());
@@ -63,6 +69,8 @@ public class RankV10Strategy implements RankStrategy {
             } else {
                 score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
                         + item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value())
                         + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value())

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV11Strategy.java

@@ -70,6 +70,8 @@ public class RankV11Strategy implements RankStrategy {
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.SIMILARITY.value()))
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
                 if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
                     score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
@@ -81,6 +83,8 @@ public class RankV11Strategy implements RankStrategy {
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.SIMILARITY.value()))
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + (item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value()))

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV12Strategy.java

@@ -70,6 +70,8 @@ public class RankV12Strategy implements RankStrategy {
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.SIMILARITY.value()))
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
                 if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
                     score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
@@ -81,6 +83,8 @@ public class RankV12Strategy implements RankStrategy {
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.SIMILARITY.value()))
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + (item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value()))

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV13Strategy.java

@@ -71,6 +71,8 @@ public class RankV13Strategy implements RankStrategy {
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.SIMILARITY.value()))
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
                 if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
                     score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
@@ -82,6 +84,8 @@ public class RankV13Strategy implements RankStrategy {
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.SIMILARITY.value()))
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + (item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value()))

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV14Strategy.java

@@ -71,6 +71,8 @@ public class RankV14Strategy implements RankStrategy {
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.SIMILARITY.value()))
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
                 if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
                     score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
@@ -82,6 +84,8 @@ public class RankV14Strategy implements RankStrategy {
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.SIMILARITY.value()))
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + (item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value()))

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV5Strategy.java

@@ -63,6 +63,8 @@ public class RankV5Strategy implements RankStrategy {
                         * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
                         ScoreStrategyEnum.SIMILARITY.value())
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
                 if (item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value()) >= 0) {
                     score += item.getScore(ScoreStrategyEnum.VIEW_COUNT_RATE.value())
@@ -104,9 +106,7 @@ public class RankV5Strategy implements RankStrategy {
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
-        if (touliuAccountGhIds.contains(param.getGhId())) {
-            RankService.printSortLog(RankStrategyEnum.ArticleRankV5.getStrategy(), param.getAccountName(), pool1);
-        }
+        RankService.printSortLog(RankStrategyEnum.ArticleRankV5.getStrategy(), param.getAccountName(), pool1);
         if (CollectionUtils.isNotEmpty(pool1)) {
             result.add(pool1.get(0));
         } else {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/rank/strategy/RankV8Strategy.java

@@ -1,15 +1,14 @@
 package com.tzld.longarticle.recommend.server.service.recommend.rank.strategy;
 
 
+import com.tzld.longarticle.recommend.server.common.enums.recommend.RankStrategyEnum;
 import com.tzld.longarticle.recommend.server.common.enums.recommend.ScoreStrategyEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankItem;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankParam;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankResult;
-import com.tzld.longarticle.recommend.server.service.recommend.rank.RankStrategy;
+import com.tzld.longarticle.recommend.server.service.recommend.config.StrategyIndexScoreWeightService;
+import com.tzld.longarticle.recommend.server.service.recommend.rank.*;
 import com.tzld.longarticle.recommend.server.service.recommend.score.AccountIndexReplacePoolConfig;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreResult;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreService;
@@ -35,6 +34,8 @@ public class RankV8Strategy implements RankStrategy {
     private AccountContentPoolConfigService accountContentPoolConfigService;
     @Autowired
     private ArticleRepository articleRepository;
+    @Autowired
+    private StrategyIndexScoreWeightService weightService;
 
     public RankResult rank(RankParam param) {
         List<Content> result = new ArrayList<>();
@@ -51,9 +52,12 @@ public class RankV8Strategy implements RankStrategy {
             c.setScoreMap(scoreMap.get(c.getId()));
             item.setScoreMap(scoreMap.get(c.getId()));
             double score;
+            int index = weightService.getIndex(item.getContent().getContentPoolType(), contentPools);
             if (contentPools[0].equals(item.getContent().getContentPoolType())) {
                 score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.FLOW_CTL_DECREASE.value());
                 score += item.getScore(ScoreStrategyEnum.HIS_FISSION_AVG_READ_RATE_RATE.value());
             } else if (contentPools[1].equals(item.getContent().getContentPoolType())) {
@@ -66,6 +70,8 @@ public class RankV8Strategy implements RankStrategy {
             } else {
                 score = item.getScore(ScoreStrategyEnum.SIMILARITY.value())
                         + item.getScore(ScoreStrategyEnum.CATEGORY.value())
+                        * weightService.getWeight(param.getStrategy(), param.getGhId(), index,
+                        ScoreStrategyEnum.CATEGORY.value())
                         + item.getScore(ScoreStrategyEnum.ACCOUNT_PRE_DISTRIBUTE.value())
                         + item.getScore(ScoreStrategyEnum.PUBLISH_TIMES.value())
                         + item.getScore(ScoreStrategyEnum.CRAWLER_DAYS_DECREASE_STRATEGY.value())
@@ -98,6 +104,7 @@ public class RankV8Strategy implements RankStrategy {
 
         // 头
         List<Content> pool1 = contentMap.get(contentPools[0]);
+        RankService.printSortLog(RankStrategyEnum.ArticleRankV8.getStrategy(), param.getAccountName(), pool1);
         if (CollectionUtils.isNotEmpty(pool1)) {
             result.add(pool1.get(0));
         } else {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/recall/RecallService.java

@@ -12,12 +12,16 @@ import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountAvgInfo
 import com.tzld.longarticle.recommend.server.model.entity.crawler.AccountCorrelation;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.Article;
 import com.tzld.longarticle.recommend.server.model.entity.crawler.ArticleDetailInfo;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticlePoolPromotionSource;
 import com.tzld.longarticle.recommend.server.remote.AIGCRemoteService;
 import com.tzld.longarticle.recommend.server.repository.aigc.CrawlerMetaArticleRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.AccountAvgInfoRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.AccountCorrelationRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleDetailInfoRepository;
 import com.tzld.longarticle.recommend.server.repository.crawler.ArticleRepository;
+import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleCategoryRepository;
+import com.tzld.longarticle.recommend.server.repository.longArticle.ArticlePoolPromotionSourceRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountIndexAvgViewCountService;
 import com.tzld.longarticle.recommend.server.service.recommend.recall.strategy.DefaultRecallStrategy;
 import com.tzld.longarticle.recommend.server.service.recommend.score.ScoreStrategy;
@@ -42,6 +46,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 /**
@@ -67,6 +72,10 @@ public class RecallService implements ApplicationContextAware {
     CrawlerBaseMapper crawlerBaseMapper;
     @Autowired
     AccountCorrelationRepository accountCorrelationRepository;
+    @Autowired
+    ArticlePoolPromotionSourceRepository articlePoolPromotionSourceRepository;
+    @Autowired
+    ArticleCategoryRepository articleCategoryRepository;
 
     private final Map<String, RecallStrategy> strategyMap = new HashMap<>();
     private ApplicationContext applicationContext;
@@ -157,25 +166,42 @@ public class RecallService implements ApplicationContextAware {
     }
 
     public void setContentCategory(List<Content> contentList) {
-        long start = System.currentTimeMillis();
-        Map<String, String> articleMd5Map = new HashMap<>();
-        List<String> md5List = new ArrayList<>();
-        for (Content content : contentList) {
-            String md5 = generateArticleUniqueMd5(content.getCrawlerLink());
-            md5List.add(md5);
-            articleMd5Map.put(content.getId(), md5);
-        }
-        List<CrawlerMetaArticle> categoryList = getByUniqueIndexIn(md5List);
-        if (CollectionUtils.isEmpty(categoryList)) {
-            return;
-        }
-        Map<String, List<String>> categoryMap = categoryList.stream().collect(Collectors.groupingBy(CrawlerMetaArticle::getUniqueIndex,
-                Collectors.mapping(CrawlerMetaArticle::getCategory, Collectors.toList())));
+//        Map<String, String> articleMd5Map = new HashMap<>();
+//        List<String> md5List = new ArrayList<>();
+//        for (Content content : contentList) {
+//            String md5 = generateArticleUniqueMd5(content.getCrawlerLink());
+//            md5List.add(md5);
+//            articleMd5Map.put(content.getId(), md5);
+//        }
+//        List<CrawlerMetaArticle> categoryList = getByUniqueIndexIn(md5List);
+//        if (CollectionUtils.isEmpty(categoryList)) {
+//            return;
+//        }
+//        Map<String, List<String>> categoryMap = categoryList.stream().collect(Collectors.groupingBy(CrawlerMetaArticle::getUniqueIndex,
+//                Collectors.mapping(CrawlerMetaArticle::getCategory, Collectors.toList())));
+//        for (Content content : contentList) {
+//            String md5 = articleMd5Map.get(content.getId());
+//            content.setCategory(categoryMap.get(md5));
+//        }
+        List<String> channelContentIds = contentList.stream().map(Content::getCrawlerChannelContentId).collect(Collectors.toList());
+        // 查询晋升rootProduceContentId
+        List<ArticlePoolPromotionSource> sourceList = articlePoolPromotionSourceRepository.getByChannelContentIdIn(channelContentIds);
+        Map<String, ArticlePoolPromotionSource> sourceMap = sourceList.stream().collect(Collectors.toMap(ArticlePoolPromotionSource::getChannelContentId, Function.identity()));
+        List<String> produceContentIds = sourceMap.values().stream().map(ArticlePoolPromotionSource::getRootProduceContentId).collect(Collectors.toList());
+        // 根据produceContentId查询category
+        List<ArticleCategory> articleCategoryList = articleCategoryRepository.getByProduceContentIdIn(produceContentIds);
+        Map<String, ArticleCategory> categoryMap = articleCategoryList.stream().collect(Collectors.toMap(ArticleCategory::getProduceContentId, Function.identity()));
         for (Content content : contentList) {
-            String md5 = articleMd5Map.get(content.getId());
-            content.setCategory(categoryMap.get(md5));
+            ArticlePoolPromotionSource source = sourceMap.get(content.getCrawlerChannelContentId());
+            if (Objects.nonNull(source) && Objects.nonNull(source.getRootProduceContentId())) {
+                ArticleCategory category = categoryMap.get(source.getRootProduceContentId());
+                if (Objects.nonNull(category)) {
+                    content.setCategory(Collections.singletonList(category.getCategory()));
+                    continue;
+                }
+            }
+            log.error("setContentCategory NullError channelContentId:{}", content.getCrawlerChannelContentId());
         }
-        log.info("setContentCategory cost:{}", System.currentTimeMillis() - start);
     }
 
     private List<CrawlerMetaArticle> getByUniqueIndexIn(List<String> md5List) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/score/strategy/CategoryStrategy.java

@@ -1,8 +1,12 @@
 package com.tzld.longarticle.recommend.server.service.recommend.score.strategy;
 
+import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.common.enums.StatusEnum;
 import com.tzld.longarticle.recommend.server.model.dto.Content;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.AccountCategory;
 import com.tzld.longarticle.recommend.server.repository.aigc.CrawlerMetaArticleRepository;
+import com.tzld.longarticle.recommend.server.repository.longArticle.AccountCategoryRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.config.AccountContentPoolConfigService;
 import com.tzld.longarticle.recommend.server.service.recommend.score.AccountCategoryWeightConfig;
 import com.tzld.longarticle.recommend.server.service.recommend.score.Score;
@@ -12,9 +16,9 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
 
 import java.util.*;
-import java.util.stream.Collectors;
 
 @Component
 @Slf4j
@@ -24,6 +28,8 @@ public class CategoryStrategy implements ScoreStrategy {
     CrawlerMetaArticleRepository crawlerMetaArticleRepository;
     @Autowired
     AccountContentPoolConfigService accountContentPoolConfigService;
+    @Autowired
+    AccountCategoryRepository accountCategoryRepository;
 
     @ApolloJsonValue("${accountCategoryWeightConfig:{}}")
     private Map<String, AccountCategoryWeightConfig[]> accountCategoryWeightConfigMap;
@@ -36,12 +42,11 @@ public class CategoryStrategy implements ScoreStrategy {
         if (CollectionUtils.isEmpty(param.getContents())) {
             return scores;
         }
-        AccountCategoryWeightConfig[] categoryWeightConfigList = accountCategoryWeightConfigMap.get(param.getAccountName());
-        if (Objects.isNull(categoryWeightConfigList) || categoryWeightConfigList.length == 0) {
+        AccountCategory accountCategory = accountCategoryRepository.getByGhIdAndStatus(param.getGhId(), StatusEnum.ONE.getCode());
+        if (Objects.isNull(accountCategory) || !StringUtils.hasText(accountCategory.getCategoryMap())) {
             return scores;
         }
-        Map<Integer, AccountCategoryWeightConfig> categoryWeightConfigMap = Arrays.stream(categoryWeightConfigList).collect(Collectors.toMap(AccountCategoryWeightConfig::getIndex, o -> o));
-        String[] contentPools = accountContentPoolConfigService.getContentPools(param.getAccountName());
+        JSONObject categoryWeightMap = JSONObject.parseObject(accountCategory.getCategoryMap());
         for (Content content : param.getContents()) {
             if (CollectionUtils.isEmpty(content.getCategory())) {
                 continue;
@@ -49,19 +54,14 @@ public class CategoryStrategy implements ScoreStrategy {
             Score score = new Score();
             score.setStrategy(this);
             score.setContentId(content.getId());
-            for (int i = 0; i < contentPools.length; i++) {
-                if (contentPools[i].equals(content.getContentPoolType())) {
-                    AccountCategoryWeightConfig categoryWeightConfig = categoryWeightConfigMap.get(i+1);
-                    if (Objects.nonNull(categoryWeightConfig)) {
-                        List<AccountCategoryWeightConfig.CategoryWeight> categoryWeightList = categoryWeightConfig.getCategoryWeightList();
-                        for (AccountCategoryWeightConfig.CategoryWeight categoryWeight : categoryWeightList) {
-                            if (content.getCategory().contains(categoryWeight.getCategory())) {
-                                score.setScore(Double.valueOf(categoryWeight.getWeight()));
-                            }
-                        }
-                    }
+            double scoreValue = 0.0;
+            for (String category : content.getCategory()) {
+                if (!categoryWeightMap.containsKey(category)) {
+                    continue;
                 }
+                scoreValue += categoryWeightMap.getDoubleValue(category);
             }
+            score.setScore(scoreValue);
             scores.add(score);
         }
         log.info("CategoryStrategy cost:{}", System.currentTimeMillis() - start);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/web/XxlJobController.java

@@ -48,4 +48,15 @@ public class XxlJobController {
     public void articlePromotionTraceability(String channelContentId) {
         service.articlePromotionTraceability(channelContentId);
     }
+
+    @GetMapping("/articleCategoryJob")
+    public void articleCategoryJob() {
+        service.articleCategoryJob(null);
+    }
+
+    @GetMapping("/articleCategoryJobRetry")
+    public void articleCategoryJobRetry() {
+        service.articleCategoryJobRetry(null);
+    }
+
 }

long-article-recommend-service/src/main/resources/mapper/aigc/AigcBaseMapper.xml

@@ -176,6 +176,28 @@
           on record.plan_exe_id = output.plan_exe_id and output.produce_module_type = 3
         where record.plan_id = #{planId} and record.status = 2 and audit_status = 1
     </select>
+    <select id="getAllByProducePlanId"
+            resultType="com.tzld.longarticle.recommend.server.model.entity.aigc.ProducePlanExeRecord">
+        select *
+        from produce_plan_exe_record
+        where plan_id in
+        <foreach collection="producePlanIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+        and status = 2
+    </select>
+    <select id="getCrawlerContentByChannelContentIdIn"
+            resultType="com.tzld.longarticle.recommend.server.model.dto.CrawlerContent">
+        select cc.channel_content_id, cprr.plan_id as crawlerPlanId, ca.wx_gh as ghId, cc.title, cc.publish_timestamp
+        from crawler_content cc
+        join (select channel_source_id, max(plan_id) as plan_id from crawler_plan_result_rel group by channel_source_id) cprr
+            on cc.channel_content_id = cprr.channel_source_id
+        join crawler_account ca on cc.channel_account_id = ca.channel_account_id
+        where cc.channel_content_id in
+        <foreach collection="channelContentIds" item="item" open="(" close=")" separator=",">
+            #{item}
+        </foreach>
+    </select>
 
 
 </mapper>

long-article-recommend-service/src/main/resources/mapper/longArticle/LongArticleBaseMapper.xml

@@ -71,4 +71,14 @@
         </foreach>
     </insert>
 
+    <insert id="batchInsertArticleCategory">
+        INSERT INTO article_category
+        (produce_content_id, channel_content_id, crawler_plan_id, title, title_md5, create_timestamp)
+        VALUES
+        <foreach collection="list" item="item" separator=",">
+            (#{item.produceContentId}, #{item.channelContentId}, #{item.crawlerPlanId}, #{item.title}, #{item.titleMd5},
+             #{item.createTimestamp})
+        </foreach>
+    </insert>
+
 </mapper>

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/XxlJobTest.java

@@ -2,8 +2,13 @@ package com.tzld.longarticle.recommend.server;
 
 import com.tzld.longarticle.recommend.server.mapper.aigc.AigcBaseMapper;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.LongArticleBaseMapper;
+import com.tzld.longarticle.recommend.server.model.dto.CrawlerContent;
 import com.tzld.longarticle.recommend.server.model.dto.ProduceContentDTO;
+import com.tzld.longarticle.recommend.server.model.entity.aigc.ProducePlanExeRecord;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticlePoolPromotionSource;
+import com.tzld.longarticle.recommend.server.model.vo.ProduceContentCrawlerVO;
+import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleCategoryRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.ArticlePoolPromotionSourceRepository;
 import com.tzld.longarticle.recommend.server.service.recommend.ArticleService;
 import com.tzld.longarticle.recommend.server.util.Md5Util;
@@ -11,9 +16,8 @@ import org.junit.jupiter.api.Test;
 import org.springframework.boot.test.context.SpringBootTest;
 
 import javax.annotation.Resource;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 @SpringBootTest(classes = Application.class)
@@ -26,6 +30,8 @@ public class XxlJobTest {
     private LongArticleBaseMapper longArticleBaseMapper;
     @Resource
     private ArticlePoolPromotionSourceRepository articlePoolPromotionSourceRepository;
+    @Resource
+    private ArticleCategoryRepository articleCategoryRepository;
 
     @Test
     public void test() {
@@ -49,4 +55,41 @@ public class XxlJobTest {
             articlePoolPromotionSourceRepository.saveAll(saveList);
         }
     }
+
+    @Test
+    public void articleCategoryTest() {
+        List<String> producePlanIds = Arrays.asList("20240802021606053813696", "20240802080355355308981",
+                "20240805154433785506170", "20240805154359027876170", "20241024100016206421084", "20241030070010871546586");
+        List<ProducePlanExeRecord> produceContentList = aigcBaseMapper.getAllByProducePlanId(producePlanIds);
+        List<String> channelContentIds = produceContentList.stream().map(ProducePlanExeRecord::getChannelContentId).distinct().collect(Collectors.toList());
+        List<ArticleCategory> articleCategoryList = articleCategoryRepository.getAllByChannelContentIdIn(channelContentIds);
+        List<String>  articleCategoryIds = articleCategoryList.stream().map(ArticleCategory::getChannelContentId).collect(Collectors.toList());
+        List<ProduceContentCrawlerVO> list = produceContentList.stream().filter(o -> !articleCategoryIds.contains(o.getChannelContentId())).map(o -> {
+            ProduceContentCrawlerVO item = new ProduceContentCrawlerVO();
+            item.setChannelContentId(o.getChannelContentId());
+            item.setProduceContentId(o.getPlanExeId());
+            return item;
+        }).collect(Collectors.toList());
+        channelContentIds = channelContentIds.stream().filter(o -> !articleCategoryIds.contains(o)).collect(Collectors.toList());
+        List<CrawlerContent> crawlerContentList = aigcBaseMapper.getCrawlerContentByChannelContentIdIn(channelContentIds);
+        Map<String, CrawlerContent> map = crawlerContentList.stream().collect(Collectors.toMap(CrawlerContent::getChannelContentId, Function.identity()));
+        long now = System.currentTimeMillis();
+        List<ArticleCategory> saveList = new ArrayList<>();
+        for (ProduceContentCrawlerVO vo : list) {
+            ArticleCategory item = new ArticleCategory();
+            item.setChannelContentId(vo.getChannelContentId());
+            item.setProduceContentId(vo.getProduceContentId());
+            CrawlerContent crawlerContent = map.get(vo.getChannelContentId());
+            if (Objects.nonNull(crawlerContent)) {
+                String title = crawlerContent.getTitle();
+                item.setCrawlerPlanId(crawlerContent.getCrawlerPlanId());
+                item.setTitle(title);
+                item.setTitleMd5(Md5Util.encoderByMd5(title));
+                item.setCreateTimestamp(now);
+                saveList.add(item);
+            }
+        }
+        longArticleBaseMapper.batchInsertArticleCategory(saveList);
+
+    }
 }