Merge branch 'wyp/0304-articleCategory' of Server/long-article-recommend into master

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/DeepSeekApiService.java

@@ -0,0 +1,93 @@
+package com.tzld.longarticle.recommend.server.remote;
+
+import cn.hutool.core.collection.CollectionUtil;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.tzld.longarticle.recommend.server.model.dto.kimi.KimiOfficialApiResponse;
+import com.tzld.longarticle.recommend.server.model.dto.kimi.KimiResult;
+import com.tzld.longarticle.recommend.server.util.MapBuilder;
+import lombok.extern.slf4j.Slf4j;
+import okhttp3.*;
+import org.apache.http.util.TextUtils;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+@Service
+@Slf4j
+public class DeepSeekApiService {
+
+    private OkHttpClient client;
+
+    @PostConstruct
+    public void init() {
+        client = new OkHttpClient().newBuilder()
+                .connectTimeout(15, TimeUnit.MINUTES)
+                .readTimeout(15, TimeUnit.MINUTES)
+                .writeTimeout(15, TimeUnit.MINUTES)
+                .build();
+    }
+
+    public KimiResult requestOfficialApi(String prompt, String model, Double temperature, Boolean isJSON) {
+        KimiResult result = new KimiResult();
+        result.setSuccess(false);
+        if (TextUtils.isBlank(prompt) || TextUtils.isBlank(prompt.trim())) {
+            result.setFailReason("prompt is empty");
+            return result;
+        }
+
+        try {
+            JSONArray jsonArray = new JSONArray();
+            JSONObject message = new JSONObject();
+            message.put("role", "user");
+            message.put("content", prompt);
+            jsonArray.add(message);
+
+            Map<Object, Object> bodyParam = MapBuilder
+                    .builder()
+                    .put("model", Optional.ofNullable(model).orElse("deepseek-chat"))
+                    .put("temperature", Optional.ofNullable(temperature).orElse(0.3))
+                    .put("messages", jsonArray)
+                    .build();
+            if (isJSON) {
+                JSONObject formatJSON = new JSONObject();
+                formatJSON.put("type", "json_object");
+                bodyParam.put("response_format", formatJSON);
+            }
+
+            MediaType mediaType = MediaType.parse("application/json");
+            RequestBody body = RequestBody.create(mediaType, JSONObject.toJSONString(bodyParam));
+            Request request = new Request.Builder()
+                    .url("https://api.deepseek.com/chat/completions")
+                    .method("POST", body)
+                    .addHeader("Content-Type", "application/json")
+                    .addHeader("Accept", "application/json")
+                    .addHeader("Authorization", "Bearer sk-c1b18099dadc4dd1b48239bdde184f6c")
+                    .build();
+            Response response = client.newCall(request).execute();
+
+            String responseContent = response.body().string();
+            result.setResponseStr(responseContent);
+            log.info("deepseek api responseContent = {}", responseContent);
+            if (response.isSuccessful()) {
+                KimiOfficialApiResponse obj = JSONObject.parseObject(responseContent, KimiOfficialApiResponse.class);
+                if (CollectionUtil.isNotEmpty(obj.getChoices())) {
+                    result.setSuccess(true);
+                    result.setResponse(obj);
+                } else {
+                    result.setFailReason("response empty");
+                }
+            } else {
+                JSONObject json = JSONObject.parseObject(responseContent);
+                result.setFailReason("request error code:" + response.code() + " message:" + json.getString("error"));
+            }
+        } catch (Exception e) {
+            log.error("deepseek official api fail: " + e.getMessage());
+            result.setFailReason(e.getMessage());
+        }
+        return result;
+    }
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/repository/longArticle/ArticleCategoryRepository.java

@@ -1,6 +1,8 @@
 package com.tzld.longarticle.recommend.server.repository.longArticle;
 
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
@@ -13,6 +15,8 @@ public interface ArticleCategoryRepository extends JpaRepository<ArticleCategory
 
     List<ArticleCategory> getByStatus(Integer status);
 
+    Page<ArticleCategory> getByStatus(Integer status, PageRequest pageRequest);
+
     List<ArticleCategory> getAllByChannelContentIdIn(List<String> channelContentIds);
 
     List<ArticleCategory> getByStatusAndRetryTimesLessThan(Integer status, Integer retryTimes);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/recommend/ArticleCategoryService.java

@@ -20,6 +20,7 @@ import com.tzld.longarticle.recommend.server.model.param.ArticleCategoryUpdatePa
 import com.tzld.longarticle.recommend.server.model.vo.ArticleCategoryListVO;
 import com.tzld.longarticle.recommend.server.model.vo.ProduceContentCrawlerVO;
 import com.tzld.longarticle.recommend.server.remote.CrawlerContentByPlanService;
+import com.tzld.longarticle.recommend.server.remote.DeepSeekApiService;
 import com.tzld.longarticle.recommend.server.remote.KimiApiService;
 import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleCategoryRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleCrawlerPlanRepository;
@@ -57,6 +58,8 @@ public class ArticleCategoryService {
     AigcBaseMapper aigcBaseMapper;
     @Autowired
     KimiApiService kimiApiService;
+    @Autowired
+    DeepSeekApiService deepSeekApiService;
 
     @ApolloJsonValue("${cold.pool.produce.planId:[\"20240802021606053813696\", \"20240802080355355308981\",\n" +
             "\"20240805154433785506170\", \"20240805154359027876170\", \"20241024100016206421084\", " +
@@ -103,7 +106,7 @@ public class ArticleCategoryService {
         for (List<ArticleCategory> partition : partitionList) {
             List<String> partitionTitles = partition.stream().map(ArticleCategory::getTitle).distinct().collect(Collectors.toList());
             String prompt = buildKimiPrompt(partitionTitles);
-            KimiResult kimiResult = kimiApiService.requestOfficialApi(prompt, null, null, true);
+            KimiResult kimiResult = deepSeekApiService.requestOfficialApi(prompt, null, null, true);
             long now = System.currentTimeMillis();
             JSONObject obj = null;
             if (kimiResult.isSuccess()) {
@@ -305,7 +308,7 @@ public class ArticleCategoryService {
         for (ArticleCategory articleCategory : dealList) {
             List<String> partitionTitles = Collections.singletonList(articleCategory.getTitle());
             String prompt = buildKimiPrompt(partitionTitles);
-            KimiResult kimiResult = kimiApiService.requestOfficialApi(prompt, null, null, true);
+            KimiResult kimiResult = deepSeekApiService.requestOfficialApi(prompt, null, null, true);
             long now = System.currentTimeMillis();
             JSONObject obj = null;
             if (kimiResult.isSuccess()) {

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/ArticleVideoAuditTest.java

@@ -1,20 +1,40 @@
 package com.tzld.longarticle.recommend.server;
 
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Lists;
+import com.tzld.longarticle.recommend.server.common.enums.recommend.ArticleCategoryStatusEnum;
 import com.tzld.longarticle.recommend.server.mapper.longArticle.ArticleAuditMapper;
+import com.tzld.longarticle.recommend.server.model.dto.kimi.KimiResult;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.ProducePlan;
 import com.tzld.longarticle.recommend.server.model.entity.aigc.ProducePlanExeRecord;
+import com.tzld.longarticle.recommend.server.model.entity.longArticle.ArticleCategory;
 import com.tzld.longarticle.recommend.server.model.entity.longArticle.LongArticleTitleAudit;
+import com.tzld.longarticle.recommend.server.remote.DeepSeekApiService;
 import com.tzld.longarticle.recommend.server.repository.aigc.ProducePlanExeRecordRepository;
 import com.tzld.longarticle.recommend.server.repository.aigc.ProducePlanRepository;
+import com.tzld.longarticle.recommend.server.repository.longArticle.ArticleCategoryRepository;
 import com.tzld.longarticle.recommend.server.repository.longArticle.LongArticleTitleAuditRepository;
+import com.tzld.longarticle.recommend.server.service.recommend.ArticleCategoryService;
+import com.tzld.longarticle.recommend.server.util.TitleSimilarCheckUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.util.StringUtils;
 
 import javax.annotation.Resource;
+import java.io.FileOutputStream;
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 @SpringBootTest(classes = Application.class)
@@ -29,6 +49,15 @@ public class ArticleVideoAuditTest {
     private ProducePlanExeRecordRepository exeRecordRepository;
     @Resource
     private ProducePlanRepository producePlanRepository;
+    @Resource
+    private ArticleCategoryService articleCategoryService;
+    @Autowired
+    ArticleCategoryRepository articleCategoryRepository;
+    @Autowired
+    DeepSeekApiService deepSeekApiService;
+
+    @Value("${kimiCategoryPrompt:}")
+    private String kimiCategoryPrompt;
 
     @Test
     public void updateTitleAuditFlowPoolLevel() {
@@ -51,4 +80,101 @@ public class ArticleVideoAuditTest {
         }
         articleAuditMapper.updateTitleAuditFlowPoolLevel();
     }
+
+    @Test
+    public void ArticleCategoryTest() {
+        List<ArticleCategory> dealList = articleCategoryRepository.getByStatus(
+                ArticleCategoryStatusEnum.SUCCESS.getCode());
+        dealList = dealList.subList(0, 200);
+        List<List<ArticleCategory>> partitionList = Lists.partition(dealList, 50);
+        JSONArray result = new JSONArray();
+        for (List<ArticleCategory> partition : partitionList) {
+            List<String> partitionTitles = partition.stream().map(ArticleCategory::getTitle).distinct().collect(Collectors.toList());
+            String prompt = buildKimiPrompt(partitionTitles);
+            KimiResult kimiResult = deepSeekApiService.requestOfficialApi(prompt, null, null, true);
+            long now = System.currentTimeMillis();
+            JSONObject obj = null;
+            if (kimiResult.isSuccess()) {
+                try {
+                    obj = JSONObject.parseObject(kimiResult.getResponse().getChoices().get(0).getMessage().getContent());
+                } catch (Exception e) {
+                    log.error(kimiResult.getResponse().getChoices().get(0).getMessage().getContent());
+                }
+            }
+            for (ArticleCategory articleCategory : partition) {
+                JSONObject res = new JSONObject();
+                res.put("title", articleCategory.getTitle());
+                res.put("category", articleCategory.getCategory());
+                articleCategory.setKimiResult(kimiResult.getResponseStr());
+                articleCategory.setUpdateTimestamp(now);
+                if (kimiResult.isSuccess() && Objects.nonNull(obj) && obj.containsKey(articleCategory.getTitle())) {
+                    articleCategory.setCategory(obj.getString(articleCategory.getTitle()));
+                    articleCategory.setStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
+                } else {
+                    if (Objects.nonNull(obj)) {
+                        for (Map.Entry<String, Object> entry : obj.entrySet()) {
+                            if (TitleSimilarCheckUtil.isSimilar(articleCategory.getTitle(), entry.getKey(),
+                                    TitleSimilarCheckUtil.ARTICLE_PROMOTION_THRESHOLD)) {
+                                articleCategory.setCategory(obj.getString(entry.getKey()));
+                                articleCategory.setStatus(ArticleCategoryStatusEnum.SUCCESS.getCode());
+                                break;
+                            }
+                        }
+                    }
+                }
+                if (articleCategory.getStatus().equals(ArticleCategoryStatusEnum.SUCCESS.getCode())) {
+                    res.put("newCategory", articleCategory.getCategory());
+                }
+                result.add(res);
+            }
+        }
+
+        Workbook workbook = new XSSFWorkbook();
+        Sheet sheet = workbook.createSheet("ExampleSheet");
+        int rowNum = 0;
+        // 创建标题行
+        Row titleRow = sheet.createRow(rowNum);
+        Cell titleCell = titleRow.createCell(0);
+        titleCell.setCellValue("标题");
+        titleCell = titleRow.createCell(1);
+        titleCell.setCellValue("kimi品类");
+        titleCell = titleRow.createCell(2);
+        titleCell.setCellValue("deepseek品类");
+
+        for (Object obj : result) {
+            JSONObject jsonObject = (JSONObject) obj;
+            String title = jsonObject.getString("title");
+            String category = jsonObject.getString("category");
+            String newCategory = jsonObject.getString("newCategory");
+            rowNum++;
+            Row row = sheet.createRow(rowNum);
+            Cell cell = row.createCell(0);
+            cell.setCellValue(title);
+            cell = row.createCell(1);
+            cell.setCellValue(category);
+            cell = row.createCell(2);
+            cell.setCellValue(newCategory);
+        }
+
+        try (FileOutputStream outputStream = new FileOutputStream("/Users/wangyunpeng/Downloads/category.xlsx")) {
+            workbook.write(outputStream);
+        } catch (IOException e) {
+            e.printStackTrace();
+        } finally {
+            try {
+                workbook.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+    private String buildKimiPrompt(List<String> titleList) {
+        StringBuilder prompt = new StringBuilder(kimiCategoryPrompt);
+        prompt.append("\n");
+        for (String title : titleList) {
+            prompt.append(title).append("\n");
+        }
+        return prompt.toString();
+    }
 }