修改匹配小程序任务

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/LongArticleServerApplication.java

@@ -4,11 +4,24 @@ import com.ctrip.framework.apollo.spring.annotation.EnableApolloConfig;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+
 @EnableApolloConfig
 @SpringBootApplication
 public class LongArticleServerApplication {
 
     public static void main(String[] args) {
+        System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
+        Authenticator.setDefault(
+                new Authenticator() {
+                    public PasswordAuthentication getPasswordAuthentication() {
+                        return new PasswordAuthentication(
+                                "t11983523373311", "mtuhdr2z".toCharArray());
+                    }
+                }
+        );
+
         SpringApplication.run(LongArticleServerApplication.class, args);
     }
 }

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/controller/IndexController.java

@@ -1,5 +1,8 @@
 package com.tzld.piaoquan.longarticle.controller;
 
+import com.tzld.piaoquan.longarticle.service.local.impl.MatchVideoServiceImpl;
+import org.checkerframework.checker.units.qual.A;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
@@ -8,6 +11,10 @@ import org.springframework.web.bind.annotation.RestController;
 @RequestMapping("/")
 public class IndexController {
 
+
+    @Autowired
+    MatchVideoServiceImpl matchVideoService;
+
     /**
      * 探活
      *
@@ -17,4 +24,15 @@ public class IndexController {
     public String healthcheck() {
         return "ok";
     }
+
+    @GetMapping("/crawler")
+    public String crawler() throws InterruptedException {
+        matchVideoService.matchCrawlerVideo();
+        return "ok";
+    }
+
+    @GetMapping("/kimi")
+    public void kimi() throws InterruptedException {
+        matchVideoService.matchKimiVideo();
+    }
 }

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/CoreServiceImpl.java

@@ -3,9 +3,7 @@ package com.tzld.piaoquan.longarticle.service.local.impl;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.tzld.piaoquan.longarticle.common.constants.TimeConstant;
-import com.tzld.piaoquan.longarticle.common.enums.ContentStatusEnum;
-import com.tzld.piaoquan.longarticle.common.enums.PublishGzhPushTypeEnum;
-import com.tzld.piaoquan.longarticle.common.enums.SourceTypesEnum;
+import com.tzld.piaoquan.longarticle.common.enums.*;
 import com.tzld.piaoquan.longarticle.dao.mapper.longarticle.PlanAccountMapper;
 import com.tzld.piaoquan.longarticle.dao.mapper.longarticle.PublishContentMapper;
 import com.tzld.piaoquan.longarticle.dao.mapper.longarticle.PublishMiniprogramMapper;
@@ -45,9 +43,6 @@ public class CoreServiceImpl implements CoreService {
     @Autowired
     private AigcService aigcService;
 
-    @Autowired
-    private SortServiceImpl sortService;
-
     @Autowired
     private ContentServiceImpl contentService;
 

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/CrawlerVideoServiceImpl.java

@@ -99,8 +99,19 @@ public class CrawlerVideoServiceImpl {
 
 
     public List<CrawlerVideo> searchVideosFromWeb(LongArticlesText longArticlesText) {
-        String articleSummary = longArticlesText.getKimiSummary().substring(0, 15);
-        String oriTitle = longArticlesText.getKimiTitle().substring(0, 15);
+        String articleSummary;
+        if(longArticlesText.getKimiSummary().length() > 15){
+            articleSummary = longArticlesText.getKimiSummary().substring(0, 15);
+        }else {
+            articleSummary = longArticlesText.getKimiSummary();
+        }
+
+        String oriTitle;
+        if(longArticlesText.getKimiTitle().length() > 15){
+            oriTitle = longArticlesText.getKimiTitle().substring(0, 15);
+        }else {
+            oriTitle = longArticlesText.getKimiTitle();
+        }
         String kimiKeys = longArticlesText.getKimiKeys();
         JSONArray jsonArray = JSONArray.parseArray(kimiKeys);
         String newKimiKeys = jsonArray.stream()

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/MatchVideoServiceImpl.java

@@ -65,7 +65,7 @@ public class MatchVideoServiceImpl {
     private static final ArrayBlockingQueue<MatchVideo> matchKimiVideoQueue = new ArrayBlockingQueue<>(100000);
 
 
-    private static final int size = 2;
+    private static final int size = 5;
     // 定义一个线程池，设置消费线程的数量
     private static final ThreadPoolExecutor matchKimiVideoPoolExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(size);
 
@@ -165,7 +165,7 @@ public class MatchVideoServiceImpl {
                 } else {
                     break;
                 }
-            } while (CollectionUtils.isEmpty(matchVideos));
+            } while (!CollectionUtils.isEmpty(matchVideos));
 
             int pageNum = 1;
             do {
@@ -184,7 +184,7 @@ public class MatchVideoServiceImpl {
                 matchVideos = matchVideoMapper.selectByExample(example);
                 matchKimiVideoQueue.addAll(matchVideos);
                 pageNum++;
-            } while (CollectionUtils.isEmpty(matchVideos));
+            } while (!CollectionUtils.isEmpty(matchVideos));
             countDownLatch.await();
         }
     }
@@ -262,7 +262,7 @@ public class MatchVideoServiceImpl {
                 matchVideos = matchVideoMapper.selectByExample(example);
                 matchCrawlerVideoQueue.addAll(matchVideos);
                 pageNum++;
-            } while (CollectionUtils.isEmpty(matchVideos));
+            } while (!CollectionUtils.isEmpty(matchVideos));
             countDownLatch.await();
         }
     }
@@ -292,7 +292,7 @@ public class MatchVideoServiceImpl {
             }
             String lockKey = String.format(CRAWLER_LOCK_KEY, matchVideo.getContentId());
             String lockValue = UUID.randomUUID().toString();
-            boolean lock = redisLock.tryLock(lockKey, lockValue, 600, TimeUnit.SECONDS);
+            boolean lock = redisLock.tryLock(lockKey, lockValue, 20, TimeUnit.MINUTES);
             if (lock) {
                 LongArticlesText kimiText = kimiService.getKimiText(matchVideo.getContentId());
                 String rootContentId = getRootContentId(matchVideo.getContentId());

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/utils/ConfigUtil.java

@@ -0,0 +1,31 @@
+package com.tzld.piaoquan.longarticle.utils;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.io.ClassPathResource;
+
+import java.io.IOException;
+import java.util.Properties;
+
+@Slf4j
+public class ConfigUtil {
+
+    private static Properties properties = readProperties("application.properties");
+
+    private static Properties readProperties(String... confFile) {
+        final Properties properties = new Properties();
+        try {
+            for (String path : confFile) {
+                final ClassPathResource resource = new ClassPathResource(path);
+                properties.load(resource.getInputStream());
+            }
+        } catch (IOException e) {
+            log.error("ConfigUtil readProperties error", e);
+        }
+        return properties;
+    }
+
+    public static String getConfig(String key) {
+        return properties.getProperty(key);
+    }
+
+}

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/utils/other/DouyinSearch.java

@@ -32,7 +32,7 @@ public class DouyinSearch {
 
     public static void main(String[] args) {
         List<String> sensitiveWords = new ArrayList<>();
-        List<JSONObject> list = douyinSearch("北京女孩与父亲的法律纠纷引发对", sensitiveWords, "");
+        List<JSONObject> list = douyinSearch("【25万手术费】安徽男子垫付后遭拒还，法院判决大快人心！\uD83D\uDD34", sensitiveWords, "");
         for (JSONObject jsonObject : list) {
             System.out.println(jsonObject);
             CrawlerVideo crawlerVideo = dyVideoProduce(jsonObject);
@@ -41,7 +41,7 @@ public class DouyinSearch {
     }
 
     public static List<JSONObject> douyinSearch(String keyword, List<String> sensitiveWords, String traceId) {
-        String url = "http://8.217.190.241:8888/crawler/dou_yin/top_hub_content";
+        String url = "http://crawler-cn.aiddit.com/crawler/dou_yin/detail";
         JSONObject payload = new JSONObject();
         payload.put("keyword", keyword);
         payload.put("category", "全部");
@@ -85,7 +85,7 @@ public class DouyinSearch {
 
     public static JSONObject douyinDetail(String videoId) {
         System.out.println(videoId);
-        String url = "http://8.217.190.241:8888/crawler/dou_yin/detail";
+        String url = "http://crawler-cn.aiddit.com/crawler/dou_yin/detail";
         JSONObject payload = new JSONObject();
         payload.put("content_id", videoId);
 

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/utils/other/HkspSearch.java

@@ -33,6 +33,17 @@ public class HkspSearch {
     }
 
     public static void main(String[] args) {
+        // 这行代码是身份验证的关键配置，不然身份验证不起作用
+        System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
+        // 身份验证
+        Authenticator.setDefault(
+                new Authenticator() {
+                    public PasswordAuthentication getPasswordAuthentication() {
+                        return new PasswordAuthentication(
+                                "t11983523373311", "mtuhdr2z".toCharArray());
+                    }
+                }
+        );
         List<String> sensitiveWords = new ArrayList<>();
         List<JSONObject> list = hkspSearch("法律纠纷,教育投资,家庭责任", sensitiveWords, "");
         for (JSONObject jsonObject : list){
@@ -60,17 +71,6 @@ public class HkspSearch {
             params.put("timestamp", timestampMilliseconds);
 
             String base64String = Base64.getEncoder().encodeToString(UUID.randomUUID().toString().getBytes());
-            // 这行代码是身份验证的关键配置，不然身份验证不起作用
-            System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
-            // 身份验证
-            Authenticator.setDefault(
-                    new Authenticator() {
-                        public PasswordAuthentication getPasswordAuthentication() {
-                            return new PasswordAuthentication(
-                                    "t11983523373311", "mtuhdr2z".toCharArray());
-                        }
-                    }
-            );
             HttpResponse response = HttpRequest.get(url)
                     .header("authority", "haokan.baidu.com")
                     .header("accept", "*/*")
@@ -147,17 +147,6 @@ public class HkspSearch {
         params.put("_format", "json");
 
         String base64String = Base64.getEncoder().encodeToString(UUID.randomUUID().toString().getBytes());
-        // 这行代码是身份验证的关键配置，不然身份验证不起作用
-        System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
-        // 身份验证
-        Authenticator.setDefault(
-                new Authenticator() {
-                    public PasswordAuthentication getPasswordAuthentication() {
-                        return new PasswordAuthentication(
-                                "t11983523373311", "mtuhdr2z".toCharArray());
-                    }
-                }
-        );
         HttpResponse response = HttpRequest.get(url)
                 .header("Accept", "*/*")
                 .header("cookie", "BIDUPSID=" + base64String)

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/utils/other/OpenAIUtils.java

@@ -25,7 +25,7 @@ public class OpenAIUtils {
                 "        11.适当的紧迫感：使用“最新”、“首次”、“紧急”等词汇，创造一种紧迫感，促使读者立即行动。\n" +
                 "        12.情感或价值诉求：使用如“感动”、“泪目”、“经典”等词汇，直接与读者的情感或价值观产生共鸣。\n" +
                 "        避免误导：确保标题准确反映内容，避免夸大或误导读者。";
-        return DeepSeekAPI.chat(title + "\n" + single_title_prompt);
+        return KimiAPI.chat(title + "\n" + single_title_prompt);
     }
 
     public static Integer getKimiTitleSafeScore(String kimiTitle) {
@@ -98,9 +98,10 @@ public class OpenAIUtils {
                 "            \"content_title\": 一个总结性的标题，该标题应简洁并能够反映文章的主要内容\n" +
                 "        }\n" +
                 "        你需要处理的文本是：";
-        Map<String, String> responseFormat = new HashMap<>();
-        responseFormat.put("type", "json_object");
-        String content = DeepSeekAPI.chat(textPrompt + text, responseFormat);
+//        Map<String, String> responseFormat = new HashMap<>();
+//        responseFormat.put("type", "json_object");
+//        String content = DeepSeekAPI.chat(textPrompt + text, responseFormat);
+        String content = KimiAPI.chat(textPrompt + text);
         return JSONObject.parseObject(content);
     }
 

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/utils/other/VideoDownloader.java

@@ -1,5 +1,8 @@
 package com.tzld.piaoquan.longarticle.utils.other;
 
+import cn.hutool.http.HttpUtil;
+import com.tzld.piaoquan.longarticle.utils.ConfigUtil;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.HttpGet;
@@ -13,6 +16,7 @@ import java.net.URL;
 import java.util.Objects;
 import java.util.UUID;
 
+@Slf4j
 public class VideoDownloader {
 
     private static final String PROXY_HOST = "l901.kdltps.com";
@@ -39,7 +43,6 @@ public class VideoDownloader {
 
     public static boolean download(String path, String videoUrl, String platform) {
         try {
-
             int retries = 0;
             long fileSize = 0;
 
@@ -94,10 +97,10 @@ public class VideoDownloader {
                 }
             }
 
-            System.out.println("下载失败，已达到最大重试次数：" + MAX_RETRIES);
+            log.error("下载失败，已达到最大重试次数：" + MAX_RETRIES);
             return false;
         } catch (Exception e) {
-            e.printStackTrace();
+            log.error("download error", e);
         }
         return false;
     }
@@ -106,14 +109,14 @@ public class VideoDownloader {
         String index = String.format("%s-%s-%s", platform, videoId, UUID.randomUUID());
         String md5Hash = DigestUtils.md5Hex(index);
         String fileName = String.format("%s.mp4", md5Hash);
-        return String.join(File.separator, "/Users/shimeng/Desktop/download", fileName); // 返回文件路径和封面路径
+        return String.join(File.separator, ConfigUtil.getConfig("download.path"), fileName); // 返回文件路径和封面路径
     }
 
     private static String generateCoverPath(String platform, String videoId) {
         String index = String.format("%s-%s-%s", platform, videoId, UUID.randomUUID());
         String md5Hash = DigestUtils.md5Hex(index);
         String coverName = String.format("%s.png", md5Hash);
-        return String.join(File.separator, "/Users/shimeng/Desktop/download", coverName); // 返回文件路径和封面路径
+        return String.join(File.separator, ConfigUtil.getConfig("download.path"), coverName); // 返回文件路径和封面路径
     }
 
 }

long-article-server/src/main/resources/application-prod.properties

@@ -4,6 +4,10 @@ spring.datasource.username=changwen_admin
 spring.datasource.password=changwen@123456
 spring.datasource.url=jdbc:mysql://rm-bp14529nwwcw75yr1.mysql.rds.aliyuncs.com:3306/long_articles?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&useSSL=false&allowMultiQueries=true
 
+aigc.datasource.username=crawler_admin
+aigc.datasource.password=cyber#crawler_2023
+aigc.datasource.url=jdbc:mysql://rm-t4na9qj85v7790tf84o.mysql.singapore.rds.aliyuncs.com:3306/aigc-admin-prod?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&useSSL=false&allowMultiQueries=true
+
 spring.redis.database=2
 spring.redis.host=r-bp154bpw97gptefiqkpd.redis.rds.aliyuncs.com
 spring.redis.port=6379

long-article-server/src/main/resources/application.properties

@@ -50,6 +50,8 @@ xxl.job.executor.logpath=${datalog}/xxl-job/
 xxl.job.executor.logretentiondays=30
 
 
+download.path=/Users/shimeng/Desktop/download
+