articleSensitive

long-article-recommend-service/pom.xml

@@ -102,6 +102,11 @@
             <artifactId>aliyun-log-logback-appender</artifactId>
             <version>0.1.18</version>
         </dependency>
+        <dependency>
+            <groupId>com.tencentcloudapi</groupId>
+            <artifactId>tencentcloud-sdk-java-tms</artifactId>
+            <version>3.1.1010</version>
+        </dependency>
     </dependencies>
 
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/model/remote/ArticleSensitive.java

@@ -0,0 +1,10 @@
+package com.tzld.longarticle.recommend.server.model.remote;
+
+import lombok.Data;
+
+@Data
+public class ArticleSensitive {
+    private String md5;
+    private String title;
+    private Boolean sensitive;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/ArticleSensitiveRemoteService.java

@@ -1,55 +1,96 @@
 package com.tzld.longarticle.recommend.server.remote;
 
 import com.alibaba.fastjson.JSONObject;
-import com.tzld.longarticle.recommend.server.common.HttpPoolFactory;
+import com.tencentcloudapi.common.Credential;
+import com.tencentcloudapi.common.profile.ClientProfile;
+import com.tencentcloudapi.common.profile.HttpProfile;
+import com.tencentcloudapi.tms.v20201229.TmsClient;
+import com.tencentcloudapi.tms.v20201229.models.TextModerationRequest;
+import com.tencentcloudapi.tms.v20201229.models.TextModerationResponse;
+import com.tzld.longarticle.recommend.server.model.remote.ArticleSensitive;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.http.HttpEntity;
-import org.apache.http.StatusLine;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
+import org.apache.tomcat.util.security.MD5Encoder;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
 import org.springframework.stereotype.Service;
 
-import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 import java.util.Objects;
 
 @Service
 @Slf4j
 public class ArticleSensitiveRemoteService {
 
-    private final CloseableHttpClient client = HttpPoolFactory.aigcPool();
-    private static final String articleSensitiveUrl = "http://61.48.133.26:8177/sensitive/is_sensitive";
+    @Autowired
+    JdbcTemplate piaoquanCrawlerJdbcTemplate;
+
+    @Value("${tecent.cloud.secret.id:AKIDbLGZox42mZaloeIo16zeJrDRUelkbFfw}")
+    private String tecentCloudSecretId;
+    @Value("${tecent.cloud.secret.key:CFArLAnCONHLany1qRrLyW2Rmw3ITXi1}")
+    private String tecentCloudSecretKey;
 
     public Boolean articleSensitive(String title) {
         long start = System.currentTimeMillis();
-        Boolean isSensitive = false;
-        JSONObject bodyParam = new JSONObject();
-        bodyParam.put("text", title);
+        boolean isSensitive = false;
+        String md5 = MD5Encoder.encode(title.getBytes());
+        ArticleSensitive articleSensitive = getArticleSensitive(md5);
+        if (Objects.nonNull(articleSensitive)) {
+            return articleSensitive.getSensitive();
+        }
+        String content = Base64.getEncoder().encodeToString(title.getBytes());
         try {
-            HttpPost httpPost = new HttpPost(articleSensitiveUrl);
-            StringEntity stringEntity = new StringEntity(bodyParam.toJSONString(), StandardCharsets.UTF_8);
-            httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
-            httpPost.setEntity(stringEntity);
-            CloseableHttpResponse response = client.execute(httpPost);
-            StatusLine statusLine = response.getStatusLine();
-            if (statusLine.getStatusCode() == 200) {
-                HttpEntity responseEntity = response.getEntity();
-                if (Objects.nonNull(responseEntity)) {
-                    String responseBody = EntityUtils.toString(responseEntity, "UTF-8");
-                    // log.info("articleSensitive 返回的数据：{}", responseBody);
-                    JSONObject articleSensitiveResponse = JSONObject.parseObject(responseBody);
-                    if (Objects.nonNull(articleSensitiveResponse) && articleSensitiveResponse.containsKey("is_sensitive")) {
-                        isSensitive = articleSensitiveResponse.getBoolean("is_sensitive");
-                    }
-                }
+            // 实例化一个认证对象，入参需要传入腾讯云账户 SecretId 和 SecretKey，此处还需注意密钥对的保密
+            // 代码泄露可能会导致 SecretId 和 SecretKey 泄露，并威胁账号下所有资源的安全性。以下代码示例仅供参考，建议采用更安全的方式来使用密钥，请参见：https://cloud.tencent.com/document/product/1278/85305
+            // 密钥可前往官网控制台 https://console.cloud.tencent.com/cam/capi 进行获取
+            Credential cred = new Credential(tecentCloudSecretId, tecentCloudSecretKey);
+            // 实例化一个http选项，可选的，没有特殊需求可以跳过
+            HttpProfile httpProfile = new HttpProfile();
+            httpProfile.setEndpoint("tms.tencentcloudapi.com");
+            // 实例化一个client选项，可选的，没有特殊需求可以跳过
+            ClientProfile clientProfile = new ClientProfile();
+            clientProfile.setHttpProfile(httpProfile);
+            // 实例化要请求产品的client对象,clientProfile是可选的
+            TmsClient client = new TmsClient(cred, "ap-beijing", clientProfile);
+            // 实例化一个请求对象,每个接口都会对应一个request对象
+            TextModerationRequest req = new TextModerationRequest();
+            req.setContent(content);
+            // 返回的resp是一个TextModerationResponse的实例，与请求对象对应
+            TextModerationResponse resp = client.TextModeration(req);
+            log.info("articleSensitive title:{} response:{}", title, JSONObject.toJSONString(resp));
+            if ("Pass".equals(resp.getSuggestion())
+                    || ("Polity".equals(resp.getLabel()) && "ForeignLeader".equals(resp.getSubLabel()))) {
+                return isSensitive;
+            } else {
+                isSensitive = true;
             }
         } catch (Exception e) {
             log.error("articleSensitive error", e);
         }
+        insertArticleSensitive(md5, title, isSensitive);
         log.info("articleSensitive耗时：{}", System.currentTimeMillis() - start);
         return isSensitive;
     }
 
+    private ArticleSensitive getArticleSensitive(String md5) {
+        MapSqlParameterSource parameters = new MapSqlParameterSource();
+        parameters.addValue("md5", md5);
+        ArticleSensitive result = piaoquanCrawlerJdbcTemplate.queryForObject(
+                "select md5, title, sensitiv from articles_sensitive where md5 = ?", ArticleSensitive.class, md5);
+        return result;
+    }
+
+    private void insertArticleSensitive(String md5, String title, boolean sensitive) {
+        // 插入
+        piaoquanCrawlerJdbcTemplate.update(
+                "insert into articles_sensitive(md5, title, sensitive) values (?,?,?)",
+                md5, title, sensitive);
+    }
+
+
+    public static void main(String[] args) {
+        // articleSensitive("特朗普说的双发F-35，听着耳熟，东北上空的试飞，看出歼35新定位");
+    }
 }
+

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/strategy/HistoryTitleStrategy.java

@@ -41,9 +41,9 @@ public class HistoryTitleStrategy implements FilterStrategy {
         List<String> result = new ArrayList<>();
         List<Content> filterContents = new ArrayList<>();
         List<Article> firstSecondArticleList = articleListRemoteService.articleList(param.getAccountName(), firstSecondIndex);
-        List<String> firstSecondTitleList = firstSecondArticleList.stream().map(Article::getTitle).collect(Collectors.toList());
+        List<String> firstSecondTitleList = firstSecondArticleList.stream().map(Article::getTitle).distinct().collect(Collectors.toList());
         List<Article> allArticleList = articleListRemoteService.articleList(param.getAccountName(), allIndex);
-        List<String> allTitleList = allArticleList.stream().map(Article::getTitle).collect(Collectors.toList());
+        List<String> allTitleList = allArticleList.stream().map(Article::getTitle).distinct().collect(Collectors.toList());
 
         List<String> firstSecondContentPool = new ArrayList<>();
         String[] contentPoolConfig = accountContentPoolConfigService.getContentPools(param.getAccountName());

long-article-recommend-service/src/test/java/com/tzld/longarticle/recommend/server/FilterStrategyTest.java

@@ -1,9 +1,17 @@
 package com.tzld.longarticle.recommend.server;
 
+import com.alibaba.fastjson.JSONObject;
+import com.tzld.longarticle.recommend.server.model.Content;
+import com.tzld.longarticle.recommend.server.remote.AIGCRemoteService;
+import com.tzld.longarticle.recommend.server.service.filter.FilterParam;
+import com.tzld.longarticle.recommend.server.service.filter.FilterResult;
 import com.tzld.longarticle.recommend.server.service.filter.FilterStrategy;
+import com.tzld.longarticle.recommend.server.service.recall.RecallParam;
+import org.junit.jupiter.api.Test;
 import org.springframework.boot.test.context.SpringBootTest;
 
 import javax.annotation.Resource;
+import java.util.List;
 
 @SpringBootTest(classes = Application.class)
 public class FilterStrategyTest {
@@ -12,6 +20,8 @@ public class FilterStrategyTest {
     FilterStrategy badStrategy;
     @Resource
     FilterStrategy historyTitleStrategy;
+    @Resource
+    AIGCRemoteService aigcRemoteService;
 //    @Test
 //    public void badStrategyTest() {
 //        FilterParam param = new FilterParam();
@@ -28,24 +38,18 @@ public class FilterStrategyTest {
 //        FilterResult result = badStrategy.filter(param);
 //        System.out.println(JSONObject.toJSONString(result));
 //    }
-//    @Test
-//    public void historyTitleStrategyTest() {
-//        FilterParam param = new FilterParam();
-//        List<Content> contentList = new ArrayList<>();
-//        Content content0 = new Content();
-//        content0.setId("0");
-//        content0.setTitle("婆婆照顾十二年，醒后第一句话出乎意料");
-//        content0.setProducePlanName("【1】");
-//        contentList.add(content0);
-//        Content content1 = new Content();
-//        content1.setId("1");
-//        content1.setTitle("邯郸惨案的深层原因");
-//        content1.setProducePlanName("【2】");
-//        content1.setContentPoolType("autoArticlePoolLevel2");
-//        contentList.add(content1);
-//        param.setAccountName("生活良读");
-//        param.setContents(contentList);
-//        FilterResult result = historyTitleStrategy.filter(param);
-//        System.out.println(JSONObject.toJSONString(result));
-//    }
+    @Test
+    public void historyTitleStrategyTest() {
+        // accountId:20231225071712749835955 accountName:奇趣百味生活 planId:20240627143100701713157
+        RecallParam param = new RecallParam();
+        param.setAccountName("票圈最新消息");
+        param.setAccountId("20231212144543068928298");
+        param.setPlanId("20240724092832149199870");
+        List<Content> content = aigcRemoteService.getAllContent(param);;
+        FilterParam filterParam = new FilterParam();
+        filterParam.setAccountName(param.getAccountName());
+        filterParam.setContents(content);
+        FilterResult result = historyTitleStrategy.filter(filterParam);
+        System.out.println(JSONObject.toJSONString(result));
+    }
 }