nlp

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/NLPRemoteService.java

@@ -1,10 +1,11 @@
 package com.tzld.longarticle.recommend.server.remote;
 
-import com.google.common.reflect.TypeToken;
+import com.alibaba.fastjson.JSONObject;
 import com.tzld.longarticle.recommend.server.common.HttpPoolFactory;
-import com.tzld.longarticle.recommend.server.util.JSONUtils;
+import com.tzld.longarticle.recommend.server.model.Content;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.StringEntity;
@@ -12,7 +13,9 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.util.EntityUtils;
 import org.springframework.stereotype.Service;
 
+import java.nio.charset.StandardCharsets;
 import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * @author dyp
@@ -22,57 +25,48 @@ import java.util.*;
 public class NLPRemoteService {
 
     private final CloseableHttpClient client = HttpPoolFactory.defaultPool();
+    private static final String scoreListUrl = "http://192.168.100.31:8179/score_list";
 
-    private String nlpServerUrl;
-
-    // TODO 如果过滤失败，那么认为所有视频都被过滤掉
-    public Map<String, Double> score() {
-
-        CloseableHttpResponse chr = null;
+    public Map<String, Double> score(String accountName, List<Content> contentList) {
+        long start = System.currentTimeMillis();
+        Map<String, Double> result = new HashMap<>();
+        List<String> titleList = contentList.stream().map(Content::getTitle).collect(Collectors.toList());
+        JSONObject bodyParam = new JSONObject();
+        bodyParam.put("account_nickname_list", Collections.singletonList(accountName));
+        bodyParam.put("text_list", titleList);
+        bodyParam.put("interest_type", "by_avg");
+        bodyParam.put("sim_type", "mean");
+        bodyParam.put("rate", 0.1);
         try {
-            HttpPost post = new HttpPost(nlpServerUrl);
-            post.addHeader("Connection", "close");
-            post.addHeader("content-type", "application/json");
-            Map<String, Object> param = new HashMap<>();
-            param.put("appType", appType);
-            post.setEntity(new StringEntity(JSONUtils.toJson(param)));
-
-            chr = client.execute(post);
-            if (chr == null
-                    || chr.getStatusLine() == null
-                    || chr.getStatusLine().getStatusCode() != 200) {
-                log.error("filterViewedVideo failed http status exception!");
-                return Collections.emptyMap();
-            }
-            HttpEntity entity = chr.getEntity();
-            if (entity == null) {
-                return Collections.emptyMap();
-            }
-            String content = EntityUtils.toString(entity);
-            Map<String, String> data = JSONUtils.fromJson(content,
-                    new TypeToken<Map<String, String>>() {
-                    },
-                    Collections.emptyMap());
-            if (data.get("code") == null
-                    || !data.get("code").equals("0")) {
-                log.error("filterViewedVideo failed videoFilterUrl return code exception! data={}",
-                        JSONUtils.toJson(data));
-                return Collections.emptyMap();
-            }
-            return null;
-
-        } catch (Exception e) {
-            log.error("invoke http filterViewedVideo error", e);
-        } finally {
-            try {
-                if (chr != null) {
-                    chr.close();
+            HttpPost httpPost = new HttpPost(scoreListUrl);
+            StringEntity stringEntity = new StringEntity(bodyParam.toJSONString(), StandardCharsets.UTF_8);
+            httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
+            httpPost.setEntity(stringEntity);
+            CloseableHttpResponse response = client.execute(httpPost);
+            StatusLine statusLine = response.getStatusLine();
+            if (statusLine.getStatusCode() == 200) {
+                HttpEntity responseEntity = response.getEntity();
+                if (Objects.nonNull(responseEntity)) {
+                    String responseBody = EntityUtils.toString(responseEntity, "UTF-8");
+                    log.info("scoreList 返回的数据：{}", responseBody);
+                    JSONObject scoreListResponse = JSONObject.parseObject(responseBody);
+                    if (Objects.nonNull(scoreListResponse)) {
+                        JSONObject accountScoreList = scoreListResponse.getJSONObject(accountName);
+                        if (Objects.nonNull(accountScoreList)) {
+                            List<Double> scoreList = accountScoreList.getJSONArray("score_list").toJavaList(Double.class);
+                            for (int i = 0; i < contentList.size(); i++) {
+                                result.put(contentList.get(i).getId(), scoreList.get(i));
+                            }
+                            return result;
+                        }
+                    }
                 }
-            } catch (Exception ex) {
-                log.error("close CloseableHttpResponse error", ex);
             }
+        } catch (Exception e) {
+            log.error("score error", e);
         }
-        return Collections.emptyList();
+        log.info("score耗时：{}", System.currentTimeMillis() - start);
+        return null;
     }
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/RecommendService.java

@@ -1,5 +1,6 @@
 package com.tzld.longarticle.recommend.server.service;
 
+import com.tzld.longarticle.recommend.server.model.Content;
 import com.tzld.longarticle.recommend.server.model.RecommendRequest;
 import com.tzld.longarticle.recommend.server.model.RecommendResponse;
 import com.tzld.longarticle.recommend.server.service.rank.RankParam;
@@ -9,10 +10,15 @@ import com.tzld.longarticle.recommend.server.service.recall.RecallParam;
 import com.tzld.longarticle.recommend.server.service.recall.RecallResult;
 import com.tzld.longarticle.recommend.server.service.recall.RecallService;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
 /**
  * @author dyp
  */
@@ -54,11 +60,11 @@ public class RecommendService {
 
     public RankParam convertToRankParam(RecommendParam param, RecallResult recallResult) {
         RankParam rankParam = new RankParam();
-<<<<<<< HEAD
-        BeanUtils.copyProperties(param, rankParam);
-        rankParam.setRecallResult(recallResult);
-=======
->>>>>>> b5c926c (init)
+        List<Content> contentList = new ArrayList<>();
+        if (Objects.nonNull(recallResult) && CollectionUtils.isNotEmpty(recallResult.getData())) {
+            recallResult.getData().forEach(item -> contentList.addAll(item.getContents()));
+        }
+        rankParam.setContents(contentList);
         return rankParam;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/RankService.java

@@ -10,8 +10,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 
 /**
  * @author dyp
@@ -38,4 +37,49 @@ public class RankService {
         return null;
     }
 
+
+    private List<Content> removeDuplicateContent(List<Content> contentList, Set<String> existsContentTitle) {
+        List<Content> result = new ArrayList<>();
+        for (Content content : contentList) {
+            if (existsContentTitle.contains(content.getTitle())
+                    || isDuplicateContent(content.getTitle(), existsContentTitle)) {
+                continue;
+            }
+            result.add(content);
+            existsContentTitle.add(content.getTitle());
+        }
+        return result;
+    }
+
+    private static final double SIMILARITY_THRESHOLD = 0.8; // 相似度阈值
+
+    private boolean  isDuplicateContent(String title, Set<String> existsContentTitle) {
+        boolean result = false;
+        for (String existsTitle : existsContentTitle) {
+            if (isSimilar(title, existsTitle, SIMILARITY_THRESHOLD)) {
+                return true;
+            }
+        }
+        return result;
+    }
+
+    private boolean isSimilar(String titleA, String titleB, double threshold) {
+        if (titleA.isEmpty() || titleB.isEmpty()) {
+            return false;
+        }
+        Set<Character> setA = new HashSet<>();
+        for (char c : titleA.toCharArray()) {
+            setA.add(c);
+        }
+        Set<Character> setB = new HashSet<>();
+        for (char c : titleB.toCharArray()) {
+            setB.add(c);
+        }
+        Set<Character> setCross = new HashSet<>(setA);
+        setCross.retainAll(setB);
+        int minLen = Math.max(Math.min(setA.size(), setB.size()), 1);
+        double rate = (double) setCross.size() / minLen;
+        return rate >= threshold;
+    }
+
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/score/ScoreParam.java

@@ -16,5 +16,6 @@ import java.util.List;
 @AllArgsConstructor
 @NoArgsConstructor
 public class ScoreParam {
+    private String accountName;
     private List<Content> contents;
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/score/strategy/SimilarityStrategy.java

@@ -23,7 +23,7 @@ public class SimilarityStrategy implements ScoreStrategy {
     @Override
     public List<Score> score(ScoreParam param) {
 
-        Map<String, Double> scoreMap = nlpRemoteService.score();
+        Map<String, Double> scoreMap = nlpRemoteService.score(param.getAccountName(), param.getContents());
 
         List<Score> scores = CommonCollectionUtils.toList(param.getContents(), c -> {
             Score score = new Score();