init

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/common/ThreadPoolFactory.java

@@ -32,6 +32,13 @@ public final class ThreadPoolFactory {
             new LinkedBlockingQueue<>(1000),
             new ThreadFactoryBuilder().setNameFormat("FilterService-%d").build(),
             new ThreadPoolExecutor.AbortPolicy());
+    private final static ExecutorService SCORE = new CommonThreadPoolExecutor(
+            32,
+            128,
+            0L, TimeUnit.SECONDS,
+            new LinkedBlockingQueue<>(1000),
+            new ThreadFactoryBuilder().setNameFormat("ScoreService-%d").build(),
+            new ThreadPoolExecutor.AbortPolicy());
 
     public static ExecutorService defaultPool() {
         return DEFAULT;
@@ -45,5 +52,9 @@ public final class ThreadPoolFactory {
         return FILTER;
     }
 
+    public static ExecutorService scorePool() {
+        return SCORE;
+    }
+
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/NLPRemoteService.java

@@ -26,8 +26,7 @@ public class NLPRemoteService {
     private String nlpServerUrl;
 
     // TODO 如果过滤失败，那么认为所有视频都被过滤掉
-    public List<Long> filterViewedVideo(int appType, String mid, String uid, List<Long> videoIds, String cityCode,
-                                        Set<String> abExpCodes, Long hotSceneType, String clientIp) {
+    public Map<String, Double> score() {
 
         CloseableHttpResponse chr = null;
         try {
@@ -43,11 +42,11 @@ public class NLPRemoteService {
                     || chr.getStatusLine() == null
                     || chr.getStatusLine().getStatusCode() != 200) {
                 log.error("filterViewedVideo failed http status exception!");
-                return Collections.emptyList();
+                return Collections.emptyMap();
             }
             HttpEntity entity = chr.getEntity();
             if (entity == null) {
-                return Collections.emptyList();
+                return Collections.emptyMap();
             }
             String content = EntityUtils.toString(entity);
             Map<String, String> data = JSONUtils.fromJson(content,
@@ -58,11 +57,9 @@ public class NLPRemoteService {
                     || !data.get("code").equals("0")) {
                 log.error("filterViewedVideo failed videoFilterUrl return code exception! data={}",
                         JSONUtils.toJson(data));
-                return Collections.emptyList();
+                return Collections.emptyMap();
             }
-            return JSONUtils.fromJson(data.get("data"),
-                    new TypeToken<List<Long>>() {
-                    }, Collections.emptyList());
+            return null;
 
         } catch (Exception e) {
             log.error("invoke http filterViewedVideo error", e);

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/odps/ODPSManager.java → long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/remote/ODPSManager.java

@@ -1,4 +1,4 @@
-package com.tzld.longarticle.recommend.server.service.odps;
+package com.tzld.longarticle.recommend.server.remote;
 
 import com.aliyun.odps.Instance;
 import com.aliyun.odps.Odps;

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/RecommendService.java

@@ -54,8 +54,11 @@ public class RecommendService {
 
     public RankParam convertToRankParam(RecommendParam param, RecallResult recallResult) {
         RankParam rankParam = new RankParam();
+<<<<<<< HEAD
         BeanUtils.copyProperties(param, rankParam);
         rankParam.setRecallResult(recallResult);
+=======
+>>>>>>> b5c926c (init)
         return rankParam;
     }
 

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/FilterParam.java

@@ -11,6 +11,4 @@ import java.util.List;
 @Data
 public class FilterParam {
     private List<Content> contents;
-    private boolean flowPool;
-
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/FilterService.java

@@ -5,7 +5,7 @@ import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
 import com.tzld.longarticle.recommend.server.service.ServiceBeanFactory;
 import com.tzld.longarticle.recommend.server.service.filter.strategy.CategoryStrategy;
 import com.tzld.longarticle.recommend.server.service.filter.strategy.DuplicateStrategy;
-import com.tzld.longarticle.recommend.server.service.filter.strategy.SimilarityStrategy;
+import com.tzld.longarticle.recommend.server.service.filter.strategy.HistoryTitleStrategy;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
@@ -60,21 +60,18 @@ public class FilterService {
         if (CollectionUtils.isEmpty(contentIds)) {
             return Collections.emptyList();
         }
-        List<Long> result = Lists.newArrayList(param.getVideoIds());
-        for (int i = 0; i < contentIds.size(); ++i) {
-            result.retainAll(contentIds.get(i));
-        }
-        return result;
+//        List<Long> result = Lists.newArrayList(param.getContents());
+//        for (int i = 0; i < contentIds.size(); ++i) {
+//            result.retainAll(contentIds.get(i));
+//        }
+        return null;
     }
 
     private List<FilterStrategy> getStrategies(FilterParam param) {
         List<FilterStrategy> strategies = new ArrayList<>();
-        strategies.add(ServiceBeanFactory.getBean(SimilarityStrategy.class));
+        strategies.add(ServiceBeanFactory.getBean(HistoryTitleStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(DuplicateStrategy.class));
-
-        if (param.isFlowPool()) {
-            strategies.add(ServiceBeanFactory.getBean(CategoryStrategy.class));
-        }
+        strategies.add(ServiceBeanFactory.getBean(CategoryStrategy.class));
 
         return strategies;
     }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/filter/strategy/CategoryStrategy.java

@@ -1,11 +1,14 @@
 package com.tzld.longarticle.recommend.server.service.filter.strategy;
 
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.longarticle.recommend.server.service.filter.FilterParam;
 import com.tzld.longarticle.recommend.server.service.filter.FilterStrategy;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
+import java.util.Map;
 
 /**
  * @author dyp
@@ -13,6 +16,10 @@ import java.util.List;
 @Component
 @Slf4j
 public class CategoryStrategy implements FilterStrategy {
+    @Value("${aaa:528}")
+    private String aaa;
+    @ApolloJsonValue("${bbb:{}}")
+    protected Map<String, String> bbb;
 
     @Override
     public List<Long> filter(FilterParam param) {

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/RankParam.java

@@ -1,14 +1,14 @@
 package com.tzld.longarticle.recommend.server.service.rank;
 
-import com.tzld.longarticle.recommend.server.service.recall.RecallResult;
+import com.tzld.longarticle.recommend.server.model.Content;
 import lombok.Data;
 
+import java.util.List;
+
 /**
  * @author dyp
  */
 @Data
 public class RankParam {
-    private RecallResult recallResult;
-    private int size;
-
+    private List<Content> contents;
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/rank/RankService.java

@@ -1,12 +1,17 @@
 package com.tzld.longarticle.recommend.server.service.rank;
 
 
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.longarticle.recommend.server.model.Content;
+import com.tzld.longarticle.recommend.server.service.score.ScoreParam;
+import com.tzld.longarticle.recommend.server.service.score.ScoreResult;
+import com.tzld.longarticle.recommend.server.service.score.ScoreService;
+import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Value;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.util.Map;
+import java.util.Collections;
+import java.util.List;
 
 /**
  * @author dyp
@@ -14,18 +19,23 @@ import java.util.Map;
 @Service
 @Slf4j
 public class RankService {
-    @Value("${aaa:528}")
-    private String aaa;
-    @ApolloJsonValue("${bbb:{}}")
-    protected Map<String, String> bbb;
+
+    @Autowired
+    private ScoreService scoreService;
 
     public RankResult rank(RankParam param) {
 
+        ScoreResult result = scoreService.score(convertToScoreParam(param));
+
 
-        RankResult result = new RankResult();
+        Collections.sort(items, (o1, o2) -> Double.compare(o2.getScore(), o1.getScore()));
+        List<Content> contents = CommonCollectionUtils.toList(items, RankItem::getContent);
 
+        return new RankResult(contents);
+    }
 
-        return result;
+    private ScoreParam convertToScoreParam(RankParam param) {
+        return null;
     }
 
 }

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/score/Score.java

@@ -0,0 +1,19 @@
+package com.tzld.longarticle.recommend.server.service.score;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+/**
+ * @author dyp
+ */
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+public class Score {
+    private Class<? extends ScoreStrategy> strategy;
+    private String contentId;
+    private double score;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/score/ScoreParam.java

@@ -0,0 +1,20 @@
+package com.tzld.longarticle.recommend.server.service.score;
+
+import com.tzld.longarticle.recommend.server.model.Content;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.List;
+
+/**
+ * @author dyp
+ */
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+public class ScoreParam {
+    private List<Content> contents;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/score/ScoreResult.java

@@ -0,0 +1,19 @@
+package com.tzld.longarticle.recommend.server.service.score;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.Map;
+
+/**
+ * @author dyp
+ */
+@Setter
+@Getter
+@AllArgsConstructor
+@NoArgsConstructor
+public class ScoreResult {
+    private Map<String, Map<Class<? extends ScoreStrategy>, Double>> scoreMap;
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/score/ScoreService.java

@@ -0,0 +1,92 @@
+package com.tzld.longarticle.recommend.server.service.score;
+
+
+import com.tzld.longarticle.recommend.server.common.ThreadPoolFactory;
+import com.tzld.longarticle.recommend.server.service.score.strategy.SimilarityStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author dyp
+ */
+@Service
+@Slf4j
+public class ScoreService implements ApplicationContextAware {
+
+    private final Map<String, ScoreStrategy> strategyMap = new HashMap<>();
+    private ApplicationContext applicationContext;
+    private final ExecutorService pool = ThreadPoolFactory.scorePool();
+
+    @PostConstruct
+    public void init() {
+        Map<String, ScoreStrategy> type = applicationContext.getBeansOfType(ScoreStrategy.class);
+        for (Map.Entry<String, ScoreStrategy> entry : type.entrySet()) {
+            ScoreStrategy value = entry.getValue();
+            strategyMap.put(value.getClass().getSimpleName(), value);
+        }
+    }
+
+    public ScoreResult score(ScoreParam param) {
+        List<ScoreStrategy> strategies = getScoreStrategy(param);
+        CountDownLatch cdl = new CountDownLatch(strategies.size());
+        List<Future<List<Score>>> futures = new ArrayList<>();
+        for (final ScoreStrategy strategy : strategies) {
+            Future<List<Score>> future = pool.submit(() -> {
+                List<Score> result = new ArrayList<>();
+                result.add(strategy.score(param);
+                cdl.countDown();
+                return result;
+            });
+            futures.add(future);
+        }
+        try {
+            cdl.await(30000, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            log.error("rank error", e);
+            return null;
+        }
+
+        Map<String, Map<Class<? extends ScoreStrategy>, Double>> scoreMap = new HashMap<>();
+        for (Future<List<Score>> f : futures) {
+            try {
+                List<Score> data = f.get();
+                for (Score score : data) {
+                    Map<Class<? extends ScoreStrategy>, Double> map
+                            = scoreMap.computeIfAbsent(score.getContentId(), k -> new HashMap<>());
+                    map.put(score.getStrategy(), score.getScore());
+                }
+            } catch (Exception e) {
+                log.error("future get error ", e);
+            }
+        }
+
+        return new ScoreResult(scoreMap);
+    }
+
+    private List<ScoreStrategy> getScoreStrategy(ScoreParam param) {
+        List<ScoreStrategy> strategies = new ArrayList<>();
+        strategies.add(strategyMap.get(SimilarityStrategy.class.getSimpleName()));
+        strategies.add(strategyMap.get(ContentPoolStrategy.class.getSimpleName()));
+
+        return strategies;
+    }
+
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+        this.applicationContext = applicationContext;
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/score/ScoreStrategy.java

@@ -0,0 +1,11 @@
+package com.tzld.longarticle.recommend.server.service.score;
+
+
+import java.util.List;
+
+/**
+ * @author dyp
+ */
+public interface ScoreStrategy {
+    List<Score> score(ScoreParam param);
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/score/strategy/ContentPoolStrategy.java

@@ -0,0 +1,33 @@
+package com.tzld.longarticle.recommend.server.service.score.strategy;
+
+import com.tzld.longarticle.recommend.server.remote.NLPRemoteService;
+import com.tzld.longarticle.recommend.server.service.score.Score;
+import com.tzld.longarticle.recommend.server.service.score.ScoreParam;
+import com.tzld.longarticle.recommend.server.service.score.ScoreStrategy;
+import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author dyp
+ */
+@Component
+public class ContentPoolStrategy implements ScoreStrategy {
+
+    @Override
+    public List<Score> score(ScoreParam param) {
+
+        List<Score> scores = CommonCollectionUtils.toList(param.getContents(), c -> {
+            Score score = new Score();
+            score.setContentId(c.getId());
+            score.setScore();
+            score.setStrategy(this.getClass());
+            return score;
+        });
+        return scores;
+    }
+
+}

long-article-recommend-service/src/main/java/com/tzld/longarticle/recommend/server/service/score/strategy/SimilarityStrategy.java

@@ -0,0 +1,38 @@
+package com.tzld.longarticle.recommend.server.service.score.strategy;
+
+import com.tzld.longarticle.recommend.server.remote.NLPRemoteService;
+import com.tzld.longarticle.recommend.server.service.score.Score;
+import com.tzld.longarticle.recommend.server.service.score.ScoreParam;
+import com.tzld.longarticle.recommend.server.service.score.ScoreStrategy;
+import com.tzld.longarticle.recommend.server.util.CommonCollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author dyp
+ */
+@Component
+public class SimilarityStrategy implements ScoreStrategy {
+
+    @Autowired
+    private NLPRemoteService nlpRemoteService;
+
+    @Override
+    public List<Score> score(ScoreParam param) {
+
+        Map<String, Double> scoreMap = nlpRemoteService.score();
+
+        List<Score> scores = CommonCollectionUtils.toList(param.getContents(), c -> {
+            Score score = new Score();
+            score.setContentId(c.getId());
+            score.setScore(scoreMap.get(c.getId()));
+            score.setStrategy(this.getClass());
+            return score;
+        });
+        return scores;
+    }
+
+}