Merge branch 'feature_refactor' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/RecommendParam.java

@@ -46,9 +46,6 @@ public class RecommendParam {
     private Set<String> abExpCodes;
     private boolean riskUser;
 
-    // 层 - 实验
-    private Map<String, String> expIdMap;
-
     private Integer categoryId;
 
     private Long hotSceneType;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/remote/FeatureRemoteService.java

@@ -1,143 +0,0 @@
-package com.tzld.piaoquan.recommend.server.remote;
-
-import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserActionFeature;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.feature.model.feature.UserActionFeatureProto;
-import com.tzld.piaoquan.recommend.feature.model.feature.UserFeatureProto;
-import com.tzld.piaoquan.recommend.feature.model.feature.VideoFeatureProto;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.math.NumberUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-
-/**
- * @author dyp
- */
-@Component
-@Slf4j
-public class FeatureRemoteService {
-
-    @Autowired
-    private FeatureClient client;
-
-    // 有性能问题再增加localCache
-    public UserFeature getUserFeature(String mid) {
-        if (StringUtils.isBlank(mid)) {
-            return null;
-        }
-        UserFeatureProto proto = client.getUserFeature(mid);
-        return convert(proto);
-    }
-
-    private UserFeature convert(UserFeatureProto proto) {
-        if (proto == null) {
-            return null;
-        }
-        UserFeature feature = new UserFeature();
-        feature.setUid(proto.getUid());
-        feature.setMid(proto.getMid());
-        if (proto.hasDay1CntFeature()) {
-            feature.setDay1_cnt_features(convert(proto.getDay1CntFeature()));
-        } else {
-            feature.setDay1_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasDay3CntFeature()) {
-            feature.setDay3_cnt_features(convert(proto.getDay3CntFeature()));
-        } else {
-            feature.setDay3_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasDay7CntFeature()) {
-            feature.setDay7_cnt_features(convert(proto.getDay7CntFeature()));
-        } else {
-            feature.setDay7_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasMonth3CntFeature()) {
-            feature.setMonth3_cnt_features(convert(proto.getMonth3CntFeature()));
-        } else {
-            feature.setMonth3_cnt_features(new UserActionFeature());
-        }
-        feature.setUser_cycle_bucket_7days(proto.getUserCycleBucket7Day());
-        feature.setUser_cycle_bucket_30days(proto.getUserCycleBucket30Day());
-        feature.setUser_share_bucket_30days(proto.getUserShareBucket30Day());
-
-        return feature;
-    }
-
-    private UserActionFeature convert(UserActionFeatureProto proto) {
-        UserActionFeature feature = new UserActionFeature();
-        feature.setClick_cnt(proto.getClickCnt());
-        feature.setCtr(proto.getCtr());
-        feature.setExp_cnt(proto.getExpCnt());
-        feature.setReturn_cnt(proto.getReturnCnt());
-        feature.setRov(proto.getRov());
-        feature.setShare_cnt(proto.getShareCnt());
-        feature.setStr(proto.getStr());
-        return feature;
-    }
-
-    public ItemFeature getVideoFeature(Long videoId) {
-        if (videoId == null) {
-            return null;
-        }
-        VideoFeatureProto proto = client.getVideoFeature(videoId);
-        return convert(proto);
-    }
-
-    public Map<Long, ItemFeature> getVideoFeatureMap(List<Long> videoIds) {
-        if (CollectionUtils.isEmpty(videoIds)) {
-            return Collections.emptyMap();
-        }
-        List<VideoFeatureProto> protoList = client.getAllVideoFeature(videoIds);
-        return CommonCollectionUtils.toMap(protoList, p -> NumberUtils.toLong(p.getVideoId(), 0L), this::convert);
-    }
-
-
-    private ItemFeature convert(VideoFeatureProto proto) {
-        if (proto == null) {
-            return null;
-        }
-        ItemFeature feature = new ItemFeature();
-
-        feature.setPlayLength(proto.getPlayLength());
-        feature.setTags(proto.getTags());
-        feature.setTotalTime(proto.getTotalTime());
-        feature.setUpId(proto.getUpId());
-        feature.setVideoId(proto.getVideoId());
-        feature.setDaysSinceUpload(proto.getDaysSinceUpload());
-
-
-        if (proto.hasVideoDay1CntFeature()) {
-            feature.setDay1_cnt_features(convert(proto.getVideoDay1CntFeature()));
-        } else {
-            feature.setDay1_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasVideoDay3CntFeature()) {
-            feature.setDay3_cnt_features(convert(proto.getVideoDay3CntFeature()));
-        } else {
-            feature.setDay3_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasVideoDay7CntFeature()) {
-            feature.setDay7_cnt_features(convert(proto.getVideoDay7CntFeature()));
-        } else {
-            feature.setDay7_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasVideoMonth3CntFeature()) {
-            feature.setMonth3_cnt_features(convert(proto.getVideoMonth3CntFeature()));
-        } else {
-            feature.setMonth3_cnt_features(new UserActionFeature());
-        }
-        return feature;
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -71,15 +71,6 @@ public class RecommendService {
     @ApolloJsonValue("${city_code:[]}")
     private Set<String> cityCodes;
 
-    @ApolloJsonValue("${new.exp.list:[6]}")
-    private Set<Integer> newExpList;
-
-    @ApolloJsonValue("${new.exp,config.v2:[]}")
-    private List<ExpConfig> expConfigs;
-
-    @Value("${new.exp.abtest.service.switch:true}")
-    private boolean newExpABTestServiceSwitch;
-
     @Autowired
     private FlowPoolConfigService flowPoolConfigService;
     @Autowired
@@ -431,40 +422,6 @@ public class RecommendService {
             param.setMachineInfo(machineInfo);
         }
 
-
-        // TODO：hard code 为了快速做AB验证，应该由AB系统支持
-        if (newExpList.contains(param.getAppType())) {
-            if (newExpABTestServiceSwitch) {
-                param.setExpIdMap(abTestRemoteService.getExp(param.getMid(), param.getUid(), param.getAppType(),
-                        request.getNewExpGroup()));
-            } else {
-                Map<String, List<Layer>> layerMap = CommonCollectionUtils.toMap(expConfigs, c -> c.getAppType(),
-                        c -> c.getLayers());
-                if (MapUtils.isNotEmpty(layerMap)) {
-                    Map<String, Integer> bucketMap = JSONUtils.fromJson(request.getNewExpGroup(),
-                            new TypeToken<Map<String, Integer>>() {
-                            }, Collections.emptyMap());
-                    if (MapUtils.isNotEmpty(bucketMap)) {
-                        List<Layer> layers = layerMap.get(param.getAppType() + "");
-
-                        if (CollectionUtils.isNotEmpty(layers)) {
-                            Map<String, String> expIdMap = new HashMap<>();
-                            for (Layer layer : layers) {
-                                for (Exp exp : layer.getExps()) {
-                                    if (bucketMap.containsKey(layer.getLayerId())
-                                            && exp.getRange().length == 2
-                                            && exp.getRange()[0] <= bucketMap.get(layer.getLayerId())
-                                            && exp.getRange()[1] >= bucketMap.get(layer.getLayerId())) {
-                                        expIdMap.put(layer.getLayerId(), exp.getExpId());
-                                    }
-                                }
-                            }
-                            param.setExpIdMap(expIdMap);
-                        }
-                    }
-                }
-            }
-        }
         if (StringUtils.isNotBlank(request.getCategoryId()) && StringUtils.isNumeric(request.getCategoryId())) {
             param.setCategoryId(Integer.parseInt(request.getCategoryId()));
         }
@@ -521,19 +478,6 @@ public class RecommendService {
         recallParam.setDataKey(param.getDataKey());
         recallParam.setHRuleKey(param.getHRuleKey());
         recallParam.setHDataKey(param.getHDataKey());
-        if (newExpList.contains(param.getAppType())) {
-            recallParam.setAbCode("");
-            if (MapUtils.isNotEmpty(param.getExpIdMap()) && param.getExpIdMap().containsKey("recall")) {
-                String expId = param.getExpIdMap().get("recall");
-                if (abExpCodeMap.containsKey(expId)) {
-                    recallParam.setAbCode(abExpCodeMap.get(expId).get("ab_code"));
-                    recallParam.setRuleKey(abExpCodeMap.get(expId).get("rule_key"));
-                    recallParam.setDataKey(abExpCodeMap.get(expId).get("data_key"));
-                    recallParam.setHDataKey(abExpCodeMap.get(expId).get("h_data_key"));
-                    recallParam.setHRuleKey(abExpCodeMap.get(expId).get("h_rule_key"));
-                }
-            }
-        }
 
         recallParam.setVideoId(param.getVideoId());
         recallParam.setFlowPoolAbtestGroup(param.getFlowPoolAbtestGroup());
@@ -554,7 +498,6 @@ public class RecommendService {
         recallParam.setAbExpCodes(param.getAbExpCodes());
 
         recallParam.setProvince(param.getProvince());
-        recallParam.setExpIdMap(param.getExpIdMap());
         recallParam.setCategoryId(param.getCategoryId());
 
         recallParam.setCityCode(param.getCityCode());
@@ -576,23 +519,6 @@ public class RecommendService {
         // note 避免非实验产品被覆盖
         rankParam.setAbCode(param.getAbCode());
         rankParam.setRankKeyPrefix(param.getRankKeyPrefix());
-        if (newExpList.contains(param.getAppType())) {
-            rankParam.setAbCode("");
-//            if (MapUtils.isNotEmpty(param.getExpIdMap())
-//                    && param.getExpIdMap().containsKey("rank")) {
-//                String expId = param.getExpIdMap().get("rank");
-            // TODO hard code  MVP版本 为了快速验证，对rank强制染色，不做实验编排
-            if (MapUtils.isNotEmpty(param.getExpIdMap())
-                    && param.getExpIdMap().containsKey("recall")) {
-                String expId = param.getExpIdMap().get("recall");
-                if (abExpCodeMap.containsKey(expId)) {
-                    rankParam.setAbCode(abExpCodeMap.get(expId).get("ab_code"));
-                    rankParam.setRankKeyPrefix(StringUtils.isNotBlank(abExpCodeMap.get(expId).get("rank_key_prefix"))
-                            ? abExpCodeMap.get(expId).get("rank_key_prefix")
-                            : "rank:score1:");
-                }
-            }
-        }
 
         rankParam.setSize(param.getSize());
         rankParam.setFlowPoolP(param.getFlowPoolP());
@@ -603,7 +529,6 @@ public class RecommendService {
         rankParam.setCity(param.getCity());
         rankParam.setMachineInfo(param.getMachineInfo());
         rankParam.setAbExpCodes(param.getAbExpCodes());
-        rankParam.setExpIdMap(param.getExpIdMap());
         rankParam.setCategoryId(param.getCategoryId());
         rankParam.setHeadVid(param.getVideoId());
         rankParam.setHotSceneType(param.getHotSceneType());
@@ -674,85 +599,4 @@ public class RecommendService {
         consumer.accept(RegionHWithoutDupRecallStrategy.PUSH_FORM);
     }
 
-    public static class ExpConfig {
-        private String appType;
-        private List<Layer> layers;
-
-        public String getAppType() {
-            return appType;
-        }
-
-        public void setAppType(String appType) {
-            this.appType = appType;
-        }
-
-        public List<Layer> getLayers() {
-            return layers;
-        }
-
-        public void setLayers(List<Layer> layers) {
-            this.layers = layers;
-        }
-    }
-
-    public static class Layer {
-        private String layerId;
-        private int bucketNum;
-        private List<Exp> exps;
-        private Map<String, String> groupRule;
-
-        public String getLayerId() {
-            return layerId;
-        }
-
-        public void setLayerId(String layerId) {
-            this.layerId = layerId;
-        }
-
-        public int getBucketNum() {
-            return bucketNum;
-        }
-
-        public void setBucketNum(int bucketNum) {
-            this.bucketNum = bucketNum;
-        }
-
-        public List<Exp> getExps() {
-            return exps;
-        }
-
-        public void setExps(List<Exp> exps) {
-            this.exps = exps;
-        }
-
-        public Map<String, String> getGroupRule() {
-            return groupRule;
-        }
-
-        public void setGroupRule(Map<String, String> groupRule) {
-            this.groupRule = groupRule;
-        }
-    }
-
-    public static class Exp {
-        private String expId;
-        private int[] range;
-
-        public String getExpId() {
-            return expId;
-        }
-
-        public void setExpId(String expId) {
-            this.expId = expId;
-        }
-
-        public int[] getRange() {
-            return range;
-        }
-
-        public void setRange(int[] range) {
-            this.range = range;
-        }
-    }
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -76,7 +76,6 @@ public abstract class AbstractFilterService {
         strategies.add(ServiceBeanFactory.getBean(PreViewedStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(ViewedStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(RecommendStatusStrategy.class));
-        strategies.add(ServiceBeanFactory.getBean(SupplyExpStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(AppletVideoStatusStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(RiskVideoStrategy.class));
         switch (param.getAppType()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterParam.java

@@ -22,9 +22,6 @@ public class FilterParam {
     private String cityCode;
     private Set<String> abExpCodes;
 
-    // 层 - 实验
-    private Map<String, String> expIdMap;
-
     private String abCode;
 
     private Long hotSceneType;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/SupplyExpStrategy.java

@@ -1,65 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.filter.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.collect.Lists;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang.math.NumberUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * @author dyp
- */
-@Component
-public class SupplyExpStrategy implements FilterStrategy {
-    @ApolloJsonValue("${supply.exp.list:[6]}")
-    private Set<Integer> supplyExps;
-
-    @Value("${supply.exp.id:666}")
-    private int supplyExpId;
-
-    @ApolloJsonValue("${supply.exp.video.list:[]}")
-    private Set<Long> supplyExpVideos;
-    @ApolloJsonValue("${not.supply.exp.video.list:[]}")
-    private Set<Long> notSupplyExpVideos;
-
-    @Override
-    public List<Long> filter(FilterParam param) {
-
-        if (param == null) {
-            return Collections.emptyList();
-        }
-        if (StringUtils.isBlank(param.getMid())
-                || CollectionUtils.isEmpty(param.getVideoIds())) {
-            return param.getVideoIds();
-        }
-
-        if (!supplyExps.contains(param.getAppType())) {
-            return Lists.newArrayList(param.getVideoIds());
-        }
-        if (MapUtils.isEmpty(param.getExpIdMap())) {
-            return Lists.newArrayList(param.getVideoIds());
-        }
-        // 供给实验
-        if (supplyExpId == NumberUtils.toInt(param.getExpIdMap().get("supply"), -1)) {
-            // 对照组视频只在对照组出
-            return param.getVideoIds().stream()
-                    .filter(l -> !notSupplyExpVideos.contains(l))
-                    .collect(Collectors.toList());
-        } else {
-            // 实验组视频只在实验组出
-            return param.getVideoIds().stream()
-                    .filter(l -> !supplyExpVideos.contains(l))
-                    .collect(Collectors.toList());
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankParam.java

@@ -28,9 +28,6 @@ public class RankParam {
     private Long headVid=0L;
     private Long hotSceneType=0L;
 
-    // 层 - 实验
-    private Map<String, String> expIdMap;
-
     private Integer categoryId;
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -2,20 +2,14 @@ package com.tzld.piaoquan.recommend.server.service.rank;
 
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.MachineInfo;
 import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.remote.FeatureRemoteService;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
@@ -39,12 +33,6 @@ public class RankService {
     @Autowired
     @Qualifier("redisTemplate")
     public RedisTemplate<String, String> redisTemplate;
-    @Autowired
-    private FeatureRemoteService featureRemoteService;
-    @Value("${newRankSwitch:false}")
-    private boolean newRankSwitch;
-    @Value("${newRankAbExpCode:528}")
-    private String newRankAbExpCode;
     @ApolloJsonValue("${alg.recall.special.app&vid:{}}")
     protected Map<String, List<Long>> specialAppVid;
     @ApolloJsonValue("${region.recall.return.size:{}}")
@@ -283,32 +271,6 @@ public class RankService {
         }
     }
 
-    private List<Video> mergeAndRankRovRecallNew(RankParam param) {
-        UserFeature userFeature = featureRemoteService.getUserFeature(param.getMid());
-
-        List<Video> recallVideos = param.getRecallResult().mergeRecallVideos();
-        List<RankItem> rankItems = CommonCollectionUtils.toList(recallVideos, RankItem::new);
-
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(recallVideos, Video::getVideoId);
-        Map<Long, ItemFeature> videoFeatureMap = featureRemoteService.getVideoFeatureMap(videoIds);
-
-        for (RankItem rankItem : rankItems) {
-            rankItem.setItemFeature(videoFeatureMap.get(rankItem.getVideoId()));
-        }
-
-        // TODO
-        ScoreParam scoreParam = convert(param);
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(scoreParam, userFeature, rankItems);
-
-        return CommonCollectionUtils.toList(rovRecallScore, i -> {
-            // hard code 将排序分数 赋值给video的sortScore
-            Video v = i.getVideo();
-            v.setSortScore(i.getScore());
-            return v;
-        });
-    }
-
     protected ScoreParam convert(RankParam param) {
         ScoreParam scoreParam = new ScoreParam();
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/FilterParamFactory.java

@@ -17,7 +17,6 @@ public class FilterParamFactory {
         filterParam.setMid(param.getMid());
         filterParam.setUid(param.getUid());
         filterParam.setFlowPoolMap(videoFlowPoolMap);
-        filterParam.setExpIdMap(param.getExpIdMap());
 
         // 风险过滤
         filterParam.setRiskUser(param.isRiskUser());
@@ -36,7 +35,6 @@ public class FilterParamFactory {
         filterParam.setAppType(param.getAppType());
         filterParam.setMid(param.getMid());
         filterParam.setUid(param.getUid());
-        filterParam.setExpIdMap(param.getExpIdMap());
 
         // 风险过滤
         filterParam.setRiskUser(param.isRiskUser());

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallParam.java

@@ -31,8 +31,6 @@ public class RecallParam {
     private String uid;
     private boolean specialRecommend;
     private Set<String> abExpCodes;
-    // 层 - 实验
-    private Map<String, String> expIdMap;
     private Integer categoryId;
 
     private Long hotSceneType;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -36,8 +36,6 @@ public class RecallService implements ApplicationContextAware {
     private final Map<String, RecallStrategy> strategyMap = new HashMap<>();
     private ApplicationContext applicationContext;
     private final ExecutorService pool = ThreadPoolFactory.recallPool();
-    @ApolloJsonValue("${last.digit.abcode:{}}")
-    protected Map<Integer, String> lastDigitAbcode;
     @ApolloJsonValue("${alg.recall.special.app&vid:{}}")
     protected Map<String, List<Long>> specialAppVid;
     @Resource
@@ -60,7 +58,7 @@ public class RecallService implements ApplicationContextAware {
 
     public RecallResult recall(RecallParam param) {
         List<RecallStrategy> strategies = getRecallStrategy(param);
-        log.info("strategies {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies, o -> o.getClass().getSimpleName())));
+        //log.info("strategies {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies,o -> o.getClass().getSimpleName())));
         CountDownLatch cdl = new CountDownLatch(strategies.size());
         List<Future<RecallResult.RecallData>> recallResultFutures = new ArrayList<>();
         for (final RecallStrategy strategy : strategies) {