|
|
@@ -0,0 +1,265 @@
|
|
|
+// package com.tzld.piaoquan.recommend.server.service.rank.strategy;
|
|
|
+//
|
|
|
+// import com.alibaba.fastjson.JSON;
|
|
|
+// import com.aliyun.openservices.eas.predict.http.HttpConfig;
|
|
|
+// import com.aliyun.openservices.eas.predict.http.PredictClient;
|
|
|
+// import com.aliyun.openservices.eas.predict.request.TFRequest;
|
|
|
+// import com.aliyun.openservices.eas.predict.response.TFResponse;
|
|
|
+// import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
|
|
|
+// import com.tzld.piaoquan.recommend.server.common.base.RankItem;
|
|
|
+// import com.tzld.piaoquan.recommend.server.model.Video;
|
|
|
+// import com.tzld.piaoquan.recommend.server.service.FeatureService;
|
|
|
+// import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
|
|
|
+// import com.tzld.piaoquan.recommend.server.service.rank.bo.UserShareReturnProfile;
|
|
|
+// import com.tzld.piaoquan.recommend.server.service.rank.tansform.FeatureFGEncoder;
|
|
|
+// import com.tzld.piaoquan.recommend.server.service.rank.tansform.FeatureV6;
|
|
|
+// import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
|
|
|
+// import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
|
|
|
+// import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
|
|
|
+// import com.tzld.piaoquan.recommend.server.util.JSONUtils;
|
|
|
+// import com.tzld.piaoquan.recommend.server.util.RecallUtils;
|
|
|
+// import lombok.extern.slf4j.Slf4j;
|
|
|
+// import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+// import org.springframework.stereotype.Service;
|
|
|
+//
|
|
|
+// import javax.annotation.PostConstruct;
|
|
|
+// import java.util.*;
|
|
|
+//
|
|
|
+// /**
|
|
|
+// * DNN 模型排序 Demo
|
|
|
+// *
|
|
|
+// * 流程:
|
|
|
+// * 1. 特征提取: 复用 FeatureV6(跟 FM/XGB 完全一样)
|
|
|
+// * 2. FG 编码: FeatureFGEncoder 按 fg_config.json 顺序编码为 chr(2) 字符串
|
|
|
+// * 3. 模型打分: 调用 EAS 服务(TODO: 接入 EAS SDK)
|
|
|
+// *
|
|
|
+// * 特征配置: resources/fg/fg_config.json(975 特征,跟离线训练一致)
|
|
|
+// */
|
|
|
+// @Service
|
|
|
+// @Slf4j
|
|
|
+// public class RankStrategy4RegionMergeModelDNNDemo extends RankStrategy4RegionMergeModelBasic {
|
|
|
+//
|
|
|
+// @ApolloJsonValue("${rank.score.merge.weightv569:}")
|
|
|
+// private Map<String, Double> mergeWeight;
|
|
|
+//
|
|
|
+// @Autowired
|
|
|
+// private FeatureService featureService;
|
|
|
+//
|
|
|
+// private FeatureFGEncoder fgEncoder;
|
|
|
+//
|
|
|
+// private PredictClient predictClient;
|
|
|
+//
|
|
|
+// @PostConstruct
|
|
|
+// public void init() {
|
|
|
+// fgEncoder = new FeatureFGEncoder("fg/fg_config.json");
|
|
|
+// log.info("DNN Demo 初始化完成: {} 个特征", fgEncoder.getFeatureCount());
|
|
|
+//
|
|
|
+// predictClient = new PredictClient(new HttpConfig());
|
|
|
+// predictClient.setModelName("recsys_dnn_v1_20260327");
|
|
|
+// predictClient.setToken("MmEwYzVlZGFiOTM4YWM3ZTE5ZDMzNTgzY2Q5YjZlYjVjODE5ZjIzYQ==");
|
|
|
+// predictClient.setEndpoint("1894469520484605.cn-hangzhou.pai-eas.aliyuncs.com");
|
|
|
+//
|
|
|
+// }
|
|
|
+//
|
|
|
+// @Override
|
|
|
+// public List<Video> mergeAndRankRovRecall(RankParam param) {
|
|
|
+// Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
|
|
|
+//
|
|
|
+// //-------------------融-------------------
|
|
|
+// //-------------------合-------------------
|
|
|
+// //-------------------逻-------------------
|
|
|
+// //-------------------辑-------------------
|
|
|
+//
|
|
|
+// long currentMs = System.currentTimeMillis();
|
|
|
+// Set<Long> setVideo = new HashSet<>();
|
|
|
+// setVideo.add(param.getHeadVid());
|
|
|
+// List<Video> rovRecallRank = new ArrayList<>();
|
|
|
+// // -------------------5路特殊旧召回------------------
|
|
|
+// RecallUtils.extractOldSpecialRecall(mergeWeight.getOrDefault("oldSpecialN", (double) param.getSize()).intValue(), param, setVideo, rovRecallRank);
|
|
|
+// //-------------------return相似召回------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("v6", 5.0).intValue(), param, ReturnVideoRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+// //-------------------新地域召回------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("v1", 5.0).intValue(), param, RegionRealtimeRecallStrategyV1.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+// //-------------------scene cf rovn------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), param, SceneCFRovnRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+// //-------------------scene cf rosn------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), param, SceneCFRosnRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+// // -------------------user cate1------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("cate1RecallN", 5.0).intValue(), param, UserCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+// // -------------------user cate2------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("cate2RecallN", 5.0).intValue(), param, UserCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+// // -------------------head province cate1------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate1RecallN", 3.0).intValue(), param, HeadProvinceCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+// // -------------------head province cate2------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate2RecallN", 3.0).intValue(), param, HeadProvinceCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+// //-------------------head cate2 of rovn------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("headCate2Rov", 5.0).intValue(), param, HeadCate2RovRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+// //-------------------city rovn------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("cityRov", 5.0).intValue(), param, CityRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+// //-------------------priori province rovn------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRov", 3.0).intValue(), param, PrioriProvinceRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+// //-------------------priori province str------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceStr", 1.0).intValue(), param, PrioriProvinceStrRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+// //-------------------priori province ros------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRos", 1.0).intValue(), param, PrioriProvinceRosRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
|
|
|
+// //-------------------return1 cate2 ros------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("return1Cate2Ros", 5.0).intValue(), param, Return1Cate2RosRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+// //-------------------return1 cate2 str------------------
|
|
|
+// RecallUtils.extractRecall(mergeWeight.getOrDefault("return1Cate2Str", 5.0).intValue(), param, Return1Cate2StrRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
|
|
|
+//
|
|
|
+// // 记录召回源中的视频
|
|
|
+// this.rankBeforePostProcessor(rovRecallRank);
|
|
|
+//
|
|
|
+// // ============ 特征提取 ============
|
|
|
+// Map<String, String> headVideoInfo = param.getHeadInfo();
|
|
|
+// List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
|
|
|
+// Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", vids);
|
|
|
+//
|
|
|
+// FeatureService.Feature feature = featureService.getFeatureV4(param, headVideoInfo, videoBaseInfoMap, vids);
|
|
|
+// Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
|
|
|
+// Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
|
|
|
+//
|
|
|
+// Map<String, Map<String, String[]>> c7Map = FeatureV6.parseUCFScore(featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>()));
|
|
|
+// Map<String, Map<String, String[]>> c8Map = FeatureV6.parseUCFScore(featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>()));
|
|
|
+// UserShareReturnProfile userProfile = parseUserProfile(featureOriginUser);
|
|
|
+// Map<String, String> creativeInfo = param.getCreativeInfoFeature();
|
|
|
+// Map<String, Map<String, String>> historyVideoMap = param.getBehaviorVideos();
|
|
|
+// Map<String, String> baseInfo = featureOriginUser.getOrDefault("mid_base_info", new HashMap<>());
|
|
|
+//
|
|
|
+// // ============ 用户级特征(所有视频共享)============
|
|
|
+// Map<String, Double> userNumericFeatures = new HashMap<>();
|
|
|
+// Map<String, String> userStringFeatures = new HashMap<>();
|
|
|
+//
|
|
|
+// // 上下文特征
|
|
|
+// String appType = Optional.of(param.getAppType()).map(String::valueOf).orElse("0");
|
|
|
+// String hotSceneType = Optional.ofNullable(param.getHotSceneType()).map(String::valueOf).orElse("other");
|
|
|
+// FeatureV6.getContextFeature(currentMs, appType, hotSceneType, userNumericFeatures);
|
|
|
+// userStringFeatures.put("ctx_app", appType);
|
|
|
+// userStringFeatures.put("ctx_hot", hotSceneType);
|
|
|
+// userStringFeatures.put("ctx_week", String.valueOf(Calendar.getInstance().get(Calendar.DAY_OF_WEEK)));
|
|
|
+// userStringFeatures.put("ctx_hour", String.valueOf(Calendar.getInstance().get(Calendar.HOUR_OF_DAY)));
|
|
|
+//
|
|
|
+// // 用户特征
|
|
|
+// FeatureV6.getUserFeature(featureOriginUser, userNumericFeatures);
|
|
|
+// FeatureV6.getUserProfileFeature(userProfile, baseInfo, userNumericFeatures);
|
|
|
+// userStringFeatures.put("mid", param.getMid());
|
|
|
+// userStringFeatures.put("user_province", baseInfo.getOrDefault("province", ""));
|
|
|
+// userStringFeatures.put("user_city", baseInfo.getOrDefault("city", ""));
|
|
|
+// userStringFeatures.put("user_brand", baseInfo.getOrDefault("brand", ""));
|
|
|
+// userStringFeatures.put("user_model", baseInfo.getOrDefault("model", ""));
|
|
|
+// userStringFeatures.put("user_system", baseInfo.getOrDefault("system", ""));
|
|
|
+// userStringFeatures.put("user_channel", baseInfo.getOrDefault("user_channel", ""));
|
|
|
+// userStringFeatures.put("user_level", baseInfo.getOrDefault("user_level", ""));
|
|
|
+//
|
|
|
+// // 创意特征
|
|
|
+// FeatureV6.getCreativeBaseFeature("e1", creativeInfo, userNumericFeatures);
|
|
|
+// userStringFeatures.put("e1_ghid", creativeInfo.getOrDefault("ghid", ""));
|
|
|
+// userStringFeatures.put("e1_name", creativeInfo.getOrDefault("name", ""));
|
|
|
+//
|
|
|
+// // 头部视频特征
|
|
|
+// FeatureV6.getVideoBaseFeature("h", currentMs, headVideoInfo, userNumericFeatures);
|
|
|
+// putVideoStringFeatures("h", headVideoInfo, userStringFeatures);
|
|
|
+//
|
|
|
+// // ============ 逐视频特征 + FG 编码 + 打分 ============
|
|
|
+// // 批量收集 FG 字符串,后续可批量调 EAS
|
|
|
+// List<TFRequest> tfRequests= new ArrayList<>(rovRecallRank.size());
|
|
|
+// List<String> fgEncodes = new ArrayList<>(rovRecallRank.size());
|
|
|
+//
|
|
|
+// List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
|
|
|
+//
|
|
|
+// for (RankItem rankItem : rankItems) {
|
|
|
+// Video video = rankItem.getVideo();
|
|
|
+//
|
|
|
+// String vid = String.valueOf(video.getVideoId());
|
|
|
+// Map<String, String> rankInfo = videoBaseInfoMap
|
|
|
+// .getOrDefault(vid, new HashMap<>())
|
|
|
+// .getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
|
|
|
+//
|
|
|
+// // 复制用户级特征
|
|
|
+// Map<String, Double> numericFeatures = new HashMap<>(userNumericFeatures);
|
|
|
+// Map<String, String> stringFeatures = new HashMap<>(userStringFeatures);
|
|
|
+//
|
|
|
+// // 当前视频特征
|
|
|
+// FeatureV6.getVideoBaseFeature("r", currentMs, rankInfo, numericFeatures);
|
|
|
+// FeatureV6.getVideoFeature(vid, featureOriginVideo, numericFeatures);
|
|
|
+// putVideoStringFeatures("r", rankInfo, stringFeatures);
|
|
|
+// stringFeatures.put("r_vid_source", rankInfo.getOrDefault("vid_source", ""));
|
|
|
+//
|
|
|
+// // CF 特征
|
|
|
+// FeatureV6.getUserTagsCrossVideoFeature("c5", rankInfo, featureOriginUser.get("alg_mid_feature_return_tags"), numericFeatures);
|
|
|
+// FeatureV6.getUserTagsCrossVideoFeature("c6", rankInfo, featureOriginUser.get("alg_mid_feature_share_tags"), numericFeatures);
|
|
|
+// FeatureV6.getUserCFFeature("c7", vid, c7Map, numericFeatures);
|
|
|
+// FeatureV6.getUserCFFeature("c8", vid, c8Map, numericFeatures);
|
|
|
+//
|
|
|
+// // 交叉特征
|
|
|
+// FeatureV6.getHeadRankVideoCrossFeature(headVideoInfo, rankInfo, numericFeatures);
|
|
|
+// FeatureV6.getCreativeCrossFeature("e1", creativeInfo, rankInfo, numericFeatures);
|
|
|
+// FeatureV6.getProfileVideoCrossFeature(currentMs, userProfile, rankInfo, historyVideoMap, numericFeatures);
|
|
|
+//
|
|
|
+// rankItem.setFeatureMapDouble(numericFeatures);
|
|
|
+// rankItem.setFeatureMapString(stringFeatures);
|
|
|
+//
|
|
|
+// tfRequests.add(fgEncoder.tfRequests(numericFeatures, stringFeatures));
|
|
|
+// }
|
|
|
+//
|
|
|
+// // ============ 调用 EAS 打分(TODO)============
|
|
|
+// // List<Double> scores = easClient.predict(fgFeaturesList);
|
|
|
+// // 临时: 用随机分数
|
|
|
+//
|
|
|
+// for (int i = 0; i < rovRecallRank.size(); i++) {
|
|
|
+// Video video = rovRecallRank.get(i);
|
|
|
+//
|
|
|
+// try {
|
|
|
+// TFRequest tfRequest = tfRequests.get(i);
|
|
|
+// TFResponse tfPredict = predictClient.predict(tfRequest);
|
|
|
+// System.out.println(tfPredict);
|
|
|
+// } catch (Exception e) {
|
|
|
+// log.error("dnn predict error videoId: {}", video.getVideoId(), e);
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// for (int i = 0; i < rovRecallRank.size(); i++) {
|
|
|
+// Video video = rovRecallRank.get(i);
|
|
|
+// double score = Math.random(); // TODO: 替换为 EAS 返回的分数
|
|
|
+// video.setScore(score);
|
|
|
+// video.setSortScore(score);
|
|
|
+// }
|
|
|
+//
|
|
|
+// rovRecallRank.sort((a, b) -> Double.compare(b.getSortScore(), a.getSortScore()));
|
|
|
+// return rovRecallRank;
|
|
|
+// }
|
|
|
+//
|
|
|
+// /**
|
|
|
+// * 视频 ID 类特征放入 stringFeatures
|
|
|
+// */
|
|
|
+// private void putVideoStringFeatures(String prefix, Map<String, String> videoInfo, Map<String, String> stringFeatures) {
|
|
|
+// stringFeatures.put(prefix + "_resolution", videoInfo.getOrDefault("resolution", ""));
|
|
|
+// stringFeatures.put(prefix + "_time_type", videoInfo.getOrDefault("time_type", ""));
|
|
|
+// stringFeatures.put(prefix + "_cate2", videoInfo.getOrDefault("merge_second_level_cate", ""));
|
|
|
+// stringFeatures.put(prefix + "_cate1_list", videoInfo.getOrDefault("merge_first_level_cate", ""));
|
|
|
+// stringFeatures.put(prefix + "_festive_label1", videoInfo.getOrDefault("festive_label1", ""));
|
|
|
+// stringFeatures.put(prefix + "_channel", videoInfo.getOrDefault("channel", ""));
|
|
|
+// stringFeatures.put(prefix + "_vid_source", videoInfo.getOrDefault("vid_source", ""));
|
|
|
+// stringFeatures.put(prefix + "_uid", videoInfo.getOrDefault("uid", ""));
|
|
|
+// stringFeatures.put(prefix + "_keywords", videoInfo.getOrDefault("keywords", ""));
|
|
|
+// stringFeatures.put(prefix + "_merge_first_level_cate", videoInfo.getOrDefault("merge_first_level_cate", ""));
|
|
|
+// stringFeatures.put(prefix + "_merge_second_level_cate", videoInfo.getOrDefault("merge_second_level_cate", ""));
|
|
|
+// }
|
|
|
+//
|
|
|
+// private UserShareReturnProfile parseUserProfile(Map<String, Map<String, String>> userOriginInfo) {
|
|
|
+// if (null != userOriginInfo) {
|
|
|
+// Map<String, String> c9 = userOriginInfo.get("alg_recsys_feature_user_share_return_stat");
|
|
|
+// if (null != c9 && !c9.isEmpty()) {
|
|
|
+// String c9Str = JSONUtils.toJson(c9);
|
|
|
+// if (!c9Str.isEmpty()) {
|
|
|
+// try {
|
|
|
+// return JSON.parseObject(c9Str, UserShareReturnProfile.class);
|
|
|
+// } catch (Exception e) {
|
|
|
+// log.error("parseObject user profile error! value=[{}]", c9Str, e);
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// return null;
|
|
|
+// }
|
|
|
+// }
|