Merge branch 'feature_model' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/AbstractScorer.java

@@ -10,6 +10,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -37,7 +38,7 @@ public abstract class AbstractScorer {
         if (StringUtils.isNotBlank(modelPath)) {
             try {
                 // 使用 modelPath 作为 modelName 注册
-                modelManager.registerModel(modelPath, modelPath, modelClass);
+                modelManager.registerModel(modelPath, modelPath, modelClass, Collections.emptyMap());
                 LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
             } catch (ModelManager.ModelRegisterException e) {
                 LOGGER.error("register model fail [{}]:[{}]", modelPath, e);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/VideoSourceTypeStrategy.java

@@ -154,11 +154,11 @@ public class VideoSourceTypeStrategy implements FilterStrategy {
                         || (vid2UidMap.containsKey(l) && notUserUploadUserIds.contains(vid2UidMap.get(l))))
                 .collect(Collectors.toList());
 
-        log.info("VideoSourceTypeStrategy \t param={} \t before={} \t " +
-                        "after={}",
-                JSONUtils.toJson(param),
-                JSONUtils.toJson(param.getVideoIds()),
-                JSONUtils.toJson(videoIds));
+//        log.info("VideoSourceTypeStrategy \t param={} \t before={} \t " +
+//                        "after={}",
+//                JSONUtils.toJson(param),
+//                JSONUtils.toJson(param.getVideoIds()),
+//                JSONUtils.toJson(videoIds));
 
         return videoIds;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/AbstractScorer.java

@@ -1,6 +1,7 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
 
+import com.typesafe.config.ConfigObject;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.service.score.model.Model;
@@ -10,6 +11,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -37,7 +39,9 @@ public abstract class AbstractScorer {
         if (StringUtils.isNotBlank(modelPath)) {
             try {
                 // 使用 modelPath 作为 modelName 注册
-                modelManager.registerModel(modelPath, modelPath, modelClass);
+                ConfigObject paramMap = scorerConfigInfo.getParamMap();
+                modelManager.registerModel(modelPath, modelPath, modelClass, paramMap == null ?
+                        Collections.emptyMap() : paramMap.unwrapped());
                 LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
             } catch (ModelManager.ModelRegisterException e) {
                 LOGGER.error("register model fail [{}]:[{}]", modelPath, e);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/Model.java

@@ -3,13 +3,22 @@ package com.tzld.piaoquan.recommend.server.service.score.model;
 
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.Map;
+
+public abstract class Model {
+    protected Map<String, Object> params;
 
-abstract public class Model {
     public abstract int getModelSize();
 
     public abstract boolean loadFromStream(InputStreamReader in) throws Exception;
+
     public boolean loadFromStream(InputStream is) throws Exception {
         return loadFromStream(new InputStreamReader(is));
     }
+
+    public void setParams(Map<String, Object> params) {
+        this.params = params;
+
+    }
 }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/ModelManager.java

@@ -83,7 +83,8 @@ public class ModelManager {
      * @param path       Model在OSS上的全路径
      * @param modelClass Model的子类型
      */
-    public void registerModel(String modelName, String path, Class<? extends Model> modelClass) throws ModelRegisterException, IOException {
+    public void registerModel(String modelName, String path, Class<? extends Model> modelClass,
+                              Map<String, Object> params) throws ModelRegisterException, IOException {
         if (modelPathMap.containsKey(modelName)) {
             // fail fast
             // throw new RuntimeException(modelName + " already exists");
@@ -96,7 +97,7 @@ public class ModelManager {
             ModelLoadTask loadTask = loadTasks.get(path);
             loadTask.refCount++;
         } else {
-            ModelLoadTask task = new ModelLoadTask(path, modelClass);
+            ModelLoadTask task = new ModelLoadTask(path, modelClass, params);
             task.refCount++;
             loadTasks.put(path, task);
             loadModelWithRetry(task, false, true);
@@ -205,6 +206,7 @@ public class ModelManager {
                         loadTask.lastModifyTime, timeStamp);
 
                 Model model = loadTask.modelClass.newInstance();
+                model.setParams(loadTask.params);
                 if (model.loadFromStream(ossObj.getObjectContent())) {
                     loadTask.model = model;
                     loadTask.lastModifyTime = timeStamp;
@@ -245,12 +247,14 @@ public class ModelManager {
         private boolean isLoading;
         private final Class<? extends Model> modelClass;
         private Model model;
+        private Map<String, Object> params;
 
-        ModelLoadTask(String path, Class<? extends Model> modelClass) {
+        ModelLoadTask(String path, Class<? extends Model> modelClass, Map<String, Object> params) {
             this.refCount = 0;
             this.path = path;
             this.lastModifyTime = 0;
             this.modelClass = modelClass;
+            this.params = params;
         }
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/XGBoostModel.java

@@ -6,6 +6,7 @@ import com.tzld.piaoquan.recommend.server.util.PropertiesUtil;
 import ml.dmlc.xgboost4j.scala.DMatrix;
 import ml.dmlc.xgboost4j.scala.spark.XGBoostClassificationModel;
 import org.apache.commons.lang.math.NumberUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -13,6 +14,7 @@ import java.io.File;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.Map;
+import java.util.UUID;
 
 
 public class XGBoostModel extends Model {
@@ -21,7 +23,7 @@ public class XGBoostModel extends Model {
 
     private String[] features;
 
-    public void setFeatures(String[] features){
+    public void setFeatures(String[] features) {
         this.features = features;
     }
 
@@ -59,9 +61,11 @@ public class XGBoostModel extends Model {
 
     @Override
     public boolean loadFromStream(InputStream in) throws Exception {
-        String modelDir = PropertiesUtil.getString("model.xgboost.path");
+        Object localDir = params.getOrDefault("localDir",
+                PropertiesUtil.getString("model.xgboost.path") + "/" + UUID.randomUUID());
+        String modelDir = String.valueOf(localDir);
         CompressUtil.decompressGzFile(in, modelDir);
-        String absolutePath =new File(modelDir).getAbsolutePath();
+        String absolutePath = new File(modelDir).getAbsolutePath();
         XGBoostClassificationModel model2 = XGBoostClassificationModel.load("file://" + absolutePath);
         model2.setMissing(0.0f);
         this.model = model2;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/AbstractScorer4Recall.java

@@ -1,6 +1,7 @@
 package com.tzld.piaoquan.recommend.server.service.score4recall;
 
 
+import com.typesafe.config.ConfigObject;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerConfigInfo;
 import com.tzld.piaoquan.recommend.server.service.score.model.Model;
 import com.tzld.piaoquan.recommend.server.service.score.model.ModelManager;
@@ -10,6 +11,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -36,7 +38,9 @@ public abstract class AbstractScorer4Recall {
         if (StringUtils.isNotBlank(modelPath)) {
             try {
                 // 使用 modelPath 作为 modelName 注册
-                modelManager.registerModel(modelPath, modelPath, modelClass);
+                ConfigObject paramMap = scorerConfigInfo.getParamMap();
+                modelManager.registerModel(modelPath, modelPath, modelClass,
+                        paramMap == null ? Collections.emptyMap() : paramMap.unwrapped());
                 LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
             } catch (ModelManager.ModelRegisterException e) {
                 LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
@@ -47,6 +51,7 @@ public abstract class AbstractScorer4Recall {
             LOGGER.error("modelpath is null, for model class [{}]", modelClass);
         }
     }
+
     public Model getModel() {
         if (StringUtils.isBlank(scorerConfigInfo.getModelPath())) {
             return null;

recommend-server-service/src/main/resources/feeds_score_config_xgb_20240828.conf

@@ -4,6 +4,7 @@ scorer-config = {
     scorer-priority = 99
     model-path = "zhangbo/model_xgb_for_recsys.tar.gz"
     param = {
+      localDir = "xgboost/recsys"
       features = [
         "b123_1h_STR",
         "b123_1h_log(share)",