recommend-server-client/pom.xml

@@ -10,7 +10,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>recommend-server-client</artifactId>
-    <version>1.1.4</version>
+    <version>1.1.8</version>
 
     <dependencies>
         <dependency>

recommend-server-client/src/main/java/com/tzld/piaoquan/recommend/server/gen/recommend/Recommend.java

@@ -56,7 +56,7 @@ public final class Recommend {
       "\n2com/tzld/piaoquan/recommend/server/rec" +
       "ommend.proto\032\031google/protobuf/any.proto\032" +
       "/com/tzld/piaoquan/recommend/server/comm" +
-      "on.proto\"\256\004\n\020RecommendRequest\022\022\n\nrequest" +
+      "on.proto\"\205\005\n\020RecommendRequest\022\022\n\nrequest" +
       "_id\030\001 \001(\t\022\013\n\003mid\030\002 \001(\t\022\013\n\003uid\030\003 \001(\t\022\014\n\004s" +
       "ize\030\004 \001(\005\022\020\n\010app_type\030\005 \001(\005\022\021\n\tcity_code" +
       "\030\006 \001(\t\022\025\n\rprovince_code\030\007 \001(\t\022\023\n\013ab_exp_" +
@@ -69,25 +69,27 @@ public final class Recommend {
       "ion_id\030\021 \001(\t\022\026\n\016sub_session_id\030\022 \001(\t\022\023\n\013" +
       "page_source\030\023 \001(\t\022\023\n\013category_id\030\024 \001(\t\022\026" +
       "\n\016hot_scene_type\030\025 \001(\003\022\021\n\tclient_ip\030\026 \001(" +
-      "\t\032.\n\014EventIdEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030" +
-      "\002 \001(\t:\0028\001\"\177\n\020MachineInfoProto\022\r\n\005brand\030\001" +
-      " \001(\t\022\r\n\005model\030\002 \001(\t\022\020\n\010platform\030\003 \001(\t\022\023\n" +
-      "\013sdk_version\030\004 \001(\t\022\016\n\006system\030\005 \001(\t\022\026\n\016we" +
-      "chat_version\030\006 \001(\t\"H\n\021RecommendResponse\022" +
-      "\027\n\006result\030\001 \001(\0132\007.Result\022\032\n\005video\030\002 \003(\0132" +
-      "\013.VideoProto\"\336\001\n\nVideoProto\022\020\n\010video_id\030" +
-      "\001 \001(\003\022\021\n\trov_score\030\002 \001(\001\022\021\n\tpush_from\030\003 " +
-      "\001(\t\022\017\n\007ab_code\030\004 \001(\t\022\022\n\nsort_score\030\005 \001(\001" +
-      "\022\020\n\010position\030\006 \001(\005\022\021\n\tflow_pool\030\007 \001(\t\022\027\n" +
-      "\017is_in_flow_pool\030\010 \001(\005\022\014\n\004rand\030\t \001(\001\022\'\n\017" +
-      "push_from_index\030\n \003(\0132\016.PushFromIndex\"1\n" +
-      "\rPushFromIndex\022\021\n\tpush_from\030\001 \001(\t\022\r\n\005ind" +
-      "ex\030\002 \003(\t2\212\001\n\020RecommendService\022:\n\021Homepag" +
-      "eRecommend\022\021.RecommendRequest\032\022.Recommen" +
-      "dResponse\022:\n\021RelevantRecommend\022\021.Recomme" +
-      "ndRequest\032\022.RecommendResponseB7\n0com.tzl" +
-      "d.piaoquan.recommend.server.gen.recommen" +
-      "dP\001\210\001\001b\006proto3"
+      "\t\022\024\n\014version_code\030\027 \001(\005\022\026\n\016root_source_i" +
+      "d\030\030 \001(\t\022\026\n\016userShareDepth\030\031 \001(\005\022\017\n\007pageN" +
+      "um\030\032 \001(\005\032.\n\014EventIdEntry\022\013\n\003key\030\001 \001(\t\022\r\n" +
+      "\005value\030\002 \001(\t:\0028\001\"\177\n\020MachineInfoProto\022\r\n\005" +
+      "brand\030\001 \001(\t\022\r\n\005model\030\002 \001(\t\022\020\n\010platform\030\003" +
+      " \001(\t\022\023\n\013sdk_version\030\004 \001(\t\022\016\n\006system\030\005 \001(" +
+      "\t\022\026\n\016wechat_version\030\006 \001(\t\"H\n\021RecommendRe" +
+      "sponse\022\027\n\006result\030\001 \001(\0132\007.Result\022\032\n\005video" +
+      "\030\002 \003(\0132\013.VideoProto\"\336\001\n\nVideoProto\022\020\n\010vi" +
+      "deo_id\030\001 \001(\003\022\021\n\trov_score\030\002 \001(\001\022\021\n\tpush_" +
+      "from\030\003 \001(\t\022\017\n\007ab_code\030\004 \001(\t\022\022\n\nsort_scor" +
+      "e\030\005 \001(\001\022\020\n\010position\030\006 \001(\005\022\021\n\tflow_pool\030\007" +
+      " \001(\t\022\027\n\017is_in_flow_pool\030\010 \001(\005\022\014\n\004rand\030\t " +
+      "\001(\001\022\'\n\017push_from_index\030\n \003(\0132\016.PushFromI" +
+      "ndex\"1\n\rPushFromIndex\022\021\n\tpush_from\030\001 \001(\t" +
+      "\022\r\n\005index\030\002 \003(\t2\212\001\n\020RecommendService\022:\n\021" +
+      "HomepageRecommend\022\021.RecommendRequest\032\022.R" +
+      "ecommendResponse\022:\n\021RelevantRecommend\022\021." +
+      "RecommendRequest\032\022.RecommendResponseB7\n0" +
+      "com.tzld.piaoquan.recommend.server.gen.r" +
+      "ecommendP\001\210\001\001b\006proto3"
     };
     descriptor = com.google.protobuf.Descriptors.FileDescriptor
       .internalBuildGeneratedFileFrom(descriptorData,
@@ -100,7 +102,7 @@ public final class Recommend {
     internal_static_RecommendRequest_fieldAccessorTable = new
       com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
         internal_static_RecommendRequest_descriptor,
-        new java.lang.String[] { "RequestId", "Mid", "Uid", "Size", "AppType", "CityCode", "ProvinceCode", "AbExpCode", "EventId", "VersionAuditStatus", "RecommendTraceId", "VideoId", "City", "Province", "MachineInfo", "NewExpGroup", "SessionId", "SubSessionId", "PageSource", "CategoryId", "HotSceneType", "ClientIp", });
+        new java.lang.String[] { "RequestId", "Mid", "Uid", "Size", "AppType", "CityCode", "ProvinceCode", "AbExpCode", "EventId", "VersionAuditStatus", "RecommendTraceId", "VideoId", "City", "Province", "MachineInfo", "NewExpGroup", "SessionId", "SubSessionId", "PageSource", "CategoryId", "HotSceneType", "ClientIp", "VersionCode", "RootSourceId", "UserShareDepth", "PageNum", });
     internal_static_RecommendRequest_EventIdEntry_descriptor =
       internal_static_RecommendRequest_descriptor.getNestedTypes().get(0);
     internal_static_RecommendRequest_EventIdEntry_fieldAccessorTable = new

recommend-server-client/src/main/java/com/tzld/piaoquan/recommend/server/gen/recommend/RecommendRequest.java

@@ -31,6 +31,7 @@ private static final long serialVersionUID = 0L;
     pageSource_ = "";
     categoryId_ = "";
     clientIp_ = "";
+    rootSourceId_ = "";
   }
 
   @java.lang.Override
@@ -208,6 +209,27 @@ private static final long serialVersionUID = 0L;
             clientIp_ = s;
             break;
           }
+          case 184: {
+
+            versionCode_ = input.readInt32();
+            break;
+          }
+          case 194: {
+            java.lang.String s = input.readStringRequireUtf8();
+
+            rootSourceId_ = s;
+            break;
+          }
+          case 200: {
+
+            userShareDepth_ = input.readInt32();
+            break;
+          }
+          case 208: {
+
+            pageNum_ = input.readInt32();
+            break;
+          }
           default: {
             if (!parseUnknownField(
                 input, unknownFields, extensionRegistry, tag)) {
@@ -996,6 +1018,81 @@ private static final long serialVersionUID = 0L;
     }
   }
 
+  public static final int VERSION_CODE_FIELD_NUMBER = 23;
+  private int versionCode_;
+  /**
+   * <code>int32 version_code = 23;</code>
+   * @return The versionCode.
+   */
+  @java.lang.Override
+  public int getVersionCode() {
+    return versionCode_;
+  }
+
+  public static final int ROOT_SOURCE_ID_FIELD_NUMBER = 24;
+  private volatile java.lang.Object rootSourceId_;
+  /**
+   * <code>string root_source_id = 24;</code>
+   * @return The rootSourceId.
+   */
+  @java.lang.Override
+  public java.lang.String getRootSourceId() {
+    java.lang.Object ref = rootSourceId_;
+    if (ref instanceof java.lang.String) {
+      return (java.lang.String) ref;
+    } else {
+      com.google.protobuf.ByteString bs = 
+          (com.google.protobuf.ByteString) ref;
+      java.lang.String s = bs.toStringUtf8();
+      rootSourceId_ = s;
+      return s;
+    }
+  }
+  /**
+   * <code>string root_source_id = 24;</code>
+   * @return The bytes for rootSourceId.
+   */
+  @java.lang.Override
+  public com.google.protobuf.ByteString
+      getRootSourceIdBytes() {
+    java.lang.Object ref = rootSourceId_;
+    if (ref instanceof java.lang.String) {
+      com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString.copyFromUtf8(
+              (java.lang.String) ref);
+      rootSourceId_ = b;
+      return b;
+    } else {
+      return (com.google.protobuf.ByteString) ref;
+    }
+  }
+
+  public static final int USERSHAREDEPTH_FIELD_NUMBER = 25;
+  private int userShareDepth_;
+  /**
+   * <pre>
+   * default -1
+   * </pre>
+   *
+   * <code>int32 userShareDepth = 25;</code>
+   * @return The userShareDepth.
+   */
+  @java.lang.Override
+  public int getUserShareDepth() {
+    return userShareDepth_;
+  }
+
+  public static final int PAGENUM_FIELD_NUMBER = 26;
+  private int pageNum_;
+  /**
+   * <code>int32 pageNum = 26;</code>
+   * @return The pageNum.
+   */
+  @java.lang.Override
+  public int getPageNum() {
+    return pageNum_;
+  }
+
   private byte memoizedIsInitialized = -1;
   @java.lang.Override
   public final boolean isInitialized() {
@@ -1079,6 +1176,18 @@ private static final long serialVersionUID = 0L;
     if (!getClientIpBytes().isEmpty()) {
       com.google.protobuf.GeneratedMessageV3.writeString(output, 22, clientIp_);
     }
+    if (versionCode_ != 0) {
+      output.writeInt32(23, versionCode_);
+    }
+    if (!getRootSourceIdBytes().isEmpty()) {
+      com.google.protobuf.GeneratedMessageV3.writeString(output, 24, rootSourceId_);
+    }
+    if (userShareDepth_ != 0) {
+      output.writeInt32(25, userShareDepth_);
+    }
+    if (pageNum_ != 0) {
+      output.writeInt32(26, pageNum_);
+    }
     unknownFields.writeTo(output);
   }
 
@@ -1172,6 +1281,21 @@ private static final long serialVersionUID = 0L;
     if (!getClientIpBytes().isEmpty()) {
       size += com.google.protobuf.GeneratedMessageV3.computeStringSize(22, clientIp_);
     }
+    if (versionCode_ != 0) {
+      size += com.google.protobuf.CodedOutputStream
+        .computeInt32Size(23, versionCode_);
+    }
+    if (!getRootSourceIdBytes().isEmpty()) {
+      size += com.google.protobuf.GeneratedMessageV3.computeStringSize(24, rootSourceId_);
+    }
+    if (userShareDepth_ != 0) {
+      size += com.google.protobuf.CodedOutputStream
+        .computeInt32Size(25, userShareDepth_);
+    }
+    if (pageNum_ != 0) {
+      size += com.google.protobuf.CodedOutputStream
+        .computeInt32Size(26, pageNum_);
+    }
     size += unknownFields.getSerializedSize();
     memoizedSize = size;
     return size;
@@ -1234,6 +1358,14 @@ private static final long serialVersionUID = 0L;
         != other.getHotSceneType()) return false;
     if (!getClientIp()
         .equals(other.getClientIp())) return false;
+    if (getVersionCode()
+        != other.getVersionCode()) return false;
+    if (!getRootSourceId()
+        .equals(other.getRootSourceId())) return false;
+    if (getUserShareDepth()
+        != other.getUserShareDepth()) return false;
+    if (getPageNum()
+        != other.getPageNum()) return false;
     if (!unknownFields.equals(other.unknownFields)) return false;
     return true;
   }
@@ -1297,6 +1429,14 @@ private static final long serialVersionUID = 0L;
         getHotSceneType());
     hash = (37 * hash) + CLIENT_IP_FIELD_NUMBER;
     hash = (53 * hash) + getClientIp().hashCode();
+    hash = (37 * hash) + VERSION_CODE_FIELD_NUMBER;
+    hash = (53 * hash) + getVersionCode();
+    hash = (37 * hash) + ROOT_SOURCE_ID_FIELD_NUMBER;
+    hash = (53 * hash) + getRootSourceId().hashCode();
+    hash = (37 * hash) + USERSHAREDEPTH_FIELD_NUMBER;
+    hash = (53 * hash) + getUserShareDepth();
+    hash = (37 * hash) + PAGENUM_FIELD_NUMBER;
+    hash = (53 * hash) + getPageNum();
     hash = (29 * hash) + unknownFields.hashCode();
     memoizedHashCode = hash;
     return hash;
@@ -1499,6 +1639,14 @@ private static final long serialVersionUID = 0L;
 
       clientIp_ = "";
 
+      versionCode_ = 0;
+
+      rootSourceId_ = "";
+
+      userShareDepth_ = 0;
+
+      pageNum_ = 0;
+
       return this;
     }
 
@@ -1557,6 +1705,10 @@ private static final long serialVersionUID = 0L;
       result.categoryId_ = categoryId_;
       result.hotSceneType_ = hotSceneType_;
       result.clientIp_ = clientIp_;
+      result.versionCode_ = versionCode_;
+      result.rootSourceId_ = rootSourceId_;
+      result.userShareDepth_ = userShareDepth_;
+      result.pageNum_ = pageNum_;
       onBuilt();
       return result;
     }
@@ -1691,6 +1843,19 @@ private static final long serialVersionUID = 0L;
         clientIp_ = other.clientIp_;
         onChanged();
       }
+      if (other.getVersionCode() != 0) {
+        setVersionCode(other.getVersionCode());
+      }
+      if (!other.getRootSourceId().isEmpty()) {
+        rootSourceId_ = other.rootSourceId_;
+        onChanged();
+      }
+      if (other.getUserShareDepth() != 0) {
+        setUserShareDepth(other.getUserShareDepth());
+      }
+      if (other.getPageNum() != 0) {
+        setPageNum(other.getPageNum());
+      }
       this.mergeUnknownFields(other.unknownFields);
       onChanged();
       return this;
@@ -3332,6 +3497,187 @@ private static final long serialVersionUID = 0L;
       onChanged();
       return this;
     }
+
+    private int versionCode_ ;
+    /**
+     * <code>int32 version_code = 23;</code>
+     * @return The versionCode.
+     */
+    @java.lang.Override
+    public int getVersionCode() {
+      return versionCode_;
+    }
+    /**
+     * <code>int32 version_code = 23;</code>
+     * @param value The versionCode to set.
+     * @return This builder for chaining.
+     */
+    public Builder setVersionCode(int value) {
+      
+      versionCode_ = value;
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>int32 version_code = 23;</code>
+     * @return This builder for chaining.
+     */
+    public Builder clearVersionCode() {
+      
+      versionCode_ = 0;
+      onChanged();
+      return this;
+    }
+
+    private java.lang.Object rootSourceId_ = "";
+    /**
+     * <code>string root_source_id = 24;</code>
+     * @return The rootSourceId.
+     */
+    public java.lang.String getRootSourceId() {
+      java.lang.Object ref = rootSourceId_;
+      if (!(ref instanceof java.lang.String)) {
+        com.google.protobuf.ByteString bs =
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        rootSourceId_ = s;
+        return s;
+      } else {
+        return (java.lang.String) ref;
+      }
+    }
+    /**
+     * <code>string root_source_id = 24;</code>
+     * @return The bytes for rootSourceId.
+     */
+    public com.google.protobuf.ByteString
+        getRootSourceIdBytes() {
+      java.lang.Object ref = rootSourceId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        rootSourceId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    /**
+     * <code>string root_source_id = 24;</code>
+     * @param value The rootSourceId to set.
+     * @return This builder for chaining.
+     */
+    public Builder setRootSourceId(
+        java.lang.String value) {
+      if (value == null) {
+    throw new NullPointerException();
+  }
+  
+      rootSourceId_ = value;
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>string root_source_id = 24;</code>
+     * @return This builder for chaining.
+     */
+    public Builder clearRootSourceId() {
+      
+      rootSourceId_ = getDefaultInstance().getRootSourceId();
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>string root_source_id = 24;</code>
+     * @param value The bytes for rootSourceId to set.
+     * @return This builder for chaining.
+     */
+    public Builder setRootSourceIdBytes(
+        com.google.protobuf.ByteString value) {
+      if (value == null) {
+    throw new NullPointerException();
+  }
+  checkByteStringIsUtf8(value);
+      
+      rootSourceId_ = value;
+      onChanged();
+      return this;
+    }
+
+    private int userShareDepth_ ;
+    /**
+     * <pre>
+     * default -1
+     * </pre>
+     *
+     * <code>int32 userShareDepth = 25;</code>
+     * @return The userShareDepth.
+     */
+    @java.lang.Override
+    public int getUserShareDepth() {
+      return userShareDepth_;
+    }
+    /**
+     * <pre>
+     * default -1
+     * </pre>
+     *
+     * <code>int32 userShareDepth = 25;</code>
+     * @param value The userShareDepth to set.
+     * @return This builder for chaining.
+     */
+    public Builder setUserShareDepth(int value) {
+      
+      userShareDepth_ = value;
+      onChanged();
+      return this;
+    }
+    /**
+     * <pre>
+     * default -1
+     * </pre>
+     *
+     * <code>int32 userShareDepth = 25;</code>
+     * @return This builder for chaining.
+     */
+    public Builder clearUserShareDepth() {
+      
+      userShareDepth_ = 0;
+      onChanged();
+      return this;
+    }
+
+    private int pageNum_ ;
+    /**
+     * <code>int32 pageNum = 26;</code>
+     * @return The pageNum.
+     */
+    @java.lang.Override
+    public int getPageNum() {
+      return pageNum_;
+    }
+    /**
+     * <code>int32 pageNum = 26;</code>
+     * @param value The pageNum to set.
+     * @return This builder for chaining.
+     */
+    public Builder setPageNum(int value) {
+      
+      pageNum_ = value;
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>int32 pageNum = 26;</code>
+     * @return This builder for chaining.
+     */
+    public Builder clearPageNum() {
+      
+      pageNum_ = 0;
+      onChanged();
+      return this;
+    }
     @java.lang.Override
     public final Builder setUnknownFields(
         final com.google.protobuf.UnknownFieldSet unknownFields) {

recommend-server-client/src/main/java/com/tzld/piaoquan/recommend/server/gen/recommend/RecommendRequestOrBuilder.java

@@ -290,4 +290,38 @@ public interface RecommendRequestOrBuilder extends
    */
   com.google.protobuf.ByteString
       getClientIpBytes();
+
+  /**
+   * <code>int32 version_code = 23;</code>
+   * @return The versionCode.
+   */
+  int getVersionCode();
+
+  /**
+   * <code>string root_source_id = 24;</code>
+   * @return The rootSourceId.
+   */
+  java.lang.String getRootSourceId();
+  /**
+   * <code>string root_source_id = 24;</code>
+   * @return The bytes for rootSourceId.
+   */
+  com.google.protobuf.ByteString
+      getRootSourceIdBytes();
+
+  /**
+   * <pre>
+   * default -1
+   * </pre>
+   *
+   * <code>int32 userShareDepth = 25;</code>
+   * @return The userShareDepth.
+   */
+  int getUserShareDepth();
+
+  /**
+   * <code>int32 pageNum = 26;</code>
+   * @return The pageNum.
+   */
+  int getPageNum();
 }

recommend-server-client/src/main/proto/com/tzld/piaoquan/recommend/server/recommend.proto

@@ -31,6 +31,10 @@ message RecommendRequest {
   string category_id = 20;
   int64 hot_scene_type = 21;
   string client_ip = 22;
+  int32 version_code = 23;
+  string root_source_id = 24;
+  int32 userShareDepth = 25; // default -1
+  int32 pageNum = 26;
 }
 
 message MachineInfoProto {

recommend-server-service/pom.xml

@@ -175,7 +175,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-server-client</artifactId>
-            <version>1.1.4</version>
+            <version>1.1.8</version>
         </dependency>
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/Application.java

@@ -1,6 +1,5 @@
 package com.tzld.piaoquan.recommend.server;
 
-// import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
 
 import com.tzld.piaoquan.abtest.client.ABTestClient;
 import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
@@ -26,8 +25,6 @@ import org.springframework.scheduling.annotation.EnableScheduling;
         "com.tzld.piaoquan.recommend.server.util",
         "com.tzld.piaoquan.recommend.server.config",
         "com.tzld.piaoquan.recommend.server.service",
-        "com.tzld.piaoquan.recommend.server.implement",
-        "com.tzld.piaoquan.recommend.server.framework.utils",
         "com.tzld.piaoquan.recommend.server.grpcservice",
         "com.tzld.piaoquan.recommend.server.remote",
         "com.tzld.piaoquan.recommend.server.web",
@@ -42,21 +39,11 @@ public class Application {
     }
 
 
-    @Bean
-    public FeatureClient featureClient() {
-        return new FeatureClient();
-    }
-
     @Bean
     public FeatureV2Client featureV2Client() {
         return new FeatureV2Client();
     }
 
-    @Bean
-    public ABTestClient abTestClient() {
-        return new ABTestClient();
-    }
-
     @Bean
     public ProtobufHttpMessageConverter protobufHttpMessageConverter() {
         return new ProtobufHttpMessageConverter();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/RedisKeyConstants.java

@@ -0,0 +1,12 @@
+package com.tzld.piaoquan.recommend.server.common;
+
+/**
+ * @author dyp
+ */
+public class RedisKeyConstants {
+
+    public static class Recommend{
+        public static String riskUserMid = "risk:user:mid";
+        public static String riskUserUid = "risk:user:uid";
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/Constant.java

@@ -1,27 +0,0 @@
-package com.tzld.piaoquan.recommend.server.common.base;
-
-/**
- * 常量
- *
- * @author supeng
- * @date 2020/08/19
- */
-public class Constant {
-    /**
-     * traceID
-     */
-    public static final String LOG_TRACE_ID = "logTraceId";
-    /**
-     * 流量池头部视频redis key
-     */
-    public static final String VIDEO_PERFORMANCE_DATA_REDIS_KEY = "video_performance_data_redis_key:";
-    /**
-     * 供给流量池实验 648 random
-     */
-    public static final String SUPPLY_AB_CODE = "60600";
-    /**
-     * 供给流量池实验 648 random
-     */
-    public static final String SUPPLY_AB_CODE_ID= "648";
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -2,13 +2,10 @@ package com.tzld.piaoquan.recommend.server.common.base;
 
 import com.google.common.collect.Maps;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.CandidateInfo;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import lombok.Data;
 
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 
@@ -17,6 +14,8 @@ public class RankItem implements Comparable<RankItem> {
 
     // featureMap中保存所有的特征
     public Map<String, String> featureMap = new HashMap<>();
+    public Map<String, String> norFeatureMap = new HashMap<>();
+    public Map<String, String> rosFeatureMap = new HashMap<>();
     // 所有特征，包括视频、用户等等
     public Map<String, String> allFeatureMap = new HashMap<>();
     public Map<String, Double> featureMapDouble = new HashMap<>();
@@ -38,12 +37,6 @@ public class RankItem implements Comparable<RankItem> {
 
     // 记录召回信息
     private String queue;
-    private CandidateInfo candidateInfo;
-    private List<CandidateInfo> candidateInfoList = new ArrayList<>(); // 兼容多个召回key命中
-
-    // merge信息
-    private List<String> mergeQueuePath = new ArrayList<>();
-    private List<String> mergeDecisionLabels = new ArrayList<>();
 
     // 多样性过滤因子
     // 排序因子
@@ -57,10 +50,6 @@ public class RankItem implements Comparable<RankItem> {
     private Map<String, Double> rankerScore = Maps.newHashMap();
     private Map<String, Integer> rankerIndex = Maps.newHashMap();
 
-    public RankItem() {
-
-    }
-
 
     public RankItem(Video video) {
         this.id = String.valueOf(video.getVideoId());
@@ -77,34 +66,6 @@ public class RankItem implements Comparable<RankItem> {
         this.score = 0.0;
     }
 
-    public RankItem(RankItem other) {
-        this.id = other.id;
-        this.videoId = other.videoId;
-        this.score = other.score;
-        this.scoreRos = other.scoreRos;
-        this.scoreStr = other.scoreStr;
-        this.video = other.video;
-
-        // merge queue
-        this.queue = other.queue;
-        if (other.getMergeQueuePath() != null)
-            this.mergeQueuePath.addAll(other.getMergeQueuePath());
-        if (other.getMergeDecisionLabels() != null)
-            this.mergeDecisionLabels.addAll(other.getMergeDecisionLabels());
-
-
-        // candidateinfo
-        this.candidateInfo = other.getCandidateInfo() != null ? other.getCandidateInfo().deepcopy() : null;
-        if (other.getCandidateInfoList() != null) {
-            for (CandidateInfo tmpCandidateInfo : other.getCandidateInfoList()) {
-                this.candidateInfoList.add(tmpCandidateInfo.deepcopy());
-            }
-        }
-
-
-    }
-
-
     @Override
     public int compareTo(RankItem o) {
         if (o == null) {
@@ -119,36 +80,4 @@ public class RankItem implements Comparable<RankItem> {
         }
     }
 
-    public RankItem deepcopy() {
-        return new RankItem(this);
-    }
-
-
-    public void addToCandidateInfoList(CandidateInfo candidateInfo) {
-        this.candidateInfoList.add(candidateInfo);
-    }
-
-
-    public void addMergeQueuePath(String mergeQueueName) {
-        this.mergeQueuePath.add(mergeQueueName);
-    }
-
-    public void addMergeDecisionLabel(String label) {
-        this.mergeDecisionLabels.add(label);
-    }
-
-    public void putRankerScore(String rankerName, Double rankerScore) {
-        this.rankerScore.put(rankerName, rankerScore);
-    }
-
-    public String getLastMergeQueueName() {
-        int pathSize = this.mergeQueuePath.size();
-        if (pathSize > 0) {
-            return this.mergeQueuePath.get(pathSize - 1);
-        } else {
-            return getCandidateInfo().getCandidateQueueName();
-        }
-    }
-
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/VideoActionFeature.java

@@ -1,17 +0,0 @@
-package com.tzld.piaoquan.recommend.server.common.base;
-
-import lombok.Getter;
-import lombok.Data;
-import lombok.Getter;
-
-@Data
-public class VideoActionFeature {
-
-
-    private double view = 0d;
-    private double play = 0d;
-    private double realPlay = 0d;
-    private double share = 0d;
-    private double returns = 0d;
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/CustomMongoProperties.java

@@ -15,6 +15,10 @@ public class CustomMongoProperties {
     private Integer clusterPort;
     private String secondaryHost;
     private Integer secondaryPort;
+    private String thirdHost;
+    private Integer thirdPort;
+    private String fourthHost;
+    private Integer fourthPort;
 
     private String uri;
     private String database;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/MongoTemplateConfig.java

@@ -35,7 +35,9 @@ public class MongoTemplateConfig {
         builder.applyToClusterSettings(settings -> {
             settings.hosts(Lists.newArrayList(
                     new ServerAddress(properties.getClusterHost(), properties.getClusterPort()),
-                    new ServerAddress(properties.getSecondaryHost(), properties.getSecondaryPort())));
+                    new ServerAddress(properties.getSecondaryHost(), properties.getSecondaryPort()),
+                    new ServerAddress(properties.getThirdHost(), properties.getThirdPort()),
+                    new ServerAddress(properties.getFourthHost(), properties.getFourthPort())));
         });
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/dataloader/FeatureConstructor.java

@@ -1,213 +0,0 @@
-package com.tzld.piaoquan.recommend.server.dataloader;
-
-
-import com.aliyun.odps.Instance;
-import com.aliyun.odps.Odps;
-import com.aliyun.odps.OdpsException;
-import com.aliyun.odps.account.Account;
-import com.aliyun.odps.account.AliyunAccount;
-import com.aliyun.odps.data.Record;
-import com.aliyun.odps.task.SQLTask;
-import com.aliyun.odps.tunnel.InstanceTunnel;
-import com.aliyun.odps.tunnel.io.TunnelRecordReader;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserActionFeature;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class FeatureConstructor {
-
-    private static final String BUCKET_NAME = "ali-recommend";
-    private static final Map<String, String> ODPS_CONFIG = new HashMap<String, String>();
-
-    static {
-        ODPS_CONFIG.put("ENDPOINT", "http://service.cn.maxcompute.aliyun.com/api");
-        ODPS_CONFIG.put("ACCESSID", "LTAIWYUujJAm7CbH");
-        ODPS_CONFIG.put("ACCESSKEY", "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P");
-    }
-
-    ;
-
-    private static final Account account = new AliyunAccount(ODPS_CONFIG.get("ACCESSID"), ODPS_CONFIG.get("ACCESSKEY"));
-
-
-    public static List<Record> loadStreamDataFromOSS(String table, String dt) {
-        Odps odps = new Odps(account);
-        odps.setEndpoint(ODPS_CONFIG.get("ENDPOINT"));
-        odps.setDefaultProject("loghubods");
-        String sql = String.format("select * from %s where dt ='%s';", table, dt);
-        Instance instance;
-        List<Record> records = new ArrayList<Record>();
-        try {
-            instance = SQLTask.run(odps, sql);
-            instance.waitForSuccess();
-            records = SQLTask.getResult(instance);
-        } catch (OdpsException e) {
-            e.printStackTrace();
-        }
-        return records;
-    }
-
-
-    public static TunnelRecordReader loadDataFromOSSSession(String sql) {
-        Odps odps = new Odps(account);
-        odps.setEndpoint(ODPS_CONFIG.get("ENDPOINT"));
-        odps.setDefaultProject("loghubods");
-        TunnelRecordReader reader = null;
-        try {
-            Instance instance = SQLTask.run(odps, sql);
-            instance.waitForSuccess();
-            InstanceTunnel tunnel = new InstanceTunnel(odps);
-            InstanceTunnel.DownloadSession session = tunnel.createDownloadSession(odps.getDefaultProject(), instance.getId());
-            long count = session.getRecordCount();
-            reader = session.openRecordReader(0, count);
-        } catch (OdpsException e) {
-            e.printStackTrace();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-        return reader;
-    }
-
-
-    public static RequestContext constructRequestContext(Record record) {
-        RequestContext requestContext = new RequestContext();
-        requestContext.setApptype(record.getString("apptype"));
-        requestContext.setMachineinfo_brand(record.getString("machineinfo_brand"));
-        requestContext.setMachineinfo_model(record.getString("machineinfo_model"));
-        requestContext.setMachineinfo_platform(record.getString("machineinfo_platform"));
-        requestContext.setMachineinfo_sdkversion(record.getString("machineinfo_sdkversion"));
-        requestContext.setMachineinfo_system(record.getString("machineinfo_system"));
-        requestContext.setMachineinfo_wechatversion(record.getString("machineinfo_wechatversion"));
-        requestContext.setDay(record.getString("ctx_day"));
-        requestContext.setWeek(record.getString("ctx_week"));
-        requestContext.setHour(record.getString("ctx_hour"));
-        requestContext.setRegion(record.getString("ctx_region"));
-        requestContext.setCity(record.getString("ctx_city"));
-        return requestContext;
-    }
-
-
-    public static UserFeature constructUserFeature(Record record) {
-        UserFeature userFeature = new UserFeature();
-        userFeature.setUid(record.get("uid").toString());
-        userFeature.setUser_cycle_bucket_7days(record.getString("u_cycle_bucket_7days"));
-        userFeature.setUser_cycle_bucket_30days(record.getString("u_cycle_bucket_30days"));
-        userFeature.setUser_share_bucket_30days(record.getString("u_share_bucket_30days"));
-
-
-        // 1day features
-        UserActionFeature user1dayActionFeature = new UserActionFeature();
-        user1dayActionFeature.setExp_cnt(record.getString("u_1day_exp_cnt"));
-        user1dayActionFeature.setClick_cnt(record.getString("u_1day_click_cnt"));
-        user1dayActionFeature.setShare_cnt(record.getString("u_1day_share_cnt"));
-        user1dayActionFeature.setReturn_cnt(record.getString("u_1day_return_cnt"));
-        user1dayActionFeature.setCtr(record.getString("u_ctr_1day"));
-        user1dayActionFeature.setStr(record.getString("u_str_1day"));
-        user1dayActionFeature.setRov(record.getString("u_rov_1day"));
-        user1dayActionFeature.setRos(record.getString("u_ros_1day"));
-        userFeature.setDay1_cnt_features(user1dayActionFeature);
-
-        // 3day features
-        UserActionFeature user3dayActionFeature = new UserActionFeature();
-        user3dayActionFeature.setExp_cnt(record.getString("u_3day_exp_cnt"));
-        user3dayActionFeature.setClick_cnt(record.getString("u_3day_click_cnt"));
-        user3dayActionFeature.setShare_cnt(record.getString("u_3day_share_cnt"));
-        user3dayActionFeature.setReturn_cnt(record.getString("u_3day_return_cnt"));
-        user3dayActionFeature.setCtr(record.getString("u_ctr_3day"));
-        user3dayActionFeature.setStr(record.getString("u_str_3day"));
-        user3dayActionFeature.setRov(record.getString("u_rov_3day"));
-        user3dayActionFeature.setRos(record.getString("u_ros_3day"));
-        userFeature.setDay3_cnt_features(user3dayActionFeature);
-
-        // 7day features
-        UserActionFeature user7dayActionFeature = new UserActionFeature();
-        user7dayActionFeature.setExp_cnt(record.getString("u_7day_exp_cnt"));
-        user7dayActionFeature.setClick_cnt(record.getString("u_7day_click_cnt"));
-        user7dayActionFeature.setShare_cnt(record.getString("u_7day_share_cnt"));
-        user7dayActionFeature.setReturn_cnt(record.getString("u_7day_return_cnt"));
-        user7dayActionFeature.setCtr(record.getString("u_ctr_7day"));
-        user7dayActionFeature.setStr(record.getString("u_str_7day"));
-        user7dayActionFeature.setRov(record.getString("u_rov_7day"));
-        user7dayActionFeature.setRos(record.getString("u_ros_7day"));
-        userFeature.setDay7_cnt_features(user7dayActionFeature);
-
-        // 3month features
-        UserActionFeature user3monthActionFeature = new UserActionFeature();
-        user3monthActionFeature.setExp_cnt(record.getString("u_3month_exp_cnt"));
-        user3monthActionFeature.setClick_cnt(record.getString("u_3month_click_cnt"));
-        user3monthActionFeature.setShare_cnt(record.getString("u_3month_share_cnt"));
-        user3monthActionFeature.setReturn_cnt(record.getString("u_3month_return_cnt"));
-        user3monthActionFeature.setCtr(record.getString("u_ctr_3month"));
-        user3monthActionFeature.setStr(record.getString("u_str_3month"));
-        user3monthActionFeature.setRov(record.getString("u_rov_3month"));
-        user3monthActionFeature.setRos(record.getString("u_ros_3month"));
-        userFeature.setMonth3_cnt_features(user3monthActionFeature);
-
-        return userFeature;
-    }
-
-
-    public static ItemFeature constructItemFeature(Record record) {
-        ItemFeature itemFeature = new ItemFeature();
-        itemFeature.setVideoId(record.get("videoid").toString());
-        itemFeature.setUpId(record.get("uid").toString());
-        //itemFeature.setTitleLength(record.get("play_count").toString());
-        itemFeature.setPlayLength(record.get("total_time").toString());
-        itemFeature.setTotalTime(record.get("total_time").toString());
-        itemFeature.setDaysSinceUpload(record.get("existence_days").toString());
-
-        UserActionFeature user1dayActionFeature = new UserActionFeature();
-        user1dayActionFeature.setExp_cnt(record.getString("i_1day_exp_cnt"));
-        user1dayActionFeature.setClick_cnt(record.getString("i_1day_click_cnt"));
-        user1dayActionFeature.setShare_cnt(record.getString("i_1day_share_cnt"));
-        user1dayActionFeature.setReturn_cnt(record.getString("i_1day_return_cnt"));
-        user1dayActionFeature.setCtr(record.getString("i_ctr_1day"));
-        user1dayActionFeature.setStr(record.getString("i_str_1day"));
-        user1dayActionFeature.setRov(record.getString("i_rov_1day"));
-        user1dayActionFeature.setRos(record.getString("i_ros_1day"));
-        itemFeature.setDay1_cnt_features(user1dayActionFeature);
-
-        UserActionFeature user3dayActionFeature = new UserActionFeature();
-        user3dayActionFeature.setExp_cnt(record.getString("i_3day_exp_cnt"));
-        user3dayActionFeature.setClick_cnt(record.getString("i_3day_click_cnt"));
-        user3dayActionFeature.setShare_cnt(record.getString("i_3day_share_cnt"));
-        user3dayActionFeature.setReturn_cnt(record.getString("i_3day_return_cnt"));
-        user3dayActionFeature.setCtr(record.getString("i_ctr_3day"));
-        user3dayActionFeature.setStr(record.getString("i_str_3day"));
-        user3dayActionFeature.setRov(record.getString("i_rov_3day"));
-        user3dayActionFeature.setRos(record.getString("i_ros_3day"));
-        itemFeature.setDay3_cnt_features(user1dayActionFeature);
-
-        UserActionFeature user7dayActionFeature = new UserActionFeature();
-        user7dayActionFeature.setExp_cnt(record.getString("i_7day_exp_cnt"));
-        user7dayActionFeature.setClick_cnt(record.getString("i_7day_click_cnt"));
-        user7dayActionFeature.setShare_cnt(record.getString("i_7day_share_cnt"));
-        user7dayActionFeature.setReturn_cnt(record.getString("i_7day_return_cnt"));
-        user7dayActionFeature.setCtr(record.getString("i_ctr_7day"));
-        user7dayActionFeature.setStr(record.getString("i_str_7day"));
-        user7dayActionFeature.setRov(record.getString("i_rov_7day"));
-        user7dayActionFeature.setRos(record.getString("i_ros_7day"));
-        itemFeature.setDay7_cnt_features(user1dayActionFeature);
-
-        UserActionFeature user3monthActionFeature = new UserActionFeature();
-        user3monthActionFeature.setExp_cnt(record.getString("i_3month_exp_cnt"));
-        user3monthActionFeature.setClick_cnt(record.getString("i_3month_click_cnt"));
-        user3monthActionFeature.setShare_cnt(record.getString("i_3month_share_cnt"));
-        user3monthActionFeature.setReturn_cnt(record.getString("i_3month_return_cnt"));
-        user3monthActionFeature.setCtr(record.getString("i_ctr_3month"));
-        user3monthActionFeature.setStr(record.getString("i_str_3month"));
-        user3monthActionFeature.setRov(record.getString("i_rov_3month"));
-        user3monthActionFeature.setRos(record.getString("i_ros_3month"));
-        itemFeature.setMonth3_cnt_features(user3monthActionFeature);
-        return itemFeature;
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/dataloader/ItemFeatureToRedisLoader.java

@@ -1,76 +0,0 @@
-package com.tzld.piaoquan.recommend.server.dataloader;
-
-import com.aliyun.odps.data.Record;
-import com.aliyun.odps.tunnel.io.TunnelRecordReader;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-
-
-public class ItemFeatureToRedisLoader {
-
-    private final String videoKeyFormat = "video:%s";
-    private ExecutorService pool = ThreadPoolFactory.defaultPool();
-
-    public static RedisTemplate<String, String> buildRedisTemplate() {
-        RedisStandaloneConfiguration rsc = new RedisStandaloneConfiguration();
-        rsc.setPort(6379);
-        rsc.setPassword("Wqsd@2019");
-        rsc.setHostName("r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com");
-        RedisTemplate<String, String> template = new RedisTemplate<>();
-        JedisConnectionFactory fac = new JedisConnectionFactory(rsc);
-        fac.afterPropertiesSet();
-        template.setDefaultSerializer(new StringRedisSerializer());
-        template.setConnectionFactory(fac);
-        template.afterPropertiesSet();
-        return template;
-    }
-
-
-    public void loadFeatureToRedis(String table, String dt) {
-        RedisTemplate<String, String> redisTemplate = buildRedisTemplate();
-        String sql = String.format("select * from %s where dt ='%s';", table, dt);
-        TunnelRecordReader reader = FeatureConstructor.loadDataFromOSSSession(sql);
-        Record record;
-        Map<String, String> itemFeaRedisFormat = new HashMap<String, String>();
-        int count = 0;
-        try {
-            while ((record = reader.read()) != null) {
-                ItemFeature itemFeature = FeatureConstructor.constructItemFeature(record);
-                String key = String.format(videoKeyFormat, itemFeature.getKey());
-                String value = itemFeature.getValue();
-                itemFeaRedisFormat.put(key, value);
-                if (count < 200) {
-                    count++;
-                } else if (count == 200) {
-                    redisTemplate.opsForValue().multiSet(itemFeaRedisFormat);
-                    System.out.println("------succes add 200-----");
-                    itemFeaRedisFormat = new HashMap<String, String>();
-                    count = 0;
-                    break;
-                }
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-
-    public static void main(String[] args) {
-        if (args.length < 2) {
-            System.out.println("--------args 缺失---------");
-            return;
-        }
-        ItemFeatureToRedisLoader itemFeatureToRedisLoader = new ItemFeatureToRedisLoader();
-        itemFeatureToRedisLoader.loadFeatureToRedis(args[0], args[1]);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/dataloader/OfflineAdOutSamplesLoader.java

@@ -1,93 +0,0 @@
-package com.tzld.piaoquan.recommend.server.dataloader;
-
-import com.aliyun.odps.tunnel.io.TunnelRecordReader;
-import com.google.common.collect.ListMultimap;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
-import com.tzld.piaoquan.recommend.feature.domain.video.feature.VlogShareLRFeatureExtractor;
-import com.aliyun.odps.data.Record;
-import com.tzld.piaoquan.recommend.feature.model.sample.BaseFeature;
-import com.tzld.piaoquan.recommend.feature.model.sample.FeatureGroup;
-
-import java.io.IOException;
-import java.util.*;
-
-
-public class OfflineAdOutSamplesLoader {
-
-    private static final String BUCKET_NAME = "ali-recommend";
-    private static  final Map<String, String> ODPS_CONFIG =  new HashMap<String, String>();
-    static {
-        ODPS_CONFIG.put("ENDPOINT", "http://service.cn.maxcompute.aliyun.com/api");
-        ODPS_CONFIG.put("ACCESSID", "LTAIWYUujJAm7CbH");
-        ODPS_CONFIG.put("ACCESSKEY", "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P");
-    };
-
-
-    // 单条日志处理逻辑
-    public static String singleParse(Record record) {
-        // 数据解析
-        String label = record.getString("ui_is_out");
-        if(label == null){
-            label = "0";
-        }
-
-        // 从sql的 record中 初始化对象内容
-        RequestContext requestContext = FeatureConstructor.constructRequestContext(record);
-        UserFeature userFeature = FeatureConstructor.constructUserFeature(record);
-        ItemFeature itemFeature =  FeatureConstructor.constructItemFeature(record);
-
-        // 转化成bytes
-        RequestContextBytesFeature requestContextBytesFeature = new RequestContextBytesFeature(requestContext);
-        UserBytesFeature userBytesFeature = new UserBytesFeature(userFeature);
-        VideoBytesFeature videoBytesFeature = new VideoBytesFeature(itemFeature);
-
-        // 特征抽取
-        VlogShareLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new VlogShareLRFeatureExtractor();
-
-        bytesFeatureExtractor.getUserFeatures(userBytesFeature);
-        bytesFeatureExtractor.getItemFeature(videoBytesFeature);
-        bytesFeatureExtractor.getContextFeatures(requestContextBytesFeature);
-
-        ListMultimap<FeatureGroup, BaseFeature> featureMap = bytesFeatureExtractor.getFeatures();
-        return parseSamplesToString(label, featureMap);
-    }
-
-    // 构建样本的字符串
-    public static String parseSamplesToString(String label, ListMultimap<FeatureGroup, BaseFeature> featureMap) {
-        ArrayList<String> featureList = new ArrayList<String>();
-        for (Map.Entry<FeatureGroup, BaseFeature> entry : featureMap.entries()) {
-            FeatureGroup groupedFeature = entry.getKey();
-            BaseFeature baseFeature = entry.getValue();
-            Long featureIdentifier = baseFeature.getIdentifier();
-            featureList.add(String.valueOf(featureIdentifier) + ":1");
-        }
-        return label + "\t" + String.join("\t", featureList);
-    }
-
-
-    //  主处理逻辑
-    public static void mutiplyParser(String table, String startDay, String endDay) {
-        String sql = String.format("select * from %s where ad_ornot = '0' and apptype != '13' and dt >='%s' and dt <='%s';", table, startDay, endDay);
-        TunnelRecordReader reader = FeatureConstructor.loadDataFromOSSSession(sql);
-        Record record;
-        try {
-            while ((record = reader.read()) != null) {
-                String samples = singleParse(record);
-                System.out.println(samples);
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-
-    public static void main(String[] args) {
-        if(args.length < 2){
-            System.out.println("--------args 缺失---------");
-            return;
-        }
-        OfflineAdOutSamplesLoader.mutiplyParser(args[0], args[1], args[2]);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/dataloader/UserFeatureToRedisLoader.java

@@ -1,79 +0,0 @@
-package com.tzld.piaoquan.recommend.server.dataloader;
-
-import com.aliyun.odps.data.Record;
-import com.aliyun.odps.tunnel.io.TunnelRecordReader;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import java.util.concurrent.ExecutorService;
-
-
-public class UserFeatureToRedisLoader {
-
-    private final String userKeyFormat = "user:%s";
-    private ExecutorService pool = ThreadPoolFactory.defaultPool();
-
-    public static RedisTemplate<String, String> buildRedisTemplate() {
-        RedisStandaloneConfiguration rsc = new RedisStandaloneConfiguration();
-        rsc.setPort(6379);
-        rsc.setPassword("Wqsd@2019");
-        rsc.setHostName("r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com");
-        RedisTemplate<String, String> template = new RedisTemplate<>();
-        JedisConnectionFactory fac = new JedisConnectionFactory(rsc);
-        fac.afterPropertiesSet();
-        template.setDefaultSerializer(new StringRedisSerializer());
-        template.setConnectionFactory(fac);
-        template.afterPropertiesSet();
-        return template;
-    }
-
-
-
-    public void loadFeatureToRedis(String userTable, String dt) {
-        RedisTemplate<String, String> redisTemplate = buildRedisTemplate();
-        String sql = String.format("select * from %s where dt ='%s';", userTable, dt);
-        TunnelRecordReader reader = FeatureConstructor.loadDataFromOSSSession(sql);
-        Record record;
-        Map<String, String> userFeaRedisFormat = new HashMap<String, String>();
-        int count = 0;
-        try {
-            while ((record = reader.read()) != null) {
-                UserFeature userFeature = FeatureConstructor.constructUserFeature(record);
-                String key = String.format(userKeyFormat, userFeature.getKey());
-                String value = userFeature.getValue();
-                userFeaRedisFormat.put(key, value);
-                if(count < 200) {
-                    count++;
-                } else if (count == 200) {
-                    redisTemplate.opsForValue().multiSet(userFeaRedisFormat);
-                    System.out.println("------succes add 2000-----");
-                    userFeaRedisFormat = new HashMap<String, String>();
-                    count = 0;
-                    break;
-                }
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-
-
-    public static void main(String[] args) {
-        if(args.length < 2){
-            System.out.println("--------args 缺失---------");
-            return;
-        }
-        UserFeatureToRedisLoader userFeatureToRedisLoader = new UserFeatureToRedisLoader();
-        userFeatureToRedisLoader.loadFeatureToRedis(args[0], args[1]);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Candidate.java

@@ -1,40 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.candidiate;
-
-
-import com.google.gson.Gson;
-import lombok.Data;
-
-
-/**
- * 记录召回下一路队列索引信息
- */
-@Data
-public class Candidate {
-    private String candidateKey;
-    private int candidateNum;
-    private QueueName candidateQueueName;
-    // 上层待合并的queue name
-    private String mergeQueueName;
-    private int mergeQueueNum;
-
-    public Candidate() {
-    }
-
-    public Candidate(Candidate other) {
-        this.candidateKey = other.getCandidateKey();
-        this.candidateNum = other.getCandidateNum();
-        this.candidateQueueName = other.getCandidateQueueName();
-        this.mergeQueueName = other.getMergeQueueName();
-        this.mergeQueueNum = other.getMergeQueueNum();
-    }
-
-    public Candidate deepcopy() {
-        return new Candidate(this);
-    }
-
-
-    @Override
-    public String toString() {
-        return new Gson().toJson(this);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/CandidateInfo.java

@@ -1,31 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.candidiate;
-
-
-import com.google.gson.Gson;
-import lombok.Data;
-
-@Data
-public class CandidateInfo {
-    private String candidateQueueName;     // 队列名称
-    private int position;               // 在召回中的位置
-    private Candidate candidate;        // 队列配置的信息
-
-    public CandidateInfo() {
-    }
-
-    public CandidateInfo(CandidateInfo other) {
-        this.candidateQueueName = other.getCandidateQueueName();
-        this.position = other.getPosition();
-        this.candidate = other.candidate != null ? new Candidate(other.candidate) : null;
-    }
-
-    public CandidateInfo deepcopy() {
-        return new CandidateInfo(this);
-    }
-
-
-    @Override
-    public String toString() {
-        return new Gson().toJson(this);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Entry.java

@@ -1,39 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.candidiate;
-
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * 一个队列召回的内容实体
- * @param <T>
- */
-public class Entry<T> {
-    public final String id;
-    public final T item;
-    public final Map<String, Double> scores;
-    public final Map<String, String> explanations;
-
-
-    public Entry(T item, String id) {
-        this.item = item;
-        this.id = id;
-        this.scores = new HashMap<String, Double>();
-        this.explanations = new HashMap<String, String>();
-    }
-
-    public Entry(Entry<T> other) {
-        this.item = other.item;
-        this.id = other.id;
-        this.scores = other.scores;
-        this.explanations = other.explanations;
-    }
-
-    public void addScore(String name, double score) {
-        scores.put(name, score);
-    }
-
-    public void addExplanation(String name, String explanation) {
-        explanations.put(name, explanation);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/Queue.java

@@ -1,56 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.candidiate;
-
-
-import java.util.*;
-
-/**
- * An abstract data type represents a queue of certain item.
- *   一个队列内召回Items完整list
- */
-public class Queue<T> implements Iterable<Entry<T>> {
-    private final List<Entry<T>> entries;
-    private final String name;
-    private final String explain;
-
-    public Queue(String name) {
-        entries = new ArrayList<Entry<T>>();
-        this.name = name;
-        this.explain = null;
-    }
-
-    public Queue(String name, String explain) {
-        this.name = name;
-        this.explain = explain;
-        this.entries = new ArrayList<Entry<T>>();
-    }
-
-    public void reverse() {
-        Collections.reverse(this.entries);
-    }
-
-    public void add(Entry<T> entry) {
-        entries.add(entry);
-    }
-
-    public void addAll(Collection<Entry<T>> paramEntries) {
-        entries.addAll(paramEntries);
-    }
-
-    public List<Entry<T>> get() {
-        return entries;
-    }
-
-    public Iterator<Entry<T>> iterator() {
-        return entries.iterator();
-    }
-
-    public int size() {
-        return entries.size();
-    }
-
-    public void limit(int num) {
-        if (!entries.isEmpty() && num >= 0 && num < entries.size()) {
-            entries.subList(num, entries.size()).clear();
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/candidiate/QueueName.java

@@ -1,114 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.candidiate;
-
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.FluentIterable;
-import lombok.Data;
-import org.apache.commons.lang3.tuple.Pair;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Queue names.召回队列命名处理
- */
-@Data
-public class QueueName implements Serializable, Comparable<QueueName> {
-    private static final Function<Pair<String, String>, String> MATCH_PATTERN =
-            new Function<Pair<String, String>, String>() {
-                @Override
-                public String apply(Pair<String, String> input) {
-                    return input.getLeft() + "=" + input.getRight();
-                }
-            };
-    public static final long DEFAULT_LOCAL_CACHE_TTL = 15 * 60 * 1000;
-    private final List<Pair<String, String>> matches = new ArrayList<Pair<String, String>>();
-    private final String itemType;
-    private final String ordering;
-    private final long ttl;
-
-    private String metaChannel;  // meta 渠道
-
-    public long getTTL(){
-        return this.ttl;
-    }
-
-    public QueueName(String itemType, String ordering) {
-        this(itemType, ordering, DEFAULT_LOCAL_CACHE_TTL);
-    }
-
-    public QueueName(String itemType, String ordering, long ttl) {
-        this.itemType = itemType;
-        this.ordering = ordering;
-        this.ttl = ttl;
-    }
-
-    public static QueueName fromString(String string, long ttl) {
-        String[] parts = string.split(":");
-        String itemType = parts[0];
-        String ordering = parts[parts.length - 1].split("=")[1];
-
-        QueueName result = new QueueName(itemType, ordering, ttl);
-        for (int i = 2; i < parts.length - 1; ++i) {
-            String[] sides = parts[i].split("=");
-            result.addMatch(sides[0], sides[1]);
-        }
-        return result;
-    }
-
-    public static QueueName fromString(String string) {
-        return QueueName.fromString(string, DEFAULT_LOCAL_CACHE_TTL);
-    }
-
-    public static Pair<String, String> parseMatchPair(String match) {
-        String[] arr = match.split("=");
-        if (arr.length == 2) {
-            return Pair.of(arr[0], arr[1]);
-        }
-        return Pair.of("", "");
-    }
-
-    public QueueName addMatch(String key, String value) {
-        if (value == null || value.equals("")) {
-            value = "_";
-        } else {
-            value = value.replace("=", "_");
-            value = value.replace(":", "_");
-        }
-        if (key.equals("channel")) {
-            this.setMetaChannel(value);
-        }
-        matches.add(Pair.of(key, value));
-        return this;
-    }
-
-    public Iterable<String> getMatches() {
-        return FluentIterable.from(matches).transform(MATCH_PATTERN);
-    }
-
-
-    @Override
-    public String toString() {
-        Iterable<String> matchesString = FluentIterable.from(matches)
-                .transform(MATCH_PATTERN);
-
-        return itemType + ":queue:" + Joiner.on(":").join(matchesString) + ":ordering=" + ordering;
-    }
-
-    @Override
-    public int hashCode() {
-        return toString().hashCode();
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        return other instanceof QueueName && ((QueueName) other).toString().equals(toString());
-    }
-
-    @Override
-    public int compareTo(QueueName other) {
-        return other.toString().compareTo(ordering.toString());
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/User.java

@@ -1,52 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.common;
-
-import lombok.Data;
-
-@Data
-public class User {
-
-    private String id;
-    private String uid;
-    private String mid;
-    private UserAttention userAttention;
-
-    // user profile 基础画像信息
-    private String machine_model;
-    private String machine_brand;
-    private String sdk;
-    private String gender;
-    private String ifCreater;
-    private String userType;
-    private String region;
-    private String city;
-
-    // user-content info 用户内容tag统计数据
-    private UserAction last1monthVideoTags;
-    private UserAction last7dayVideoTags;
-    private UserAction last1dayVideoTags;
-    private UserAction lastSessionVideoTags;
-
-    private UserAction last1monthTitleTags;
-    private UserAction last7dayTitleTags;
-    private UserAction last1dayTitleTags;
-    private UserAction lastSessionTitleTags;
-
-    // 用户关注up主信息
-    private UserAction last1monthPublishers;
-
-    // user-action info 用户
-    private UserAction last1monthUserAction;
-    private UserAction last7dayUserAction;
-    private UserAction last1dayUserAction;
-    private UserAction last1hourUserAction;
-    private UserAction lastSessionUserAction;
-
-    // user-group info
-    private String userGroup;
-
-    // user-vector
-    private String userVector;
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/UserAction.java

@@ -1,21 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.common;
-
-import lombok.Data;
-
-import java.util.List;
-
-@Data
-public class UserAction {
-    //记录用户行为中 itemids，tags等信息
-
-    private List<String> followed;
-    private List<String> follow_and_realplay;
-    private List<String> played;
-    private List<String> shared;
-
-    private List<String> feedsPlayed;
-    private List<String> realPlayed;
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/common/UserAttention.java

@@ -1,26 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.common;
-
-public class UserAttention {
-    /**
-     分维度的用户特征数据, 包含：
-     1、分不同时间维度的ItemList
-     30天、7天、1天、1H、实时，可以提供给I2I类召回策略用
-
-     2、分不同时间维度的categoryList
-     30天、7天、1天、1H、实时，可以提供给CB类召回策略用
-
-     3、用户group信息
-     天级别更新数据，可以提供给usergroup -> Item召回策略用。
-
-     4、分时间维度的发布者List
-     30天、7天、1天、1H、实时，可以提供 发布者 -> Item类召回用
-
-     */
-
-
-
-
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/IndexCandidateQueue.java

@@ -1,88 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.apache.commons.lang3.tuple.Pair;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-
-/**
- * 定义召回下子队列的Queue 融合信息
- */
-public abstract class IndexCandidateQueue extends StrategyQueue {
-
-    protected Map<String, Candidate> myCandidates = new HashMap<String, Candidate>();
-
-    public IndexCandidateQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    public abstract int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId);
-
-
-    @Override
-    public int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        // TODO: sublist by every queue
-        return MergeUtils.simpleMergeWithProtection(items, rankerItemsListMap, recNum, expId);
-    }
-
-    @Override
-    public int candidate(final Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
-        myCandidates.clear();
-        int n = addCandidateKey(candidateMap, recallNum, user, requestData, requestIndex, expId);
-        candidateMap.putAll(myCandidates);
-
-        return n;
-    }
-
-    protected int addCandidateKey(Map<String, Candidate> candidates, String key, int num, String queue) {
-        if (candidates.containsKey(key)) {
-            return 0;
-        } else {
-            Candidate candidate = new Candidate();
-            candidate.setCandidateKey(key);
-            candidate.setCandidateNum(num);
-            candidate.setMergeQueueName(queue);
-            candidates.put(key, candidate);
-            return num;
-        }
-    }
-
-    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue) {
-        return this.addCandidateKey(candidates, queueName, num, queue, true, true, true);
-    }
-
-    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue, boolean needScore, boolean needPornRank) {
-        return this.addCandidateKey(candidates, queueName, num, queue, needScore, needPornRank, true);
-    }
-
-    protected int addCandidateKey(Map<String, Candidate> candidates, QueueName queueName, int num, String queue, boolean needScore, boolean needPornRank, boolean needExposeHistoryFilter) {
-        if (candidates.containsKey(queueName.toString())) {
-            return 0;
-        } else {
-            Candidate candidate = new Candidate();
-            candidate.setCandidateKey(queueName.toString());
-            candidate.setCandidateNum(num);
-            candidate.setMergeQueueName(queue);
-            candidates.put(queueName.toString(), candidate);
-            return num;
-        }
-    }
-
-    @Override
-    public String toString() {
-        return "IndexCandidateQueue{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/MergeRule.java

@@ -1,45 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import lombok.Data;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-
-/**
- * Merge 规则配置
- */
-@Data
-public class MergeRule {
-    public String queueName;
-
-    public Set<Integer> enableExpIdSet = null;
-    public Set<Integer> disableExpIdSet = null;
-
-    public double recallPercentage = 0.1; // 召回百分比, 默认占 10%
-    public int minMergeNum = 0; // 合并保护规则: 最小合并条数
-    public int maxMergeNum = 100; // 合并保护规则: 最大合并条数
-
-    private Map<String, String> properties;
-
-    public MergeRule() {
-        this.properties = new HashMap<String, String>();
-    }
-
-    public boolean isDisabled(final int expId) {
-        if (enableExpIdSet != null && !enableExpIdSet.contains(expId)) {
-            return true;
-        }
-        if (disableExpIdSet != null && disableExpIdSet.contains(expId)) {
-            return true;
-        }
-        return false;
-    }
-
-    public void putProperty(String key, String value) {
-        this.properties.put(key, value);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/MergeUtils.java

@@ -1,313 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.google.common.base.Function;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.CandidateInfo;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.apache.commons.lang3.tuple.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.*;
-
-
-public class MergeUtils {
-    public static final Logger LOGGER = LoggerFactory.getLogger(MergeUtils.class);
-
-    // mergeConfFile_topQueueName 对应的 StrategyQueue 缓存
-    private static final Map<String, Config> queueConfigMap = new HashMap<>();
-
-    public static StrategyQueue createTopQueue(String mergeConfFile, String topQueueName) {
-        String key = mergeConfFile + "_" + topQueueName;
-        Config mergeQueueConf;
-        if (queueConfigMap.containsKey(key)) {
-            mergeQueueConf = queueConfigMap.get(key);
-        } else {
-            mergeQueueConf = ConfigFactory.parseResources(mergeConfFile);
-            queueConfigMap.put(key, mergeQueueConf);
-        }
-        StrategyQueueConfig strategyQueueConfig = new StrategyQueueConfig();
-        if (strategyQueueConfig.load(mergeQueueConf)) {
-            LOGGER.debug("Merger config init succ");
-        } else {
-            LOGGER.error("Merge config init failed: init MergeBiz using queue config {}", mergeConfFile);
-        }
-
-        StrategyQueue topQueue = null;
-        try {
-            topQueue = MergeUtils.constructStrategyQueue(topQueueName, strategyQueueConfig);
-        } catch (InstantiationException e) {
-            LOGGER.error("construct top StrategyQueue: [{}]", e);
-        } catch (IllegalAccessException e) {
-            LOGGER.error("construct top StrategyQueue: [{}]", e);
-        } catch (ClassNotFoundException e) {
-            LOGGER.error("construct top StrategyQueue: [{}]", e);
-        } catch (NoSuchMethodException e) {
-            LOGGER.error("construct top StrategyQueue: [{}]", e);
-        } catch (InvocationTargetException e) {
-            LOGGER.error("construct top StrategyQueue: [{}]", e);
-        }
-        return topQueue;
-    }
-
-    public static StrategyQueue constructStrategyQueue(String queueName, StrategyQueueConfig strategyQueueConfig) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
-        if (!strategyQueueConfig.getStrategyQueueInfoMap().containsKey(queueName)) {
-            return null;
-        }
-
-        StrategyQueueInfo strategyQueueInfo = strategyQueueConfig.getStrategyQueueInfoMap().get(queueName);
-        StrategyQueue strategyQueue = (StrategyQueue) Class.forName(strategyQueueInfo.getQueueClass())
-                .getConstructor(StrategyQueueInfo.class, strategyQueueConfig.getClass())
-                .newInstance(strategyQueueInfo, strategyQueueConfig);
-
-        return strategyQueue;
-    }
-
-
-    /**
-     * 分发items到策略树中
-     * 分发的过程中不要破坏相对顺序
-     *
-     * @param strategyQueue
-     * @param items
-     */
-    public static void distributeItemsToMultiQueues(StrategyQueue strategyQueue, List<RankItem> items) {
-        List<StrategyQueue> strategyQueueList = strategyQueue.getAllQueues();
-
-        Multimap<String, RankItem> mergeQueuesItems = ArrayListMultimap.create();
-        for (RankItem item : items) {
-            for (CandidateInfo candidateInfo : item.getCandidateInfoList()) {
-                String mergeQueue = candidateInfo.getCandidate().getMergeQueueName();
-                RankItem currentItem = item.deepcopy();
-
-                // set candidate info
-                currentItem.setQueue(mergeQueue);
-                currentItem.setCandidateInfo(candidateInfo);
-
-                mergeQueuesItems.put(mergeQueue, currentItem);
-            }
-        }
-        for (StrategyQueue queue : strategyQueueList) {
-            String mergeQueueName = queue.getStrategyQueueInfo().getQueueName();
-            if (mergeQueuesItems.containsKey(mergeQueueName)) {
-                List<RankItem> currMergeQueueItems = new ArrayList<RankItem>(mergeQueuesItems.get(mergeQueueName));
-                queue.setItems(currMergeQueueItems);
-            }
-        }
-    }
-
-
-
-
-    /**
-     * 基于 score 字段融合: score 较大的 Item 优先
-     *
-     * @param resultRankerItems
-     * @param rankerItemsListMap
-     * @param freeRecNum
-     * @param expId
-     */
-    public static void simpleMergeByScore(List<RankItem> resultRankerItems,
-                                          final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap,
-                                          int freeRecNum,
-                                          int expId) {
-        // 先定义按score排序优选的独立额
-        PriorityQueue<Pair<String, Integer>> mergePriorityQueue = new PriorityQueue<Pair<String, Integer>>(freeRecNum, Comparator.comparing(o -> rankerItemsListMap.get(o.getLeft()).getRight().get(o.getRight())));
-
-        // 大于最小mergenum 同时小于maxMergenum，在队列中add
-        for (Pair<MergeRule, List<RankItem>> entry : rankerItemsListMap.values()) {
-            if (entry.getLeft().isDisabled(expId) || entry.getRight() == null || entry.getRight().isEmpty()) {
-                continue;
-            }
-
-            MergeRule myRule = entry.getLeft();
-            if (myRule.minMergeNum < myRule.maxMergeNum && entry.getRight().size() > myRule.minMergeNum) {
-                Collections.sort(entry.getRight().subList(myRule.minMergeNum, entry.getRight().size()));
-                mergePriorityQueue.add(Pair.of(myRule.queueName, myRule.minMergeNum));
-            }
-        }
-        //
-        while (freeRecNum > 0 && !mergePriorityQueue.isEmpty()) {
-            Pair<String, Integer> item = mergePriorityQueue.poll();
-            String myName = item.getLeft();
-            int myIndex = item.getRight();
-
-            resultRankerItems.add(rankerItemsListMap.get(myName).getRight().get(myIndex));
-            resultRankerItems.get(resultRankerItems.size() - 1).addMergeDecisionLabel("score:" + myName);
-            freeRecNum -= 1;
-            myIndex += 1;
-
-            if (freeRecNum > 0 && rankerItemsListMap.get(myName).getRight().size() > myIndex && myIndex < rankerItemsListMap.get(myName).getLeft().maxMergeNum) {
-                mergePriorityQueue.add(Pair.of(myName, myIndex));
-            }
-        }
-    }
-
-    /**
-     * 队列融合:
-     * 1. 各队列基于排序不变;
-     * 2. 各队列保证最小合并数量 (minMergeNum) 条目;
-     * 3. 各队列限制最大合并数量 (maxMergeNum) 条目;
-     * 4. 从各队列取 Score 最高条目.
-     *
-     * @param rankerItemsListMap
-     * @param recNum
-     * @param expId
-     * @return
-     */
-    public static int simpleMergeWithProtection(List<RankItem> resultRankerItems,
-                                                final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap,
-                                                int recNum,
-                                                int expId) {
-        if (resultRankerItems == null) {
-            resultRankerItems = new LinkedList<RankItem>();
-        }
-        // 保证各队列最小合并条目后剩余可自由调配条目数量
-        int freeRecNum = recNum;
-        for (Pair<MergeRule, List<RankItem>> entry : rankerItemsListMap.values()) {
-
-            if (entry.getLeft().isDisabled(expId) || entry.getRight() == null || entry.getRight().isEmpty()) {
-                continue;
-            }
-
-            MergeRule myRule = entry.getLeft();
-            int mySize = entry.getRight().size();
-            int myMinMergeNum = Math.min(myRule.minMergeNum, mySize);
-            freeRecNum -= myMinMergeNum;
-
-            int prevSize = resultRankerItems.size();
-            // 先保证最小长度添加
-            resultRankerItems.addAll(entry.getRight().subList(0, myMinMergeNum));
-
-            int afterSize = resultRankerItems.size();
-            for (int ix = prevSize; ix < afterSize; ++ix) {
-                resultRankerItems.get(ix).addMergeDecisionLabel("min_protect:" + myRule.queueName);
-            }
-        }
-
-        if (freeRecNum > 0) {
-            //保证最小长度后，其他按score融合
-            simpleMergeByScore(resultRankerItems, rankerItemsListMap, freeRecNum, expId);
-        }
-
-        return resultRankerItems.size();
-    }
-
-    /**
-     * 按优先级队列融合:
-     * 1. 各队列基于 priority 依次融合: 各队列保证最小合并数量 (minMergeNum) 条目;
-     * 2. 若没有凑足 recNum, 各队列限制最大合并数量 (maxMergeNum) 条目条件下按 score 优先选择;
-     *
-     * @param rankerItemsListMap
-     * @param recNum
-     * @param user
-     * @param requestData
-     * @param requestIndex
-     * @param expId
-     * @return
-     */
-    public static int simpleMergeByPriority(List<RankItem> resultRankerItems, final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, int recNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        if (resultRankerItems == null) {
-            resultRankerItems = new LinkedList<RankItem>();
-        }
-
-        final List<Map.Entry<String, Pair<MergeRule, List<RankItem>>>> rankerItemsList = new LinkedList<Map.Entry<String, Pair<MergeRule, List<RankItem>>>>(rankerItemsListMap.entrySet());
-
-        // 保证各队列最小合并条目后剩余可自由调配条目数量
-        Collections.sort(rankerItemsList, new Comparator<Map.Entry<String, Pair<MergeRule, List<RankItem>>>>() {
-            private final int DEFAULT_PRIORITY = 0;
-
-            @Override
-            public int compare(Map.Entry<String, Pair<MergeRule, List<RankItem>>> o1, Map.Entry<String, Pair<MergeRule, List<RankItem>>> o2) {
-                int p1 = getPriority(o1.getValue().getLeft()), p2 = getPriority(o2.getValue().getLeft());
-                return p1 - p2;
-            }
-
-            private int getPriority(MergeRule rule) {
-                if (rule.getProperties().containsKey("priority")) {
-                    return Integer.valueOf(rule.getProperties().get("priority"));
-                }
-                return DEFAULT_PRIORITY;
-            }
-        });
-
-        int freeRecNum = recNum;
-        for (int i = 0; i < rankerItemsList.size(); ++i) {
-            if (rankerItemsList.get(i).getValue().getLeft().isDisabled(expId) || rankerItemsList.get(i).getValue().getRight() == null || rankerItemsList.get(i).getValue().getRight().isEmpty()) {
-                continue;
-            }
-
-            MergeRule myRule = rankerItemsList.get(i).getValue().getLeft();
-            int mySize = rankerItemsList.get(i).getValue().getRight().size();
-            int myMinMergeNum = Math.min(myRule.minMergeNum, mySize);
-            freeRecNum -= myMinMergeNum;
-
-            int prevSize = resultRankerItems.size();
-            resultRankerItems.addAll(rankerItemsList.get(i).getValue().getRight().subList(0, myMinMergeNum));
-            if (mySize > myMinMergeNum) {
-                Collections.sort(rankerItemsList.get(i).getValue().getRight().subList(myMinMergeNum, mySize));
-            }
-            int afterSize = resultRankerItems.size();
-            for (int ix = prevSize; ix < afterSize; ++ix) {
-                resultRankerItems.get(ix).addMergeDecisionLabel("min_protect:" + myRule.queueName);
-            }
-        }
-
-        if (freeRecNum > 0) {
-            simpleMergeByScore(resultRankerItems, rankerItemsListMap, freeRecNum, expId);
-        }
-
-        return resultRankerItems.size();
-    }
-
-    /**
-     * 基于 tag 和 category 重排序, 保证具有相同的分类或标签的 Item 在 windowSize 窗口内只有 similarLimit 条
-     *
-     * @param items
-     * @param expectedRecommendItemsCount
-     * @param windowSize
-     * @param similarLimit
-     */
-    public static void diversityRerank(List<RankItem> items, Function<Pair<RankItem, RankItem>, Boolean> similarFunc, int expectedRecommendItemsCount, final int windowSize, final int similarLimit) {
-        List<RankItem> rerankedItems = new LinkedList<RankItem>();
-        expectedRecommendItemsCount = Math.min(expectedRecommendItemsCount, items.size());
-        for (int ix = 0; ix < expectedRecommendItemsCount; ++ix) {
-            int windowStartPosition = Math.max(ix - windowSize + 1, 0);
-            int nextId = 0;
-            while (nextId < items.size()) {
-                int similarCount = 0;
-                for (int jx = windowStartPosition; jx < ix && similarCount < similarLimit; ++jx) {
-                    if (similarFunc.apply(Pair.of(items.get(nextId), rerankedItems.get(jx)))) {
-                        similarCount += 1;
-                    }
-                }
-                if (similarCount < similarLimit) {
-                    break;
-                }
-                nextId += 1;
-            }
-            if (nextId >= items.size()) {
-                nextId = 0;
-            }
-
-            RankItem nextItem = items.get(nextId);
-            nextItem.addMergeDecisionLabel("sim_rerank:" + windowStartPosition + "_" + windowSize);
-            items.remove(nextId);
-            rerankedItems.add(nextItem);
-        }
-
-        if (items.size() > 0) {
-            rerankedItems.addAll(items);
-        }
-
-        items.clear();
-        items.addAll(rerankedItems);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/SimilarityUtils.java

@@ -1,116 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.google.common.collect.Sets;
-import com.google.common.base.Function;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import org.apache.commons.lang3.tuple.Pair;
-
-import javax.annotation.Nullable;
-
-
-public class SimilarityUtils {
-
-    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameUserTagOrCategoryFunc() {
-        return new Function<Pair<RankItem, RankItem>, Boolean>() {
-            @Nullable
-            @Override
-            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
-                return isSameCategory(itemPair.getLeft(), itemPair.getRight()) || isSameUserTag(itemPair.getLeft(), itemPair.getRight());
-            }
-        };
-    }
-
-    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameTitleTagFunc() {
-        return new Function<Pair<RankItem, RankItem>, Boolean>() {
-            @Nullable
-            @Override
-            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
-                return isSameTitleTag(itemPair.getLeft(), itemPair.getRight());
-            }
-        };
-    }
-
-    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameUserTagFunc() {
-        return new Function<Pair<RankItem, RankItem>, Boolean>() {
-            @Nullable
-            @Override
-            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
-                return isSameUserTag(itemPair.getLeft(), itemPair.getRight());
-            }
-        };
-    }
-
-    public static Function<Pair<RankItem, RankItem>, Boolean> getIsSameCategoryFunc() {
-        return new Function<Pair<RankItem, RankItem>, Boolean>() {
-            @Nullable
-            @Override
-            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
-                return isSameCategory(itemPair.getLeft(), itemPair.getRight());
-            }
-        };
-    }
-
-    public static Function<Pair<RankItem, RankItem>, Boolean> getIsUserTagFunc() {
-        return new Function<Pair<RankItem, RankItem>, Boolean>() {
-            @Nullable
-            @Override
-            public Boolean apply(@Nullable Pair<RankItem, RankItem> itemPair) {
-                return isSameUserTag(itemPair.getLeft(), itemPair.getRight());
-            }
-        };
-    }
-
-    public static boolean isSameCategory(RankItem rankerItem1, RankItem rankerItem2) {
-        if (rankerItem1 == null || rankerItem2 == null ||
-                rankerItem1.getRankItemCategories() == null ||
-                rankerItem2.getRankItemCategories() == null) {
-            return false;
-        }
-
-        return !Sets.intersection(rankerItem1.getRankItemCategories().keySet(),
-                        rankerItem2.getRankItemCategories().keySet())
-                .isEmpty();
-    }
-
-    public static boolean isSameTitleTag(RankItem rankerItem1, RankItem rankerItem2) {
-        if (rankerItem1 == null || rankerItem2 == null ||
-                rankerItem1.getRankItemTitleTags() == null ||
-                rankerItem2.getRankItemTitleTags() == null) {
-            return false;
-        }
-        return !Sets.intersection(rankerItem1.getRankItemTitleTags().keySet(),
-                        rankerItem2.getRankItemTitleTags().keySet())
-                .isEmpty();
-    }
-
-    public static boolean isSameUserTag(RankItem rankerItem1, RankItem rankerItem2) {
-        if (rankerItem1 == null || rankerItem2 == null ||
-                rankerItem1.getRankItemUserTags() == null ||
-                rankerItem2.getRankItemUserTags() == null) {
-            return false;
-        }
-        return !Sets.intersection(rankerItem1.getRankItemUserTags().keySet(),
-                        rankerItem2.getRankItemUserTags().keySet())
-                .isEmpty();
-    }
-
-    public static boolean isSameQueue(RankItem rankerItem1, RankItem rankerItem2) {
-        if (rankerItem1 == null || rankerItem2 == null ||
-                rankerItem1.getQueue() == null) {
-            return false;
-        }
-        return rankerItem1.getQueue().equals(rankerItem2.getQueue());
-    }
-
-    public static boolean isSameSubCategory(RankItem item1, RankItem item2) {
-        if (item1 == null || item2 == null || item1.getRankItemSubCategories() == null
-                || item2.getRankItemSubCategories() == null) {
-            return false;
-        }
-
-        return !Sets.intersection(item1.getRankItemSubCategories().keySet(),
-                        item2.getRankItemSubCategories().keySet())
-                .isEmpty();
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/SimpleMergeQueue.java

@@ -1,36 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.apache.commons.lang3.tuple.Pair;
-
-import java.util.*;
-
-
-public class SimpleMergeQueue extends StrategyQueue {
-    public SimpleMergeQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
-        return MergeUtils.simpleMergeWithProtection(items, rankerItemsListMap, recNum, expId);
-    }
-
-    @Override
-    public int candidate(Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
-        return simpleCandidateByPercentage(candidateMap, recallNum, user, requestData, requestIndex, expId);
-    }
-
-    @Override
-    public String toString() {
-        return "SimpleMergeQueue{" +
-                "rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueue.java

@@ -1,194 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.google.common.collect.Maps;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import lombok.Data;
-import org.apache.commons.lang3.tuple.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.*;
-
-@Data
-public abstract class StrategyQueue {
-    // 得等召回结果，定义为RankItem
-    protected final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsList = new HashMap<>();
-    private final Logger LOGGER = LoggerFactory.getLogger(StrategyQueue.class);
-    protected Map<String, StrategyQueue> children = Maps.newHashMap();
-
-    protected List<RankItem> items = new ArrayList<>();
-    private StrategyQueueInfo strategyQueueInfo;
-    private List<StrategyQueue> allQueues = null;
-
-    public StrategyQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        this.strategyQueueInfo = strategyQueueInfo;
-
-        // 自顶向下构建child queue 初始化
-        for (String childName : strategyQueueInfo.getChildren()) {
-            if (!strategyQueueConfig.getStrategyQueueInfoMap().containsKey(childName)) {
-                LOGGER.error("No config for Child queue [{}]", childName);
-                continue;
-            }
-            StrategyQueueInfo childQueueInfo = strategyQueueConfig.getStrategyQueueInfoMap().get(childName);
-
-            StrategyQueue childQueue = null;
-            String childQueueClass = childQueueInfo.getQueueClass();
-            try {
-                childQueue = (StrategyQueue) Class.forName(childQueueClass)
-                        .getConstructor(childQueueInfo.getClass(), strategyQueueConfig.getClass())
-                        .newInstance(childQueueInfo, strategyQueueConfig);
-            } catch (InstantiationException e) {
-                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
-            } catch (IllegalAccessException e) {
-                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
-            } catch (ClassNotFoundException e) {
-                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
-            } catch (NoSuchMethodException e) {
-                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
-            } catch (InvocationTargetException e) {
-                LOGGER.error("construct StrategyQueue {}: [{}]", childName + " [" + childQueueClass + "]", e);
-            }
-            if (childQueue != null) {
-                putChild(childName, childQueue);
-            } else {
-                LOGGER.error("construct child queue [{}] failed", childName);
-            }
-        }
-
-    }
-
-    public List<StrategyQueue> getAllQueues() {
-        if (allQueues == null) {
-            allQueues = new LinkedList<>();
-
-            if (!strategyQueueInfo.isLeaf()) {
-                for (Map.Entry<String, StrategyQueue> child : children.entrySet()) {
-                    allQueues.addAll(child.getValue().getAllQueues());
-                }
-            }
-
-            allQueues.add(this);
-        }
-        return allQueues;
-    }
-
-
-    private void putChild(String queueName, StrategyQueue child) {
-        children.put(queueName, child);
-    }
-
-
-    public void clearItems() {
-        items.clear();
-    }
-
-    public void addItem(RankItem item) {
-        this.items.add(item);
-    }
-
-    public abstract int doMerge(final Map<String, Pair<MergeRule, List<RankItem>>> rankerItemsListMap, final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId);
-
-    // 定义索引Key 生成方式
-    public abstract int candidate(Map<String, Candidate> candidateMap, final int recallNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId);
-
-    public final int merge(final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
-        this.beforeMerge(recNum, user, requestData, requestIndex, expId);
-        this.doMerge(rankerItemsList, recNum, user, requestData, requestIndex, expId);
-        this.doDeDup();
-        return this.afterMerge();
-    }
-
-    // Merge 前从每个Queue中获取Item
-    // 放入RankerItem中
-    public void beforeMerge(final int recNum, final User user, final RecommendRequest requestData, final int requestIndex, final int expId) {
-        if (strategyQueueInfo.isLeaf()) {
-            LOGGER.debug("leaf queue [{}] with [{}] items to merge", getStrategyQueueInfo().getQueueName(), items.size());
-        } else {
-            rankerItemsList.clear();
-            clearItems();
-            int queueCount = 0, itemCount = 0;
-            for (MergeRule rule : strategyQueueInfo.getRulesList()) {
-                if (rule.isDisabled(expId)) {
-                    continue;
-                }
-                int myRecNum = Math.min(recNum, rule.maxMergeNum);
-                //递归的执行子节点的merge逻辑
-                int actualRecNum = children.get(rule.queueName).merge(myRecNum, user, requestData, requestIndex, expId);
-                queueCount += 1;
-                itemCount += actualRecNum;
-                rankerItemsList.put(rule.queueName, Pair.of(rule, children.get(rule.queueName).getItems()));
-            }
-            LOGGER.debug(getStrategyQueueInfo().getQueueName() + " merge info: [{}] queues with [{}] items to merge", queueCount, itemCount);
-        }
-    }
-
-    /**
-     * id dedup 去重操作
-     *
-     * @return
-     */
-    public final int doDeDup() {
-        List<RankItem> pureItems = new ArrayList<RankItem>();
-        Set<String> deDupItems = new HashSet<String>();
-        for (RankItem item : items) {
-            if (deDupItems.contains(item.getId())) {
-                continue;
-            }
-            deDupItems.add(item.getId());
-            pureItems.add(item);
-        }
-        items = pureItems;
-        return items.size();
-    }
-
-    // 给item增加融合队列名称
-    public int afterMerge() {
-        int n = items.size();
-        for (RankItem item : items) {
-            item.addMergeQueuePath(getStrategyQueueInfo().getQueueName());
-        }
-        LOGGER.debug("number of items after [{}] merge: [{}]", getStrategyQueueInfo().getQueueName(), n);
-        return n;
-    }
-
-    // 读取子队列的merge规则，按队列生成candidate
-    protected int simpleCandidateByPercentage(Map<String, Candidate> candidateMap, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int n = 0;
-
-        for (MergeRule rule : strategyQueueInfo.getRulesList()) {
-            if (rule.isDisabled(expId)) {
-                continue;
-            }
-
-            int myRecallNum = (int) (recallNum * rule.recallPercentage);
-            if (!children.containsKey(rule.queueName)) {
-                LOGGER.error("Rule for not exist child queue [{}]", rule.queueName);
-            }
-
-            Map<String, Candidate> myCandidates = new HashMap<>();
-            StrategyQueue strategyQueue = children.get(rule.queueName);
-            if (strategyQueue != null) {
-                n += strategyQueue.candidate(myCandidates, myRecallNum, user, requestData, requestIndex, expId);
-            }
-            candidateMap.putAll(myCandidates);
-        }
-
-        return n;
-    }
-
-    @Override
-    public String toString() {
-        return "StrategyQueue{" +
-                "rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                ", strategyQueueInfo=" + strategyQueueInfo +
-                ", allQueues=" + allQueues +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueueConfig.java

@@ -1,229 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigObject;
-import com.typesafe.config.ConfigValue;
-import lombok.Data;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.*;
-
-@Data
-public class StrategyQueueConfig {
-    private final Logger LOGGER = LoggerFactory.getLogger(StrategyQueueConfig.class);
-
-    private static final String CHILDREN_STRING = "children";
-    private static final String STRATEGY_QUEUE_CLASS_CONF_STRING = "class";
-
-    public static final String DEFAULT_STRATEGY_CLASS_NAME = "com.tzld.piaoquan.recommend.server.framework.merger.SimpleMergeQueue";
-
-    private static final String MERGE_RULE_STRING = "merge-rule";
-    private static final String QUEUE_NAME_STRING = "queue-name";
-    private static final String RECALL_PERCENTAGE_STRING = "recall-percentage";
-    private static final String ENABLE_EXPERIMENT_LIST_STRING = "enable-exp";
-    private static final String DISABLE_EXPERIMENT_LIST_STRING = "disable-exp";
-    private static final String MIN_MERGE_NUM_STRING = "min-merge-num";
-    private static final String MAX_MERGE_NUM_STRING = "max-merge-num";
-
-    private static final Set<String> RESERVED_PROPERTIES = new HashSet<String>(Arrays.asList(QUEUE_NAME_STRING, RECALL_PERCENTAGE_STRING, ENABLE_EXPERIMENT_LIST_STRING, DISABLE_EXPERIMENT_LIST_STRING, MIN_MERGE_NUM_STRING, MAX_MERGE_NUM_STRING));
-
-    Map<String, StrategyQueueInfo> strategyQueueInfoMap;
-
-
-    public boolean load(Config config) {
-        config.checkValid(config, "queue-config");
-        Config queueConf = config.getConfig("queue-config");
-        if (loadMergeQueues(queueConf)) {
-            LOGGER.debug("Load queues config success");
-        } else {
-            LOGGER.error("Load queues config failed");
-            return false;
-        }
-
-        config.checkValid(config, "rule-config");
-        Config rulesConf = config.getConfig("rule-config");
-        if (loadMergeRules(rulesConf)) {
-            LOGGER.debug("Load rules config success");
-        } else {
-            LOGGER.error("Load rules config failed");
-            return false;
-        }
-        // TODO: validate queue structure & rules
-        for (String queueName : strategyQueueInfoMap.keySet()) {
-            StrategyQueue queue = null;
-            try {
-                queue = MergeUtils.constructStrategyQueue(queueName, this);
-            } catch (ClassNotFoundException e) {
-                e.printStackTrace();
-            } catch (NoSuchMethodException e) {
-                e.printStackTrace();
-            } catch (IllegalAccessException e) {
-                e.printStackTrace();
-            } catch (InvocationTargetException e) {
-                e.printStackTrace();
-            } catch (InstantiationException e) {
-                e.printStackTrace();
-            }
-
-            if (queue == null) {
-                LOGGER.error("Queue config info & rules error");
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    public boolean load(String configFile) {
-        Config queueConfig = ConfigFactory.parseResources(configFile);
-        return load(queueConfig);
-    }
-
-    private boolean loadMergeQueues(Config queueConf) {
-        strategyQueueInfoMap = new HashMap<String, StrategyQueueInfo>();
-
-        ConfigObject confObj = queueConf.root();
-        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
-            String myName = it.getKey();
-            Config myConf = ((ConfigObject) it.getValue()).toConfig();
-            if (loadMergeQueueInfo(myName, myConf)) {
-                LOGGER.debug("Load queue [{}] info success", myName);
-            } else {
-                LOGGER.error("Load queue [{}] info failed", myName);
-                return false;
-            }
-        }
-        return true;
-    }
-
-    private boolean loadMergeQueueInfo(String name, Config conf) {
-        LOGGER.debug("Start load queue [{}]", name);
-        if (!strategyQueueInfoMap.containsKey(name)) {
-            strategyQueueInfoMap.put(name, new StrategyQueueInfo(name));
-        }
-        StrategyQueueInfo mergeQueueInfo = strategyQueueInfoMap.get(name);
-
-        if (conf.hasPath(STRATEGY_QUEUE_CLASS_CONF_STRING)) {
-            mergeQueueInfo.setQueueClass(conf.getString(STRATEGY_QUEUE_CLASS_CONF_STRING));
-        } else {
-            mergeQueueInfo.setQueueClass(DEFAULT_STRATEGY_CLASS_NAME);
-        }
-
-        // 递归的把child节点放在queue中
-        if (conf.hasPath(CHILDREN_STRING)) {
-            ConfigObject childrenConf = conf.getObject(CHILDREN_STRING);
-            for (ConfigObject.Entry<String, ConfigValue> it : childrenConf.entrySet()) {
-                String key = it.getKey();
-                Config childConf = ((ConfigObject) it.getValue()).toConfig();
-                String childName = key;
-
-                boolean isLoadSuccess = loadMergeQueueInfo(childName, childConf);
-                if (isLoadSuccess) {
-                    LOGGER.debug("Load queue [{}] SUCCESS", childName);
-                } else {
-                    LOGGER.error("Load queue [{}] FAILED: [{}]", childName, it.getValue());
-                    return false;
-                }
-
-                mergeQueueInfo.addChild(childName);
-            }
-        } else {
-            mergeQueueInfo.setLeaf(true);
-        }
-
-        LOGGER.debug("Load queue [{}] success", name);
-        return true;
-    }
-
-    private boolean loadMergeRules(Config conf) {
-        ConfigObject confObj = conf.root();
-
-        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
-            String myName = it.getKey();
-            Config subTreeConf = ((ConfigObject) it.getValue()).toConfig();
-            boolean isParseSuccess = parseMergeRule(myName, subTreeConf);
-            if (isParseSuccess) {
-                LOGGER.debug("Parse rules for queue [{}] SUCCESS", myName);
-            } else {
-                LOGGER.error("Parse rules for queue [{}] FAILED: [{}]", myName, it.getValue());
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    private boolean parseMergeRule(String name, Config ruleConf) {
-        if (!strategyQueueInfoMap.containsKey(name)) {
-            LOGGER.error("skip rules for not exist queue [{}]", name);
-            return true;
-        } else {
-            LOGGER.debug("parse rules for queue [{}]", name);
-        }
-
-        if (!ruleConf.hasPath(MERGE_RULE_STRING)) {
-            LOGGER.debug("no merge rules for queue [{}]", name);
-            return true;
-        }
-
-        ConfigObject ruleConfObj = ruleConf.getObject(MERGE_RULE_STRING);
-
-        for (ConfigObject.Entry<String, ConfigValue> it : ruleConfObj.entrySet()) {
-            String ruleName = it.getKey();
-            MergeRule rule = new MergeRule();
-
-            Config myConf = ((ConfigObject) it.getValue()).toConfig();
-
-            rule.recallPercentage = getOrDefault(myConf, RECALL_PERCENTAGE_STRING, 0.1);
-            rule.minMergeNum = getOrDefault(myConf, MIN_MERGE_NUM_STRING, 0);
-            rule.maxMergeNum = getOrDefault(myConf, MAX_MERGE_NUM_STRING, 500);
-
-            // 子队列名规则: 用 queue-name 显式指定, 默认为对应规则名
-            rule.queueName = getOrDefault(myConf, QUEUE_NAME_STRING, ruleName);
-            if (myConf.hasPath(ENABLE_EXPERIMENT_LIST_STRING)) {
-                List<Integer> expIds = myConf.getIntList(ENABLE_EXPERIMENT_LIST_STRING);
-                rule.enableExpIdSet = new HashSet<Integer>(expIds);
-            }
-            if (myConf.hasPath(DISABLE_EXPERIMENT_LIST_STRING)) {
-                List<Integer> expIds = myConf.getIntList(DISABLE_EXPERIMENT_LIST_STRING);
-                rule.disableExpIdSet = new HashSet<Integer>(expIds);
-            }
-
-            for (ConfigObject.Entry<String, ConfigValue> properties : myConf.entrySet()) {
-                if (!RESERVED_PROPERTIES.contains(properties.getKey())) {
-                    rule.putProperty(properties.getKey(), myConf.getString(properties.getKey()));
-                }
-            }
-
-            if (strategyQueueInfoMap.get(name).getChildren().contains(rule.queueName)) {
-                strategyQueueInfoMap.get(name).addRule(rule);
-            }
-        }
-        return true;
-    }
-
-    private int getOrDefault(Config conf, String path, int defaultValue) {
-        if (conf.hasPath(path)) {
-            return conf.getInt(path);
-        }
-        return defaultValue;
-    }
-
-    private String getOrDefault(Config conf, String path, String defaultValue) {
-        if (conf.hasPath(path)) {
-            return conf.getString(path);
-        }
-        return defaultValue;
-    }
-
-    private double getOrDefault(Config conf, String path, double defaultValue) {
-        if (conf.hasPath(path)) {
-            return conf.getDouble(path);
-        }
-        return defaultValue;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/StrategyQueueInfo.java

@@ -1,39 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.merger;
-
-
-import lombok.Data;
-
-import java.util.*;
-
-
-/**
- * 每一个召回 or 队列的融合配置信息
- */
-
-@Data
-public class StrategyQueueInfo {
-    private String queueName;
-
-    private boolean isLeaf = false;
-
-    private String queueClass; // Full path of StrategyQueue Class
-
-    private List<MergeRule> rulesList;
-
-    private Set<String> children;
-
-    public StrategyQueueInfo(String name) {
-        this.queueName = name;
-        rulesList = new LinkedList<MergeRule>();
-        children = new HashSet<String>();
-    }
-
-    public void addRule(MergeRule rule) {
-        this.rulesList.add(rule);
-    }
-
-    public void addChild(String childName) {
-        this.children.add(childName);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/GBDTModel.java

@@ -1,196 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.util.*;
-
-
-
-public class GBDTModel extends Model {
-    private static final Logger LOGGER = LoggerFactory.getLogger(GBDTModel.class);
-    private static final float LINEAR_TRANSFORM_LOWER_BOUND = 0f;
-    private static final float LINEAR_TRANSFORM_SLOPE = 1f;
-    private int featureCount = 0;
-    private Map<String, Integer> featureIdMap = null;
-    private List<HashMap<Integer, Node>> boosterModel = null;
-
-    private float transform(final float score) {
-        return (score - LINEAR_TRANSFORM_LOWER_BOUND) * LINEAR_TRANSFORM_SLOPE;
-    }
-
-    private int getFeatureId(final String feature) {
-        return featureIdMap.containsKey(feature) ? featureIdMap.get(feature) : -1;
-    }
-
-    @Override
-    public int getModelSize() {
-        return boosterModel.size();
-    }
-
-    @Override
-    public boolean loadFromStream(InputStreamReader in) throws Exception {
-        List<HashMap<Integer, Node>> model = new ArrayList<HashMap<Integer, Node>>();
-        HashMap<Integer, Node> tree = null;
-
-        featureIdMap = new HashMap<String, Integer>();
-        featureCount = 0;
-
-        BufferedReader input = new BufferedReader(in);
-        String line;
-        while ((line = input.readLine()) != null) {
-            String[] tokens = line.trim().split(":");
-            if (tokens.length == 1) {
-                if (tokens[0].startsWith("booster")) {
-                    if (tree != null && tree.size() > 0) {
-                        model.add(tree);
-                    }
-
-                    tree = new HashMap<Integer, Node>();
-                }
-            } else if (tokens.length == 2) {
-                Node node = new Node();
-
-                Integer id = Integer.parseInt(tokens[0]);
-
-                String[] items = tokens[1].split(" ");
-                if (items.length == 1) {
-                    node.isLeaf = true;
-
-                    String[] dt = items[0].split("=");
-                    node.value = Float.parseFloat(dt[1]);
-                } else if (items.length == 2) {
-                    node.isLeaf = false;
-                    String[] fieldDescriptions = null;
-                    node.compareType = NodeCompareType.LT;
-                    if (items[0].substring(1, items[0].length() - 1).contains("<=")) {
-                        fieldDescriptions = items[0].substring(1, items[0].length() - 1).split("<=");
-                        node.compareType = NodeCompareType.LE;
-                    } else {
-                        fieldDescriptions = items[0].substring(1, items[0].length() - 1).split("<");
-                    }
-                    // feature to id
-                    int featureId = getFeatureId(fieldDescriptions[0]);
-                    if (featureId < 0) {
-                        featureIdMap.put(fieldDescriptions[0], featureCount);
-                        featureId = featureCount;
-                        featureCount += 1;
-                    }
-                    node.splitFeatureId = featureId;
-
-                    if (fieldDescriptions.length == 1) {
-                        node.fieldType = NodeFieldType.BINARY;
-                    } else {
-                        node.fieldType = NodeFieldType.QUANTITATIVE;
-                        node.splitCondition = Float.parseFloat(fieldDescriptions[1]);
-                    }
-
-                    String[] childrenDescriptions = items[1].split(",");
-                    for (String childDescription : childrenDescriptions) {
-                        String[] descs = childDescription.split("=");
-                        if (descs[0].equals("yes")) {
-                            node.yes = Integer.parseInt(descs[1]);
-                        } else if (descs[0].equals("no")) {
-                            node.no = Integer.parseInt(descs[1]);
-                        } else if (descs[0].equals("missing")) {
-                            node.missing = Integer.parseInt(descs[1]);
-                        }
-                    }
-                }
-                tree.put(id, node);
-            }
-        }
-        if (tree != null && !tree.isEmpty()) {
-            model.add(tree);
-        }
-
-        LOGGER.info("Boosted tree model load over and tree number is " + model.size());
-        input.close();
-        in.close();
-        if (model != null && model.size() > 0) {
-            boosterModel = model;
-            return true;
-        } else {
-            return false;
-        }
-    }
-
-    private float score(final Map<Integer, Node> tree, final boolean[] featureIsExists, final Float[] featureValues) {
-        int id = 0;
-        while (true) {
-            Node currentNode = tree.get(id);
-            if (currentNode.isLeaf) {
-                return currentNode.value;
-            }
-
-            if (currentNode.fieldType == NodeFieldType.BINARY) {
-                if (!featureIsExists[currentNode.splitFeatureId]) {
-                    id = currentNode.no;
-                } else {
-                    id = currentNode.yes;
-                }
-            } else if (currentNode.fieldType == NodeFieldType.QUANTITATIVE) {
-                if (!featureIsExists[currentNode.splitFeatureId]) {
-                    id = currentNode.missing;
-                } else {
-                    Float value = featureValues[currentNode.splitFeatureId];
-
-                    if ((currentNode.compareType == NodeCompareType.LT && value < currentNode.splitCondition)
-                            || (currentNode.compareType == NodeCompareType.LE && value <= currentNode.splitCondition)) {
-                        id = currentNode.yes;
-                    } else {
-                        id = currentNode.no;
-                    }
-                }
-            } else {
-                LOGGER.error("Reach undefined condition: {}", id);
-            }
-        }
-    }
-
-    public Float score(final Map<String, Double> features, Map<String, Double> featuresScore) {
-        Float result = 0f;
-
-        boolean[] featureIsExists = new boolean[featureCount];
-        Float[] featureValues = new Float[featureCount];
-        for (Map.Entry<String, Double> entry : features.entrySet()) {
-            int featureId = getFeatureId(entry.getKey());
-            if (featureId >= 0) {
-                featureIsExists[featureId] = true;
-                featureValues[featureId] = entry.getValue().floatValue();
-            }
-        }
-
-        for (Map<Integer, Node> tree : boosterModel) {
-            result += score(tree, featureIsExists, featureValues);
-        }
-
-        Float transformedResult = transform(result);
-
-        LOGGER.debug("[calc_dwelltime]features: " + Arrays.toString(features.entrySet().toArray()) + ", prediction:" + result + " , transformed:" + transformedResult);
-
-        return transformedResult;
-    }
-
-    private enum NodeFieldType {
-        BINARY, QUANTITATIVE
-    }
-
-    private enum NodeCompareType {
-        LT, LE
-    }
-
-    private class Node {
-
-        boolean isLeaf;
-        float value;
-        NodeFieldType fieldType;
-        NodeCompareType compareType;
-        int splitFeatureId;
-        Float splitCondition;
-        Integer yes, no, missing;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/LRModel.java

@@ -1,169 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-
-import com.tzld.piaoquan.recommend.feature.model.sample.BaseFeature;
-import com.tzld.piaoquan.recommend.feature.model.sample.GroupedFeature;
-import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
-import it.unimi.dsi.fastutil.longs.Long2FloatMap;
-import it.unimi.dsi.fastutil.longs.Long2FloatOpenHashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-
-public class LRModel extends Model {
-    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
-    private static final Logger LOGGER = LoggerFactory.getLogger(LRModel.class);
-    private final int bucketBits = 10;  // power(2, 10) => 1024 个槽位
-    private List<Long2FloatMap> lrModel;
-
-    public LRModel() {
-        //配置不同环境的hdfs conf
-        this.lrModel = constructModel();
-    }
-
-    public List<Long2FloatMap> getLrModel() {
-        return lrModel;
-    }
-
-    public List<Long2FloatMap> constructModel() {
-        List<Long2FloatMap> initModel = new ArrayList<Long2FloatMap>();
-        int buckets = (int) Math.pow(2, bucketBits);
-        for (int i = 0; i < buckets; i++) {
-            Long2FloatMap internalModel = new Long2FloatOpenHashMap();
-            internalModel.defaultReturnValue(0.0f);
-            initModel.add(internalModel);
-        }
-        return initModel;
-    }
-
-    public int getBucket(long featureHash) {
-        return (int) (((featureHash >> bucketBits) << bucketBits) ^ featureHash);
-    }
-
-    public void putFeature(List<Long2FloatMap> model, long featureHash, float weight) {
-        model.get(getBucket(featureHash)).put(featureHash, weight);
-    }
-
-    public float getWeight(List<Long2FloatMap> model, long featureHash) {
-        return model.get(getBucket(featureHash)).get(featureHash);
-    }
-
-    @Override
-    public int getModelSize() {
-        if (this.lrModel == null)
-            return 0;
-        int sum = 0;
-        for (Map<Long, Float> model : this.lrModel) {
-            sum += model.size();
-        }
-        return sum;
-    }
-
-    public void cleanModel() {
-        this.lrModel = null;
-    }
-
-    public Float score(LRSamples lrSamples) {
-        float sum = 0.0f;
-        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
-            GroupedFeature groupedFeature = lrSamples.getFeatures(i);
-            if (groupedFeature != null && groupedFeature.getFeaturesCount() != 0) {
-                for (int j = 0; j < groupedFeature.getFeaturesCount(); j++) {
-                    BaseFeature baseFeature = groupedFeature.getFeatures(j);
-                    if (baseFeature != null) {
-                        float weight = getWeight(this.lrModel, baseFeature.getIdentifier());
-                        baseFeature.toBuilder().setWeight(weight);
-                        sum += weight;
-                    }
-                }
-            }
-        }
-
-        float pro = (float) (1.0f / (1 + Math.exp(-sum)));
-        lrSamples.toBuilder().setPredictCtr(pro);
-        return pro;
-    }
-
-    public Float getWeights(LRSamples lrSamples) {
-        float sum = 0.0f;
-
-        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
-            GroupedFeature gf = lrSamples.getFeatures(i);
-            if (gf != null && gf.getFeatures(i) != null) {
-                for (int j = 0; j < gf.getFeaturesCount(); j++) {
-                    BaseFeature fea = gf.getFeatures(j);
-                    if (fea != null) {
-                        float tmp = getWeight(this.lrModel, fea.getIdentifier());
-                        fea.toBuilder().setWeight(tmp);
-                        sum += tmp;
-                    }
-                }
-            }
-        }
-        lrSamples.toBuilder().setWeight(sum);
-        return sum;
-    }
-
-    /**
-     * 目前模型比较大,分两个阶段load模型
-     * (1). load 8M 模型, 并更新;
-     * (2). load 剩余的模型
-     * 中间提供一段时间有损的打分服务
-     *
-     * @param in
-     * @return
-     * @throws IOException
-     */
-    @Override
-    public boolean loadFromStream(InputStreamReader in) throws IOException {
-
-        List<Long2FloatMap> model = constructModel();
-        BufferedReader input = new BufferedReader(in);
-        String line = null;
-        int cnt = 0;
-
-        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
-        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", lrModel.size(), curTime);
-        //first stage
-        while ((line = input.readLine()) != null) {
-            String[] items = line.split("\t");
-            if (items.length < 2) {
-                continue;
-            }
-
-            putFeature(model, new BigInteger(items[0].trim()).longValue(), Float.valueOf(items[1].trim()).floatValue());
-            if (cnt++ < 10) {
-                LOGGER.debug("fea: " + items[0] + ", weight: " + items[1]);
-            }
-            if (cnt > MODEL_FIRST_LOAD_COUNT) {
-                break;
-            }
-        }
-        //model update
-        this.lrModel = model;
-        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
-        //final stage
-        while ((line = input.readLine()) != null) {
-            String[] items = line.split("\t");
-            if (items.length < 2) {
-                continue;
-            }
-            putFeature(model, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
-        }
-        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
-
-        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
-        input.close();
-        in.close();
-        return true;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/Model.java

@@ -1,11 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-
-import java.io.InputStreamReader;
-
-abstract public class Model {
-    public abstract int getModelSize();
-
-    public abstract boolean loadFromStream(InputStreamReader in) throws Exception;
-}
-

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ModelManager.java

@@ -1,238 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-
-import com.aliyun.oss.OSS;
-import com.aliyun.oss.OSSClientBuilder;
-import com.aliyun.oss.common.auth.CredentialsProvider;
-import com.aliyun.oss.common.auth.DefaultCredentialProvider;
-import com.aliyun.oss.model.OSSObject;
-import com.ctrip.framework.apollo.Config;
-import com.ctrip.framework.apollo.ConfigService;
-import lombok.extern.slf4j.Slf4j;
-
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-
-@Slf4j
-public class ModelManager {
-    private static final int SCHEDULE_PERIOD = 10;
-    private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
-    private static ModelManager instance;
-    Map<String, ModelLoadTask> loadTasks = new HashMap<>();
-    Map<String, String> modelPathMap = new HashMap<>();
-    private OSS client;
-    private String bucketName;
-
-    private final String modelOssEndpoint = "model.oss.internal.endpoint";
-    private final String modelOssAccessKeyId = "model.oss.accessKeyId";
-    private final String modelOssAccessKeySecret = "model.oss.accessKetSecret";
-    private final String modelOssBucketName = "model.oss.bucketName";
-
-    private ModelManager() {
-        // config load
-        Config config = ConfigService.getAppConfig();
-        String endpoint = config.getProperty(modelOssEndpoint, "");
-        String accessKeyId = config.getProperty(modelOssAccessKeyId, "");
-        String accessKetSecret = config.getProperty(modelOssAccessKeySecret, "");
-        // oss client
-        CredentialsProvider credentialsProvider = new DefaultCredentialProvider(accessKeyId, accessKetSecret);
-        this.client = new OSSClientBuilder().build(endpoint, credentialsProvider);
-        this.bucketName = config.getProperty(modelOssBucketName, "");
-
-        config.addChangeListener(changeEvent -> {
-            if (changeEvent.isChanged(modelOssEndpoint)
-                    || changeEvent.isChanged(modelOssAccessKeyId)
-                    || changeEvent.isChanged(modelOssAccessKeySecret)) {
-                String endpointNew = config.getProperty(modelOssEndpoint, "");
-                String accessKeyIdNew = config.getProperty(modelOssAccessKeyId, "");
-                String accessKetSecretNew = config.getProperty(modelOssAccessKeySecret, "");
-                CredentialsProvider credentialsProviderNew = new DefaultCredentialProvider(accessKeyIdNew,
-                        accessKetSecretNew);
-                this.client = new OSSClientBuilder().build(endpointNew, credentialsProviderNew);
-            }
-            if (changeEvent.isChanged(modelOssBucketName)) {
-                this.bucketName = config.getProperty(modelOssBucketName, "");
-            }
-        });
-
-
-        start(SCHEDULE_PERIOD);
-    }
-
-    public static ModelManager getInstance() {
-        if (instance == null) {
-            synchronized (ModelManager.class) {
-                if (instance == null) {
-                    instance = new ModelManager();
-                }
-            }
-        }
-        return instance;
-    }
-
-    /**
-     * 添加一个加载任务到管理器
-     *
-     * @param modelName  Model的名字, 注册到ModelManager的不同model需要不同的名字
-     * @param path       Model在OSS上的全路径
-     * @param modelClass Model的子类型
-     */
-    public void registerModel(String modelName, String path, Class<? extends Model> modelClass) throws ModelRegisterException, IOException {
-        if (modelPathMap.containsKey(modelName)) {
-            // fail fast
-            // throw new RuntimeException(modelName + " already exists");
-            // hard code  广告需要视频模型打分数据，配置要分开
-            return;
-
-        }
-
-        modelPathMap.put(modelName, path);
-        if (loadTasks.containsKey(path)) {
-            ModelLoadTask loadTask = loadTasks.get(path);
-            loadTask.refCount++;
-        } else {
-            ModelLoadTask task = new ModelLoadTask(path, modelClass);
-            task.refCount++;
-            loadTasks.put(path, task);
-            loadModel(task, false, true);
-        }
-    }
-
-    /**
-     * 删除一个加载任务
-     *
-     * @param modelName Model的名字, 需要和registerModel的名字一致
-     */
-    private void unRegisterModel(String modelName) {
-        if (modelPathMap.containsKey(modelName)) {
-            String path = modelPathMap.get(modelName);
-            if (loadTasks.containsKey(path)) {
-                ModelLoadTask task = loadTasks.get(path);
-                task.refCount--;
-                if (task.refCount == 0) {
-                    loadTasks.remove(path);
-                }
-            }
-            modelPathMap.remove(modelName);
-        }
-    }
-
-    /**
-     * @param modelName
-     * @return
-     */
-    public Model getModel(String modelName) {
-        if (modelPathMap.containsKey(modelName) && loadTasks.containsKey(modelPathMap.get(modelName))) {
-            return loadTasks.get(modelPathMap.get(modelName)).model;
-        } else {
-            return null;
-        }
-    }
-
-    /**
-     * 开始调度
-     *
-     * @param period
-     */
-    protected void start(long period) {
-        final Runnable task = new Runnable() {
-            public void run() {
-                // 模型更新开关
-                // boolean modelUpdateSwitch = Configuration.getBoolean("recommend-service-framework.model-update-switch", true);
-                boolean modelUpdateSwitch = true;
-                log.info("model update switch [{}]", modelUpdateSwitch);
-                if (modelUpdateSwitch) {
-                    updateModels(false);
-                }
-            }
-        };
-        scheduler.scheduleAtFixedRate(task, 10, period, TimeUnit.MINUTES); // 10分钟
-    }
-
-    /**
-     * 更新模型
-     */
-    public void updateModels(final boolean isForceLoads) {
-        log.info("begin to update: [{}]", loadTasks.keySet().size());
-        for (String modelPath : loadTasks.keySet()) {
-            log.debug("load task model path [{}]", modelPath);
-            ModelLoadTask task = loadTasks.get(modelPath);
-            loadModel(task, isForceLoads, false);
-        }
-    }
-
-    /**
-     * 检查并加载模型
-     * <p>
-     * 从oss加载：
-     * https://help.aliyun.com/zh/oss/developer-reference/streaming-download-7?spm=a2c4g.11186623.0.0.4b527c7dm8LejC
-     *
-     * @param loadTask
-     */
-    private void loadModel(final ModelLoadTask loadTask, final boolean isForceLoads, final boolean isRegister) {
-        if (loadTask.isLoading) {
-            return;
-        }
-        loadTask.isLoading = true;
-        OSSObject ossObj = null;
-        try {
-            ossObj = client.getObject(bucketName, loadTask.path);
-            long timeStamp = ossObj.getObjectMetadata().getLastModified().getTime();
-            if (loadTask.lastModifyTime < timeStamp || isForceLoads) {
-                log.info("model file changed, ready to update, last modify: [{}], current model time: [{}]",
-                        loadTask.lastModifyTime, timeStamp);
-
-                Model model = loadTask.modelClass.newInstance();
-                if (model.loadFromStream(new InputStreamReader(ossObj.getObjectContent()))) {
-                    loadTask.model = model;
-                    loadTask.lastModifyTime = timeStamp;
-                }
-            }
-            ossObj.close();
-        } catch (Exception e) {
-            log.error("update model fail", e);
-        } finally {
-            loadTask.isLoading = false;
-            if (ossObj != null) {
-                try {
-                    ossObj.close();
-                } catch (IOException e) {
-                    log.error("close ossObj fail", e);
-                }
-            }
-        }
-    }
-
-    public class ModelRegisterException extends Exception {
-
-        public ModelRegisterException(String s) {
-            super(s);
-        }
-    }
-
-    /**
-     * 调度的任务单元
-     */
-    private class ModelLoadTask {
-
-        private int refCount;
-        private final String path;
-        private long lastModifyTime;
-        private boolean isLoading;
-        private final Class<? extends Model> modelClass;
-        private Model model;
-
-        ModelLoadTask(String path, Class<? extends Model> modelClass) {
-            this.refCount = 0;
-            this.path = path;
-            this.lastModifyTime = 0;
-            this.modelClass = modelClass;
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/OssConfig.java

@@ -1,15 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-import lombok.Data;
-
-/**
- * @author dyp
- */
-@Data
-public class OssConfig {
-
-    private String accessKeyId;
-    private String accessKeySecret;
-    private String endpoint;
-    private String bucketName;
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ThompsonSamplingModel.java

@@ -1,89 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.model;
-
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.base.VideoActionFeature;
-import org.apache.commons.math3.distribution.BetaDistribution;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.math.BigInteger;
-import java.util.HashMap;
-import java.util.Map;
-
-
-public class ThompsonSamplingModel extends Model {
-    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25;  // 32M
-    private static final Logger LOGGER = LoggerFactory.getLogger(ThompsonSamplingModel.class);
-
-    // key = videoid, value = < push, exp, play, realplay, share, retures >
-    private Map<Long, VideoActionFeature> thompsonSamplingModel;
-
-    private static final int alpha = 20;
-    private static final int beta_returns = 100;
-
-    public ThompsonSamplingModel() {
-        //配置不同环境的hdfs conf
-        this.thompsonSamplingModel = new HashMap<>();
-    }
-
-    public Map<Long, VideoActionFeature> getThompsonSamplingModel() {
-        return this.thompsonSamplingModel;
-    }
-
-
-
-
-    @Override
-    public boolean loadFromStream(InputStreamReader in) throws IOException {
-        Map<Long, VideoActionFeature> initModel = new HashMap<>();
-        BufferedReader input = new BufferedReader(in);
-        String line = null;
-        int cnt = 0;
-        while ((line = input.readLine()) != null) {
-            String[] items = line.split("\t");
-            if (items.length < 3) {
-                continue;
-            }
-            Long videoId = new BigInteger(items[0].trim()).longValue();
-            VideoActionFeature videoFeature = new VideoActionFeature();
-            videoFeature.setView(Double.valueOf(items[1].trim()));
-            videoFeature.setPlay(Double.valueOf(items[2].trim()));
-            videoFeature.setShare(Double.valueOf(items[3].trim()));
-            videoFeature.setReturns(Double.valueOf(items[5].trim()));
-            initModel.put(videoId, videoFeature);
-        }
-
-        this.thompsonSamplingModel = initModel;
-        input.close();
-        in.close();
-        return true;
-    }
-
-    @Override
-    public int getModelSize() {
-        if (this.thompsonSamplingModel == null)
-            return 0;
-        int sum = this.thompsonSamplingModel.size();
-        return sum;
-    }
-
-    public double score(RankItem rankItem) {
-        double score = 0.0f;
-        VideoActionFeature videoActionFeature = this.thompsonSamplingModel.getOrDefault(rankItem.getVideoId(), new VideoActionFeature());
-
-        int alpha = (int) videoActionFeature.getReturns() + ThompsonSamplingModel.alpha;
-        int beta = beta_returns + (int) videoActionFeature.getView();
-        score = this.betaSampler(alpha, beta);
-        return score;
-    }
-
-    public double betaSampler(double alpha, double beta) {
-        BetaDistribution betaSample = new BetaDistribution(alpha, beta);
-        return betaSample.sample();
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/AbstractFilter.java

@@ -1,31 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-public abstract class AbstractFilter<T> {
-    protected final static Logger LOGGER = LoggerFactory.getLogger(AbstractFilter.class);
-    protected final FilterConfigInfo filterConfigInfo;
-    protected final RecommendRequest requestContext;
-    protected final User user;
-
-    public AbstractFilter(FilterConfigInfo filterConfigInfo,
-                          RecommendRequest requestContext,
-                          User user) {
-
-        this.filterConfigInfo = filterConfigInfo;
-        this.requestContext = requestContext;
-        this.user = user;
-    }
-
-    public FilterConfigInfo getFilterConfigInfo() {
-        return filterConfigInfo;
-    }
-
-    public abstract void doFilter(Candidate candidate, List<T> t);
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/BaseRecaller.java

@@ -1,280 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller;
-
-
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.*;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.provider.QueueProvider;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
-import java.util.stream.Collectors;
-
-
-public class BaseRecaller<Video> {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(BaseRecaller.class);
-    private static final long DEFAULT_QUEUE_LOAD_TIMEOUT = 1500; // ms
-    private static final long DEFAULT_PARALLEL_FILTER_TIMEOUT = 1500; // ms
-
-    private static final String FILTER_CONF = "filter_config.conf"; // ms
-    private static final ExecutorService filterExecutorService = new ThreadPoolExecutor(128, 128,
-            0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1000), r -> new Thread(r, "filterExecutorService"),
-            new ThreadPoolExecutor.CallerRunsPolicy());
-
-    private static final ExecutorService fetchQueueExecutorService = new ThreadPoolExecutor(128, 128,
-            0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1000), r -> new Thread(r, "fetchQueueExecutorService"),
-            new ThreadPoolExecutor.CallerRunsPolicy());
-    private final QueueProvider<Video> queueProvider;
-
-    private static final FilterConfig filterConfig;
-
-    private final long QUEUE_LOAD_TIMEOUT;
-
-    static {
-        filterConfig = new FilterConfig(FILTER_CONF);
-    }
-
-    public BaseRecaller(QueueProvider<Video> queueProvider) {
-        this(queueProvider, DEFAULT_QUEUE_LOAD_TIMEOUT);
-    }
-
-    public BaseRecaller(QueueProvider<Video> queueProvider,
-                        long queueLoadTimeout) {
-        this.queueProvider = queueProvider;
-        this.QUEUE_LOAD_TIMEOUT = queueLoadTimeout;
-    }
-
-
-    public BaseRecaller(QueueProvider<Video> queueProvider,
-                        FilterConfig filterConfig,
-                        long queueLoadTimeout) {
-        this.queueProvider = queueProvider;
-        this.QUEUE_LOAD_TIMEOUT = queueLoadTimeout;
-    }
-
-
-    public String extractItemId(Entry<Video> entry) {
-        return entry.id;
-    }
-
-
-    public boolean isValidItem(Video item) {
-        return item != null;
-    }
-
-    /**
-     * 把queue中的entry, 补充candidiate信息，放入RankItem中
-     *
-     * @param entries
-     * @param candidate
-     * @return
-     */
-    private List<RankItem> toHits(final Iterable<Entry<Video>> entries, final Candidate candidate) {
-
-        List<RankItem> result = new ArrayList<RankItem>();
-        for (Entry entry : entries) {
-            RankItem item = new RankItem();
-            item.setId(extractItemId(entry));
-            item.putRankerScore("L3Score", (Double) entry.scores.get("ordering"));
-            item.setQueue(candidate.getMergeQueueName()); // merge queue
-
-            CandidateInfo candidateInfo = new CandidateInfo();
-            candidateInfo.setCandidateQueueName(candidate.getCandidateKey());
-            candidateInfo.setCandidate(candidate);
-
-            item.setCandidateInfo(candidateInfo);
-            item.addToCandidateInfoList(candidateInfo);
-            result.add(item);
-        }
-        return result;
-    }
-
-
-    // 读取redis中的数据放入queue中
-    public Map<Candidate, Queue<Video>> loadQueues(List<Candidate> candidates) {
-        // update queueName
-        Iterable<Candidate> updateCandidates = FluentIterable.from(candidates).transform(candidate -> {
-            try {
-                long ttl = QueueName.DEFAULT_LOCAL_CACHE_TTL;
-                candidate.setCandidateQueueName(QueueName.fromString(candidate.getCandidateKey(), ttl));
-            } catch (Exception e) {
-                candidate.setCandidateQueueName(null);
-                LOGGER.error("error parse QueueName [{}]", candidate.getCandidateKey());
-            }
-            return candidate;
-        });
-
-        // parse queues
-        Iterable<QueueName> queueNames = FluentIterable.from(updateCandidates).transform(candidate -> candidate.getCandidateQueueName());
-
-        // parallel load queues, redis 或者缓存获取index
-        Map<QueueName, Queue<Video>> queues = Maps.newConcurrentMap();
-        try {
-            // 格式
-            queues = queueProvider.loads(Lists.newArrayList(queueNames), QUEUE_LOAD_TIMEOUT, TimeUnit.MILLISECONDS);
-        } catch (Exception e) {
-            LOGGER.error("load queue occur error [{}]", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        // parse candidate map
-        Map<Candidate, Queue<Video>> candidateQueueMap = Maps.newConcurrentMap();
-        for (Candidate candidate : updateCandidates) {
-            QueueName name = candidate.getCandidateQueueName();
-            Queue<Video> entries = queues.get(name);
-            if (queues.containsKey(name) && entries != null) {
-                candidateQueueMap.put(candidate, entries);
-            }
-        }
-        return candidateQueueMap;
-    }
-
-
-    /**
-     * recall
-     *
-     * @param requestData
-     * @param user
-     * @param recallCandidates
-     * @return
-     */
-    public List<RankItem> recalling(final RecommendRequest requestData, final User user, List<Candidate> recallCandidates) {
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        stopwatch.reset().start();
-        // load from redis
-        List<Callable<Map<Candidate, Queue<Video>>>> fetchQueueCalls = Lists.newArrayList();
-        fetchQueueCalls.add(() -> {
-            boolean isFromRedis = true;
-            return obtainQueue(recallCandidates, requestData, user, isFromRedis);
-        });
-
-        // 多线程执行load
-        List<Future<Map<Candidate, Queue<Video>>>> fetchQueueFutures = null;
-        try {
-            fetchQueueFutures = fetchQueueExecutorService.invokeAll(fetchQueueCalls, DEFAULT_QUEUE_LOAD_TIMEOUT,
-                    TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("[fetch queue error] inter fail: {}", ExceptionUtils.getStackTrace(e));
-        } catch (Exception e) {
-            LOGGER.error("[fetch queue error] ex", ExceptionUtils.getStackTrace(e));
-        }
-
-        Map<Candidate, Queue<Video>> candidateQueueMap = Maps.newHashMap();
-        if (CollectionUtils.isNotEmpty(fetchQueueFutures)) {
-            for (Future<Map<Candidate, Queue<Video>>> future : fetchQueueFutures) {
-                if (future.isDone() && !future.isCancelled()) {
-                    Map<Candidate, Queue<Video>> result = null;
-                    try {
-                        result = future.get();
-                    } catch (InterruptedException e) {
-                        LOGGER.error("[fetch queue error] InterruptedException {}", ExceptionUtils.getStackTrace(e));
-                    } catch (ExecutionException e) {
-                        LOGGER.error("[fetch queue error] ex {}", ExceptionUtils.getStackTrace(e));
-                    }
-                    if (result != null) {
-                        candidateQueueMap.putAll(result);
-                    }
-                }
-            }
-        }
-
-        stopwatch.reset().start();
-        List<RankItem> result = convertToRankItem(candidateQueueMap, requestData, user);
-
-        return result;
-    }
-
-
-    // 转成RankItem
-    // 同时给Filter预留处理
-    private List<RankItem> convertToRankItem(Map<Candidate, Queue<Video>> candidateQueueMap,
-                                             RecommendRequest requestData,
-                                             User user) {
-
-
-        final FilterPipeline<Video> recallFilter = new FilterPipeline<>(filterConfig, requestData, user);
-
-        final List<Callable<List<RankItem>>> callables = new ArrayList<>();
-        for (final Map.Entry<Candidate, Queue<Video>> entry : candidateQueueMap.entrySet()) {
-            callables.add(() -> {
-                List<RankItem> candidateHits = new ArrayList<>();
-                final Candidate candidate = entry.getKey();
-                List<Entry<Video>> entriesValue = CommonCollectionUtils.iterableToList(entry.getValue());
-                try {
-                    final Map<Video, Entry<Video>> entryMap = new HashMap<>();
-                    List<Video> entriesList = entriesValue.stream().map(e -> {
-                        entryMap.put(e.item, e);
-                        return e.item;
-                    }).collect(Collectors.toList());
-                    List<Video> entriesAfterFilter = recallFilter.doFilter(candidate, entriesList);
-                    // 1. filter  TODO 待后续增加自定义filter
-                    if (CollectionUtils.isNotEmpty(entriesAfterFilter)) {
-                        List<Entry<Video>> entries = entriesAfterFilter.stream()
-                                .limit(candidate.getCandidateNum())
-                                .map(entryMap::get)
-                                .collect(Collectors.toList());
-                        // 2. toHits
-                        candidateHits.addAll(toHits(entries, candidate));
-                    }
-
-                } catch (Exception e) {
-                    LOGGER.error("recall filter queue occur error, queue [{}], error: [{}]", candidate.toString(), ExceptionUtils.getFullStackTrace(e));
-                }
-                return candidateHits;
-            });
-        }
-
-        Map<String, RankItem> hits = new HashMap<>();
-        try {
-            List<Future<List<RankItem>>> futures = filterExecutorService.invokeAll(callables, DEFAULT_PARALLEL_FILTER_TIMEOUT, TimeUnit.MILLISECONDS);
-            for (Future<List<RankItem>> future : futures) {
-                try {
-                    if (future.isDone() && !future.isCancelled() && future.get() != null) {
-                        List<RankItem> part = future.get();
-                        // add to result
-                        for (RankItem item : part) {
-                            // merge candidate Info
-                            if (hits.containsKey(item.getId())) {
-                                hits.get(item.getId()).addToCandidateInfoList(item.getCandidateInfo());
-                            } else {
-                                hits.put(item.getId(), item);
-                            }
-                        }
-                        LOGGER.debug("collect items from filter task, userid: [{}], item size [{}]", "", part.size());
-                    } else {
-                        LOGGER.error("parallel recall filter Canceled {} ", "");
-                    }
-                } catch (Exception e) {
-                    LOGGER.error("parallel recall filter occur error, uid: [{}], Exception [{}]",
-                            "", ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        } catch (Exception e) {
-            LOGGER.error("parallel recall filter occur error, uid: [{}], Exception [{}]",
-                    "", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        List<RankItem> result = new ArrayList<>(hits.values());
-
-        return result;
-    }
-
-
-    private Map<Candidate, Queue<Video>> obtainQueue(List<Candidate> refactorCandidates, RecommendRequest requestData, User user, boolean isFromRedis) {
-        return loadQueues(refactorCandidates);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterConfig.java

@@ -1,92 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller;
-
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigObject;
-import com.typesafe.config.ConfigValue;
-
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
-
- */
-
-public class FilterConfig {
-    private static Logger LOGGER = LoggerFactory.getLogger(FilterConfig.class);
-    private List<FilterConfigInfo> filterConfigInfoList = new ArrayList<FilterConfigInfo>();
-
-    public FilterConfig(Config config) {
-        this.load(config);
-    }
-
-
-    public FilterConfig(String configFile) {
-        this.load(configFile);
-    }
-
-    public FilterConfig() {
-    }
-
-    public boolean load(String configFile) {
-        Config filterConfig = ConfigFactory.parseResources(configFile);
-        return load(filterConfig);
-    }
-
-    public boolean load(Config config) {
-        Config recallConfig = config.getConfig("recall-config");
-
-        Config filterConf = recallConfig.getConfig("filter-config");
-        try {
-            loadFilters(filterConf);
-        } catch (Exception e) {
-            return false;
-        }
-
-        return true;
-    }
-
-    public List<FilterConfigInfo> getFilterConfigInfoList() {
-        return filterConfigInfoList;
-    }
-
-    private void loadFilters(Config config) throws Exception {
-        ConfigObject confObj = config.root();
-        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
-            Config conf = ((ConfigObject) it.getValue()).toConfig();
-
-            // parse config
-            String configName = it.getKey();
-            String filterName = conf.getString("filter-name");
-            int filterPriority = 0;
-            if (conf.hasPath("filter-priority"))
-                filterPriority = conf.getInt("filter-priority");
-
-            List<Integer> disableExpIds = new ArrayList<Integer>();
-            if (conf.hasPath("disable-expids")) {
-                disableExpIds = conf.getIntList("disable-expids");
-            }
-            FilterConfigInfo filterConfigInfo = new FilterConfigInfo(configName,
-                    filterName, filterPriority, disableExpIds);
-            LOGGER.debug("parse filter config info [{}]", filterConfigInfo);
-
-            addConfigByPriority(filterConfigInfoList, filterConfigInfo);
-        }
-    }
-
-    private void addConfigByPriority(List<FilterConfigInfo> configInfoList, FilterConfigInfo addConfigInfo) {
-
-        int pos = 0;
-        for (; pos < configInfoList.size(); pos++) {
-            if (configInfoList.get(pos).getFilterPriority() <= addConfigInfo.getFilterPriority()) {
-                break;
-            }
-        }
-        configInfoList.add(pos, addConfigInfo);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterConfigInfo.java

@@ -1,59 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller;
-
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.FluentIterable;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
-
-public class FilterConfigInfo {
-    private Set<Integer> disableExpIds;
-    private String filterName;
-    private Integer filterPriority;
-    private String configName;
-
-    public FilterConfigInfo(String configName,
-                            String filterName,
-                            Integer filterPriority,
-                            List<Integer> expIds) {
-        this.configName = configName;
-        this.filterName = filterName;
-        this.filterPriority = filterPriority;
-        this.disableExpIds = new HashSet<Integer>();
-        this.disableExpIds.addAll(expIds);
-    }
-
-    public Integer getFilterPriority() {
-        return filterPriority;
-    }
-
-    public String getFilterName() {
-        return filterName;
-    }
-
-    public Set<Integer> getDisableExpIds() {
-        return disableExpIds;
-    }
-
-    public String getConfigName() {
-        return configName;
-    }
-
-    @Override
-    public String toString() {
-        return configName + ":" + filterPriority + ":" + filterName + ":" +
-                Joiner.on(",").join(FluentIterable.from(disableExpIds).transform(new Function<Integer, String>() {
-                    @Nullable
-                    @Override
-                    public String apply(Integer integer) {
-                        return integer.toString();
-                    }
-                }));
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/FilterPipeline.java

@@ -1,62 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-
-public class FilterPipeline<T> {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(FilterPipeline.class);
-    public int filterNum;
-    private final FilterConfig config;
-    private final List<AbstractFilter<T>> filters;
-    private final RecommendRequest requestContext;
-    private final User user;
-
-    public FilterPipeline(FilterConfig config,
-                          RecommendRequest requestContext,
-                          User user) {
-
-        this.config = config;
-        this.requestContext = requestContext;
-        this.user = user;
-        this.filters = new ArrayList<>();
-        this.constructFilters(config);
-    }
-
-    public List<AbstractFilter<T>> getFilters() {
-        return filters;
-    }
-
-    public void constructFilters(FilterConfig config) {
-        // add filter
-        if (config != null) {
-            for (FilterConfigInfo filterConfigInfo : config.getFilterConfigInfoList()) {
-                try {
-                    AbstractFilter<T> filter = (AbstractFilter) Class.forName(filterConfigInfo.getFilterName())
-                            .getConstructor(FilterConfigInfo.class, RecommendRequest.class, User.class)
-                            .newInstance(filterConfigInfo, this.requestContext, this.user);
-
-                    this.filters.add(filter);
-                } catch (Exception e) {
-                    LOGGER.error("Filter config info construct error, [{}] [{}]",
-                            filterConfigInfo, ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-    }
-
-    public List<T> doFilter(Candidate candidate, List<T> list) {
-        for (AbstractFilter filter : filters) {
-            filter.doFilter(candidate, list);
-        }
-        return list;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/CacheEntry.java

@@ -1,24 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-
-import com.google.common.base.Optional;
-import lombok.Data;
-
-@Data
-public class CacheEntry<T> {
-    private String itemId; // itemId
-    private Optional<T> item;
-    private Long expireTime; // 静态特征失效时间
-    private Long dynamicFeatureExpireTime; // 动态特征失效时间
-
-    public CacheEntry(String itemId,
-                      Long expireTime,
-                      Long dynamicFeatureExpireTime,
-                      Optional<T> item) {
-        this.itemId = itemId;
-        this.expireTime = expireTime;
-        this.dynamicFeatureExpireTime = dynamicFeatureExpireTime;
-        this.item = item;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/Index.java

@@ -1,44 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-import com.alibaba.fastjson.JSONObject;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.Data;
-import org.apache.commons.lang3.StringUtils;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-@Data
-public class Index {
-    private String indexName = null;
-    private List<IndexEntry> indexEntryList = new ArrayList<>();
-
-
-    // TODO
-    // redis中一路召回队列key 的结果 转化到Index中
-    public Index(String indexName, String indexValue) {
-        if (StringUtils.isBlank(indexName) || StringUtils.isBlank(indexValue)) {
-            return;
-        }
-        this.indexName = indexName;
-        JSONObject jsonObject = JSONObject.parseObject(indexValue);
-        String valueList = jsonObject.getString("value_list");
-        valueList = "[" + valueList + "]";
-        List<List<String>> videoScores = JSONUtils.fromJson(valueList, new TypeToken<List<List<String>>>() {
-        }, Collections.emptyList());
-
-        videoScores.stream().forEach(itemAndScore -> {
-                    IndexEntry indexEntry = new IndexEntry();
-                    if (itemAndScore.size() >= 2) {
-                        indexEntry.setItemId(itemAndScore.get(0));
-                        indexEntry.setScore(Double.valueOf(itemAndScore.get(1)));
-                        this.indexEntryList.add(indexEntry);
-                    }
-                }
-        );
-
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/IndexEntry.java

@@ -1,10 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-import lombok.Data;
-
-@Data
-public class IndexEntry {
-    private String itemId;
-    private Double score;
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/QueueProvider.java

@@ -1,20 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-
-
-
-public interface QueueProvider<T> {
-
-    Queue<T> load(QueueName name) throws Exception;
-
-    Index get(QueueName name) throws Exception;
-
-    Map<QueueName, Queue<T>> loads(List<QueueName> names) throws Exception;
-
-    Map<QueueName, Queue<T>> loads(List<QueueName> names, long timeout, TimeUnit timeUnit) throws Exception;
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/RedisBackedQueue.java

@@ -1,196 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Iterables;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.utils.FixedThreadPoolHelper;
-import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
-
-
-public class RedisBackedQueue implements QueueProvider<Video> {
-    private static final Logger logger = LoggerFactory.getLogger(RedisBackedQueue.class);
-    private static final int DEFAULT_TTL = 7 * 24 * 60 * 60;
-    private static final long CACHE_MAXIMUMSIZE = 50 * 10000; // default 50w
-    private static final long CACHE_TIMEOUT_MS = 5 * 60 * 1000L;
-
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(64);
-
-    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
-    private final ExecutorService queueRefreshExecutorService = new FixedThreadPoolHelper(4, "queueRefresh").getThreadPoolExecutor();
-
-    // 不能为static
-    private final BlockingQueue<QueueName> asyncRefreshQueue = new LinkedBlockingQueue<QueueName>(50000);
-    private final RedisSmartClient client;
-    private final LoadingCache<QueueName, Pair<Long, Queue<Video>>> cache;
-
-    /**
-     * 实例化 RedisBackedQueue, 并提供基于Guava Cache的本地缓存管理工作
-     *
-     * @param client         redis-cluster 连接池
-     * @param cacheTimeOutMs 缓存有效期, 开启缓存的情况下,默认缓存5min,以写时间为基准
-     */
-    public RedisBackedQueue(RedisSmartClient client, final long cacheTimeOutMs) {
-        this.client = client;
-
-        cache = CacheBuilder.newBuilder()
-                .expireAfterWrite((long) (cacheTimeOutMs * 1.3), TimeUnit.MILLISECONDS)
-                .maximumSize(CACHE_MAXIMUMSIZE)
-                .build(new RedisBackedQueueWithoutMetaCacheLoader(client));
-
-        // protect key
-        for (int i = 0; i < 4; i++) {
-            queueRefreshExecutorService.execute(new CacheRefreshRunnable());
-        }
-    }
-
-    public RedisBackedQueue(RedisSmartClient client) {
-        this(client, CACHE_TIMEOUT_MS);
-    }
-
-    public RedisSmartClient getClient() {
-        return client;
-    }
-
-
-    // redis读取队列内内容
-    public Index get(QueueName name) throws Exception {
-        String valueJson = client.get(name.toString());
-        Index index = new Index(name.toString(), valueJson);
-        return index;
-    }
-
-
-
-    /**
-     * 加载指定索引, 并指定索引本地缓存的TTL
-     * @param name QueueName
-     * @return
-     * @throws ExecutionException
-     */
-    public Queue<Video> load(QueueName name) throws ExecutionException {
-
-        Pair<Long, Queue<Video>> cachedQueue = cache.get(name);
-        if (cachedQueue == null) {
-            // 清理本地缓存中当前key的数据
-            cache.invalidate(name);
-            logger.debug("invalidate queue [{}]", name);
-            return new Queue<Video>(name.toString());
-        }
-        // update key refresh time
-        if (cachedQueue.getKey() == Long.MIN_VALUE) {
-            cache.put(name, Pair.of(System.currentTimeMillis() + name.getTTL(), cachedQueue.getValue()));
-        }
-
-        if (name.getTTL() == 0) {
-            cache.invalidate(name);
-            logger.debug("invalidate queue [{}]", name);
-        }
-
-        // check and refresh
-        if (name.getTTL() > 0 && System.currentTimeMillis() > cachedQueue.getKey()) {
-            boolean isok = asyncRefreshQueue.offer(name);
-            if (logger.isDebugEnabled()) {
-                logger.debug("offer async queue name [{}], result [{}]", name, isok);
-            }
-            if (asyncRefreshQueue.size() > CACHE_MAXIMUMSIZE) {
-                logger.warn("index backgroud blocking queue length [{}], it is danger", asyncRefreshQueue.size());
-            }
-            if (!isok) {
-                logger.error("index backgroud offer blocking queue error, queuename [{}], with queue lenght [{}]", name, asyncRefreshQueue.size());
-            }
-        }
-
-        return cachedQueue.getValue();
-    }
-
-    public Map<QueueName, Queue<Video>> loads(List<QueueName> names) throws ExecutionException {
-        return this.loads(names, 200, TimeUnit.MILLISECONDS);
-    }
-
-    public Map<QueueName, Queue<Video>> loads(List<QueueName> names, long timeout, TimeUnit timeUnit) throws ExecutionException {
-
-        final Iterable<List<QueueName>> namesIterable = Iterables.partition(names, 1);
-        final List<Callable<Map<QueueName, Queue<Video>>>> callables =
-                new ArrayList<Callable<Map<QueueName, Queue<Video>>>>();
-        for (final List<QueueName> queueNames : namesIterable) {
-            callables.add(() -> {
-                Map<QueueName, Queue<Video>> result = new HashMap<>();
-                for (final QueueName name : queueNames) {
-                    try {
-                        Queue queue = load(name);
-                        result.put(name, queue);
-                    } catch (Exception e) {
-                        logger.error("load queue error [{}] [{}]", name, ExceptionUtils.getFullStackTrace(e));
-                    }
-                }
-                return result;
-            });
-        }
-
-        Map<QueueName, Queue<Video>> loadQueue = new ConcurrentHashMap<QueueName, Queue<Video>>();
-        try {
-            List<Future<Map<QueueName, Queue<Video>>>> futures = executorService.invokeAll(callables);
-            for (Future<Map<QueueName, Queue<Video>>> future : futures) {
-                if (future.isDone() && !future.isCancelled()) {
-                    try {
-                        Map<QueueName, Queue<Video>> ret = future.get();
-                        loadQueue.putAll(ret);
-                    } catch (ExecutionException ee) {
-                        logger.error("Failed to execute load a queue partition for {}", ExceptionUtils.getFullStackTrace(ee));
-                    }
-                }
-            }
-        } catch (InterruptedException ie) {
-            logger.error("Interrupted when waiting for parallel queue load finish for {}", ExceptionUtils.getFullStackTrace(ie));
-        }
-        return loadQueue;
-    }
-
-
-
-    // 使用Guava Cache 的异步刷新接口, 异步刷新到期的key
-    public class CacheRefreshRunnable implements Runnable {
-
-        @Override
-        public void run() {
-            while (true) {
-                QueueName queueName = null;
-                try {
-                    queueName = asyncRefreshQueue.take(); // take async item, blocked when no item
-                    final long startTime = System.currentTimeMillis();
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("async refresh thread get queuename [{}]", queueName); // get itemid
-                    }
-
-                    Pair<Long, Queue<Video>> entry = cache.get(queueName);
-                    // check refresh
-                    if (startTime > entry.getKey()) {
-                        cache.refresh(queueName);
-                    }
-
-                    long endTime = System.currentTimeMillis();
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("parallel refresh queue [{}] spenttime [{}]", queueName, endTime - startTime);
-                    }
-                } catch (Exception e) {
-                    logger.error("schedule refresh queue error [{}], exception [{}]", queueName, ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/recaller/provider/RedisBackedQueueWithoutMetaCacheLoader.java

@@ -1,56 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.recaller.provider;
-
-import com.google.common.base.Function;
-import com.google.common.cache.CacheLoader;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Entry;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Queue;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-
-public class RedisBackedQueueWithoutMetaCacheLoader extends CacheLoader<QueueName, Pair<Long, Queue<Video>>> {
-
-    private static final Logger logger = LoggerFactory.getLogger(RedisBackedQueueWithoutMetaCacheLoader.class);
-    private final RedisSmartClient client;
-
-    public RedisBackedQueueWithoutMetaCacheLoader(RedisSmartClient client) {
-        this.client = client;
-    }
-
-    @Override
-    public Pair<Long, Queue<Video>> load(QueueName name) throws Exception {
-        String valueJson = client.get(name.toString());
-        Index index = new Index(name.toString(), valueJson);
-
-        if (null == index || CollectionUtils.isEmpty(index.getIndexEntryList())) {
-//            logger.error("empty_index_fetch");
-        }
-
-        //IndexEntry 转化到Entry中 video 格式
-        Queue<Video> queue = new Queue<Video>(name.toString());
-        Iterable<Entry<Video>> iterable = Iterables.transform(index.getIndexEntryList(), new Function<IndexEntry, Entry<Video>>() {
-            @Nullable
-            @Override
-            public Entry<Video> apply(IndexEntry indexEntry) {
-                Video recallData = new Video();
-                recallData.setVideoId(Long.valueOf(indexEntry.getItemId()));
-                recallData.setPushFrom(name.toString());
-                Entry<Video> entry = new Entry<Video>(recallData, indexEntry.getItemId());
-                entry.addScore("ordering", indexEntry.getScore());
-                return entry;
-            }
-        });
-        queue.addAll(Lists.newArrayList(iterable));
-
-        return Pair.of(Long.MIN_VALUE, queue);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/AbstractScorer.java

@@ -1,71 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.service.score.model.Model;
-import com.tzld.piaoquan.recommend.server.service.score.model.ModelManager;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-
-public abstract class AbstractScorer {
-    public static Logger LOGGER = LoggerFactory.getLogger(AbstractScorer.class);
-    protected ScorerConfigInfo scorerConfigInfo;
-    protected ModelManager modelManager = ModelManager.getInstance();
-
-    public AbstractScorer(ScorerConfigInfo scorerConfigInfo) {
-        this.scorerConfigInfo = scorerConfigInfo;
-    }
-
-    public void loadModel() {
-    }
-
-    public boolean isEnable() {
-        return !getScorerConfigInfo().getDisableSwitch();
-    }
-
-
-    public void doLoadModel(Class<? extends Model> modelClass) {
-
-        String modelPath = scorerConfigInfo.getModelPath();
-        if (StringUtils.isNotBlank(modelPath)) {
-            try {
-                // 使用 modelPath 作为 modelName 注册
-                modelManager.registerModel(modelPath, modelPath, modelClass);
-                LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
-            } catch (ModelManager.ModelRegisterException e) {
-                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
-            } catch (IOException e) {
-                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
-            }
-        } else {
-            LOGGER.error("modelpath is null, for model class [{}]", modelClass);
-        }
-    }
-
-    public Model getModel() {
-        if (StringUtils.isBlank(scorerConfigInfo.getModelPath())) {
-            return null;
-        }
-        return modelManager.getModel(scorerConfigInfo.getModelPath());
-    }
-
-    public ScorerConfigInfo getScorerConfigInfo() {
-        return scorerConfigInfo;
-    }
-
-    public abstract List<RankItem> scoring(final ScoreParam param,
-                                           final UserFeature userFeature,
-                                           final List<RankItem> rankItems);
-
-    public abstract List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
-                                           final Map<String, String> userFeatureMap,
-                                           final List<RankItem> rankItems);
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseGBDTModelScorer.java

@@ -1,19 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.tzld.piaoquan.recommend.server.service.score.model.GBDTModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class BaseGBDTModelScorer extends AbstractScorer {
-    private static Logger LOGGER = LoggerFactory.getLogger(BaseGBDTModelScorer.class);
-
-    public BaseGBDTModelScorer(ScorerConfigInfo scorerConfigInfo) {
-        super(scorerConfigInfo);
-    }
-
-    @Override
-    public synchronized void loadModel() {
-        doLoadModel(GBDTModel.class);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseLRModelScorer.java

@@ -1,20 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public abstract class BaseLRModelScorer extends AbstractScorer {
-
-    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRModelScorer.class);
-
-    public BaseLRModelScorer(ScorerConfigInfo scorerConfigInfo) {
-        super(scorerConfigInfo);
-    }
-
-    @Override
-    public void loadModel() {
-        doLoadModel(LRModel.class);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/BaseThompsonSamplingScorer.java

@@ -1,32 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.service.score.model.ThompsonSamplingModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-
-public abstract class BaseThompsonSamplingScorer extends AbstractScorer {
-
-    private static Logger LOGGER = LoggerFactory.getLogger(BaseThompsonSamplingScorer.class);
-
-    public BaseThompsonSamplingScorer(ScorerConfigInfo scorerConfigInfo) {
-        super(scorerConfigInfo);
-    }
-
-    @Override
-    public void loadModel() {
-        doLoadModel(ThompsonSamplingModel.class);
-    }
-    @Override
-    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
-                                           final Map<String, String> userFeatureMap,
-                                           final List<RankItem> rankItems){
-
-        return rankItems;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScoreParam.java

@@ -1,13 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
-import lombok.Data;
-
-/**
- * @author dyp
- */
-@Data
-public class ScoreParam {
-    private String mid;
-    private RequestContext requestContext;
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerConfig.java

@@ -1,140 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigObject;
-import com.typesafe.config.ConfigValue;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-
-public class ScorerConfig {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
-    private final List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
-
-    public ScorerConfig(Config config) {
-        this.load(config);
-    }
-
-    public ScorerConfig(String configFile) {
-        this.load(configFile);
-    }
-
-    public ScorerConfig() {
-    }
-
-    public static Config mergeConfig(Config baselineConfig, List<Config> expConfigs) {
-        if (expConfigs == null || expConfigs.size() == 0) {
-            return baselineConfig;
-        }
-        Config resultConfig = expConfigs.get(0);
-        for (Config config : expConfigs) {
-            resultConfig = resultConfig.withFallback(config);
-        }
-        resultConfig = resultConfig.withFallback(baselineConfig);
-        return resultConfig;
-    }
-
-    public boolean load(String configFile) {
-        Config config = ConfigFactory.parseResources(configFile);
-        return load(config);
-    }
-
-    public boolean load(Config baselineConfig, List<Config> expConfigs) {
-        Config config = mergeConfig(baselineConfig, expConfigs);
-        return load(config);
-    }
-
-    public boolean load(Config config) {
-        Config scorerConfig = config.getConfig("scorer-config");
-
-        try {
-            loadScorers(scorerConfig);
-            int pos = 0;
-            for (ScorerConfigInfo scorerConfigInfo : configInfoList) {
-                LOGGER.debug("scorer at position [{}], priority [{}], scorer name [{}]",
-                        pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName());
-            }
-            LOGGER.debug("Load scorer config success");
-        } catch (Exception e) {
-            LOGGER.error("Load scorer config failed, [{}]", ExceptionUtils.getFullStackTrace(e));
-            return false;
-        }
-
-        return true;
-    }
-
-    public List<ScorerConfigInfo> getConfigInfoList() {
-        return configInfoList;
-    }
-
-
-    public String loadOptionStringConfig(Config config, String path) {
-        return config.hasPath(path) ? config.getString(path) : null;
-    }
-
-    private Config loadOptionConfig(Config config, String path) {
-        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
-    }
-
-    private void loadScorers(Config config) throws Exception {
-
-        ConfigObject confObj = config.root();
-        for (ConfigObject.Entry<String, ConfigValue> it : confObj.entrySet()) {
-            Config conf = ((ConfigObject) it.getValue()).toConfig();
-            // parse config
-            String configName = it.getKey();
-            String scorerName = conf.getString("scorer-name");
-            int scorerPriority = 0;
-            if (conf.hasPath("scorer-priority"))
-                scorerPriority = conf.getInt("scorer-priority");
-            Boolean disableSwitch = false;
-            if (conf.hasPath("disable-switch")) {
-                disableSwitch = conf.getBoolean("disable-switch");
-            }
-            Config paramConfig = loadOptionConfig(conf, "param-config");
-            // model path
-            String modelPath = loadOptionStringConfig(conf, "model-path");
-            if (modelPath == null) {
-                modelPath = loadOptionStringConfig(conf, "default-model-path");
-                LOGGER.debug("model-path is not exists in config file, use default-model-path instead, modelPath={}", modelPath);
-            }
-            // enable queues
-            Set<String> enableQueues = new HashSet<String>();
-            if (conf.hasPath("enable-queues")) {
-                enableQueues.addAll(conf.getStringList("enable-queues"));
-            }
-            ScorerConfigInfo configInfo = new ScorerConfigInfo(configName,
-                    scorerName,
-                    scorerPriority,
-                    disableSwitch,
-                    enableQueues,
-                    modelPath,
-                    paramConfig
-            );
-            LOGGER.debug("parse scorer config info [{}]", configInfo);
-            // add to ConfigInfoList
-            addConfigByPriority(configInfoList, configInfo);
-        }
-    }
-
-    private void addConfigByPriority(List<ScorerConfigInfo> configInfoList, ScorerConfigInfo addConfigInfo) {
-
-        int pos = 0;
-        for (; pos < configInfoList.size(); pos++) {
-            if (configInfoList.get(pos).getScorerPriority() <= addConfigInfo.getScorerPriority()) {
-                break;
-            }
-        }
-
-        configInfoList.add(pos, addConfigInfo);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerConfigInfo.java

@@ -1,78 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.google.gson.Gson;
-import com.typesafe.config.Config;
-
-import java.util.Set;
-
-
-public class ScorerConfigInfo {
-
-    private String configName;
-    private Integer scorerPriority;
-    private Boolean disableSwitch;
-    private String scorerName;
-    private Set<String> enableQueues;
-    private String modelPath;
-    private Config paramConfig; // param config
-
-    public ScorerConfigInfo(String configName,
-                            String scorerName,
-                            Integer scorerPriority,
-                            Boolean disableSwitch,
-                            Set<String> enableQueues,
-                            String modelPath,
-                            Config paramConfig) {
-
-        this.configName = configName;
-        this.scorerName = scorerName;
-        this.scorerPriority = scorerPriority;
-        this.disableSwitch = disableSwitch;
-        this.enableQueues = enableQueues;
-        this.modelPath = modelPath;
-        this.paramConfig = paramConfig;
-    }
-
-    public Config getParamConfig() {
-        return paramConfig;
-    }
-
-    public Set<String> getEnableQueues() {
-        return enableQueues;
-    }
-
-    public void setEnableQueues(Set<String> enableQueues) {
-        this.enableQueues = enableQueues;
-    }
-
-    public Integer getScorerPriority() {
-        return scorerPriority;
-    }
-
-    public String getScorerName() {
-        return scorerName;
-    }
-
-    public String getConfigName() {
-        return configName;
-    }
-
-    public boolean isQueueEnable(String queueName) {
-        return this.enableQueues == null ||
-                this.enableQueues.isEmpty() ||
-                this.enableQueues.contains(queueName);
-    }
-
-    public String getModelPath() {
-        return modelPath;
-    }
-
-    public Boolean getDisableSwitch() {
-        return disableSwitch;
-    }
-
-    @Override
-    public String toString() {
-        return new Gson().toJson(this);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerPipeline.java

@@ -1,187 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.*;
-
-
-@Slf4j
-public class ScorerPipeline {
-    public static final int corePoolSize = 128;
-    public static final int SCORE_TIME_OUT = 400;
-    public static final Logger LOGGER = LoggerFactory.getLogger(ScorerPipeline.class);
-    public static final ExecutorService executorService = Executors.newFixedThreadPool(corePoolSize);
-
-    public List<AbstractScorer> scorers;
-
-    public ScorerPipeline(List<AbstractScorer> scorers) {
-        this.scorers = scorers;
-    }
-
-    /**
-     * scoring
-     *
-     * @return
-     */
-    public List<RankItem> scoring(final ScoreParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
-
-        if (CollectionUtils.isEmpty(scorers)) {
-            log.error("scorers is empty");
-            return rankItems;
-        }
-        List<RankItem> items = rankItems;
-
-        for (final AbstractScorer scorer : scorers) {
-            if (!scorer.isEnable()) {
-                continue;
-            }
-
-            final int beforeSize = rankItems.size();
-            final long startTime = System.currentTimeMillis();
-
-            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
-            String[] scorerNames = fullScorerName.split("\\.");
-            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
-
-            final List<RankItem> scoreRankerItems = items;
-            Callable<List<RankItem>> callable = () -> scorer.scoring(param, userFeature, scoreRankerItems);
-
-            // execute score use thread to protected score worst time
-            List<RankItem> scoredItems = new ArrayList<RankItem>();
-            try {
-                List<Future<List<RankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
-                for (Future<List<RankItem>> future : futures) {
-                    try {
-                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
-                            scoredItems.addAll(future.get());
-                        } else {
-                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
-                                    new Object[]{scorerName, scoreRankerItems.size()});
-                        }
-                    } catch (Exception e) {
-                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
-                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
-                    }
-                }
-            } catch (Exception e) {
-                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
-                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
-            }
-
-            //  变更item
-            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
-                items = scoreRankerItems;
-            } else {
-                items = new ArrayList<>(items);
-            }
-
-            int position = 0;
-            for (RankItem item : items) {
-                item.getRankerIndex().put(scorerName, position++);
-                item.getRankerScore().put(scorerName, item.getScore());
-            }
-
-            //
-            long spentTime = System.currentTimeMillis() - startTime;
-            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
-                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
-        }
-
-        int position = 0;
-        for (RankItem item : items) {
-            item.getRankerIndex().put("finalScore", position++);
-            item.getRankerScore().put("finalScore", item.getScore());
-        }
-
-        return items;
-    }
-
-    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
-                                  final Map<String, String> userFeatureMap,
-                                  final List<RankItem> rankItems) {
-
-        if (CollectionUtils.isEmpty(scorers)) {
-            log.error("scorers is empty");
-            return rankItems;
-        }
-        List<RankItem> items = rankItems;
-
-        for (final AbstractScorer scorer : scorers) {
-            if (!scorer.isEnable()) {
-                continue;
-            }
-
-            final int beforeSize = rankItems.size();
-            final long startTime = System.currentTimeMillis();
-
-            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
-            String[] scorerNames = fullScorerName.split("\\.");
-            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
-
-            final List<RankItem> scoreRankerItems = items;
-            Callable<List<RankItem>> callable = () -> scorer.scoring(sceneFeatureMap, userFeatureMap, scoreRankerItems);
-
-            // execute score use thread to protected score worst time
-            List<RankItem> scoredItems = new ArrayList<RankItem>();
-            try {
-                List<Future<List<RankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
-                for (Future<List<RankItem>> future : futures) {
-                    try {
-                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
-                            scoredItems.addAll(future.get());
-                        } else {
-                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
-                                    new Object[]{scorerName, scoreRankerItems.size()});
-                        }
-                    } catch (Exception e) {
-                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
-                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
-                    }
-                }
-            } catch (Exception e) {
-                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
-                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
-            }
-
-            //  变更item
-            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
-                items = scoreRankerItems;
-            } else {
-                items = new ArrayList<>(items);
-            }
-
-            int position = 0;
-            for (RankItem item : items) {
-                item.getRankerIndex().put(scorerName, position++);
-                if(scorerName.equals("finalScore")) {
-                    item.getRankerScore().put(scorerName, item.getScore());
-                }
-            }
-
-            //
-            long spentTime = System.currentTimeMillis() - startTime;
-            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
-                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
-        }
-
-        int position = 0;
-        for (RankItem item : items) {
-            item.getRankerIndex().put("finalScore", position++);
-            item.getRankerScore().put("finalScore", item.getScore());
-        }
-
-        return items;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerUtils.java

@@ -1,151 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.score;
-
-
-import com.typesafe.config.Config;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-
-@Slf4j
-public final class ScorerUtils {
-    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
-
-    private static final Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
-
-    public static String BASE_CONF_FEED = "feeds_score_config_new_baseline.conf";
-
-    public static String FLOWPOOL_CONF = "feeds_score_config_thompson_new.conf";
-
-
-    public static void warmUp() {
-        log.info("scorer warm up ");
-        ScorerUtils.init(BASE_CONF_FEED);
-        ScorerUtils.init(FLOWPOOL_CONF);
-    }
-
-    private ScorerUtils() {
-        // init(BASE_CONF);
-    }
-
-    public static void init(String configFile) {
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(configFile);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-
-        initLoadModel(scorers);
-        scorerPipelineCache.put(configFile, new ScorerPipeline(scorers));
-    }
-
-    /**
-     * init load model
-     *
-     * @param scorers
-     */
-    public static void initLoadModel(List<AbstractScorer> scorers) {
-        for (AbstractScorer scorer : scorers) {
-            if (scorer.isEnable()) {
-                scorer.loadModel();
-            }
-        }
-    }
-
-    public static void initLoadModel(String configFile) {
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(configFile);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-        initLoadModel(scorers);
-    }
-
-
-    public static void initLoadModel(Config config) {
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(config);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-        initLoadModel(scorers);
-    }
-
-    public static ScorerPipeline getScorerPipeline(String configFile) {
-        // 不需要保证严格意义的单例
-        if (scorerPipelineCache.containsKey(configFile)) {
-            return scorerPipelineCache.get(configFile);
-        }
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(configFile);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-        ScorerPipeline pipeline = new ScorerPipeline(scorers);
-        scorerPipelineCache.put(configFile, pipeline);
-        return pipeline;
-    }
-
-    public static ScorerPipeline getScorerPipeline(Config mergeConfig) {
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(mergeConfig);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-        return new ScorerPipeline(scorers);
-    }
-
-    public static ScorerPipeline getScorerPipeline(Config baselineConfig, List<Config> configList) {
-        ScorerConfig scorerConfig = new ScorerConfig();
-        scorerConfig.load(baselineConfig, configList);
-        List<AbstractScorer> scorers = ScorerUtils.constructScorers(scorerConfig);
-        return new ScorerPipeline(scorers);
-    }
-
-    /**
-     * construct scorers
-     *
-     * @param scorerConfig
-     * @return
-     */
-    public static List<AbstractScorer> constructScorers(ScorerConfig scorerConfig) {
-
-        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
-        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
-            if (!configInfo.getDisableSwitch()) {
-                try {
-                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
-                            .getConstructor(ScorerConfigInfo.class)
-                            .newInstance(configInfo);
-                    scorers.add(scorer);
-                    LOGGER.debug("construct score [{}]", configInfo.getScorerName());
-                } catch (Exception e) {
-                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
-                }
-
-            }
-        }
-        return scorers;
-    }
-
-    /**
-     * construct scorers
-     *
-     * @param scorerConfig
-     * @return
-     */
-    public static List<AbstractScorer> constructQueueScorers(ScorerConfig scorerConfig, String queueName) {
-
-        List<AbstractScorer> scorers = new ArrayList<AbstractScorer>();
-        for (ScorerConfigInfo configInfo : scorerConfig.getConfigInfoList()) {
-            if (!configInfo.getDisableSwitch() && configInfo.isQueueEnable(queueName)) {
-                try {
-                    AbstractScorer scorer = (AbstractScorer) Class.forName(configInfo.getScorerName())
-                            .getConstructor(ScorerConfigInfo.class)
-                            .newInstance(configInfo);
-                    scorers.add(scorer);
-                    LOGGER.debug("construct queue scorer [{}] [{}]", queueName, configInfo.getScorerName());
-                } catch (Exception e) {
-                    LOGGER.error("instance scorer {} failed {}", configInfo.getScorerName(), ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        return scorers;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/AbstractUserAttentionExtractor.java

@@ -1,28 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.userattention;
-
-
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-public abstract class AbstractUserAttentionExtractor {
-    protected UserAttentionExtractorConfig extractorConfig;
-
-    public AbstractUserAttentionExtractor(UserAttentionExtractorConfig extractorConfig) {
-        this.extractorConfig = extractorConfig;
-    }
-
-    public String getExtractorName() {
-        String[] arr = this.extractorConfig.getName().split(".");
-        return arr.length > 0 ? arr[arr.length - 1] : "UnKnownExtractor";
-    }
-
-    public UserAttentionExtractorConfig getExtractorConfig() {
-        return extractorConfig;
-    }
-
-
-    public abstract void extractAttention(final RecommendRequest requestData, final User user);
-
-    public void loadModel() {
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorConfig.java

@@ -1,44 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.userattention;
-
-import com.typesafe.config.Config;
-
-
-
-public class UserAttentionExtractorConfig {
-    private final String name;  // attention extractor name
-    private final Integer priority;
-    private final String modelPath;
-    private final Config paramConfig; // param config
-
-    public UserAttentionExtractorConfig(String name,
-                                        Config paramConfig,
-                                        Integer priority,
-                                        String modelPath) {
-        this.name = name;
-        this.paramConfig = paramConfig;
-        this.priority = priority;
-        this.modelPath = modelPath;
-    }
-
-    public Integer getPriority() {
-        return priority;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    public Config getParamConfig() {
-        return paramConfig;
-    }
-
-    @Override
-    public String toString() {
-        return "AttentionExtractorConfig{" +
-                "name='" + name + '\'' +
-                ", priority=" + priority +
-                ", modelPath='" + modelPath + '\'' +
-                ", paramConfig=" + paramConfig +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorPipeline.java

@@ -1,36 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.userattention;
-
-
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.common.UserAttention;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class UserAttentionExtractorPipeline {
-    private List<AbstractUserAttentionExtractor> extractorList = new ArrayList<AbstractUserAttentionExtractor>();
-
-    public UserAttentionExtractorPipeline(List<AbstractUserAttentionExtractor> extractorList) {
-        this.extractorList = extractorList;
-    }
-
-    public void extractAttention(final RecommendRequest requestData, final User user) {
-        if (user.getUserAttention() == null) {
-            UserAttention userAttention = new UserAttention();
-            user.setUserAttention(userAttention);
-        }
-
-        long startTime = System.currentTimeMillis();
-
-        for (AbstractUserAttentionExtractor extractor : extractorList) {
-            try {
-                long currStartTime = System.currentTimeMillis();
-                extractor.extractAttention(requestData, user);
-                // log
-            } catch (Exception e) {
-                // log
-            }
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorUtils.java

@@ -1,134 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.userattention;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigObject;
-import com.typesafe.config.ConfigValue;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-
-public class UserAttentionExtractorUtils {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(UserAttentionExtractorUtils.class);
-
-    public static final String BASE_CONF = "";
-
-    /**
-     * 主调用函数
-     *
-     * @param config
-     */
-    public static void initLoadModel(Config config) {
-        initLoadModel(constructExtractorList(constructExtractorConfigList(config)));
-    }
-
-
-    public static UserAttentionExtractorPipeline getAtttentionPipeline(String configFile) {
-
-        Config attentionConfig = ConfigFactory.parseResources(configFile);
-        List<AbstractUserAttentionExtractor> attentionExtractors = constructExtractorList(constructExtractorConfigList(attentionConfig));
-        UserAttentionExtractorPipeline pipeline = new UserAttentionExtractorPipeline(attentionExtractors);
-
-        return pipeline;
-    }
-
-
-    public static List<UserAttentionExtractorConfig> constructExtractorConfigList(Config config) {
-        long startTime = System.currentTimeMillis();
-        List<UserAttentionExtractorConfig> configList = new ArrayList<UserAttentionExtractorConfig>();
-        try {
-            Config mergedConfig = config.getConfig("attention-extractor-config");
-            for (ConfigObject.Entry<String, ConfigValue> it : mergedConfig.root().entrySet()) {
-                LOGGER.debug("deal with base config, attention key: [{}]", it.getKey());
-                Config entryConf = ((ConfigObject) it.getValue()).toConfig();
-                UserAttentionExtractorConfig extractorConfig = parseExtractorConfig(entryConf);
-                addConfigByPriority(configList, extractorConfig);
-            }
-            // debug log
-            if (LOGGER.isDebugEnabled()) {
-                for (UserAttentionExtractorConfig extractorConfig : configList) {
-                    LOGGER.debug("attention extractor config [{}] [{}]", extractorConfig.getName(), extractorConfig);
-                }
-            }
-        } catch (Exception e) {
-            LOGGER.error("construct extractor config {} failed {}", config.toString(), ExceptionUtils.getFullStackTrace(e));
-        }
-        return configList;
-    }
-
-
-    private static List<AbstractUserAttentionExtractor> constructExtractorList(List<UserAttentionExtractorConfig> configList) {
-        long startTime = System.currentTimeMillis();
-        List<AbstractUserAttentionExtractor> extractorList = new ArrayList<AbstractUserAttentionExtractor>();
-        for (UserAttentionExtractorConfig config : configList) {
-            try {
-                AbstractUserAttentionExtractor extractor = (AbstractUserAttentionExtractor) Class.forName(config.getName())
-                        .getConstructor(UserAttentionExtractorConfig.class)
-                        .newInstance(config);
-                extractorList.add(extractor);
-            } catch (Exception e) {
-                LOGGER.error("instance extractor {} failed {}", config.getName(), ExceptionUtils.getFullStackTrace(e));
-            }
-        }
-        return extractorList;
-    }
-
-    private static String loadOptionStringConfig(Config config, String path) {
-        return config.hasPath(path) ? config.getString(path) : null;
-    }
-
-    private static Config loadOptionConfig(Config config, String path) {
-        return config.hasPath(path) ? config.getConfig(path) : ConfigFactory.empty();
-    }
-
-    private static int loadOptionIntConfig(Config config, String path) {
-        return config.hasPath(path) ? config.getInt(path) : 0;
-    }
-
-    /**
-     * attention层的配置化参数
-     */
-    private static UserAttentionExtractorConfig parseExtractorConfig(Config extractorConfig) {
-        String attentionExtractorName = extractorConfig.getString("name");
-        Integer priority = loadOptionIntConfig(extractorConfig, "attention-priority");
-        Config paramConfig = loadOptionConfig(extractorConfig, "param-config");
-        String modelPath = loadOptionStringConfig(extractorConfig, "model-path");
-        return new UserAttentionExtractorConfig(attentionExtractorName,
-                paramConfig,
-                priority,
-                modelPath);
-    }
-
-
-    /**
-     * attention支持模型获取
-     *
-     * @param attentionExtractors
-     */
-    private static void initLoadModel(List<AbstractUserAttentionExtractor> attentionExtractors) {
-        for (AbstractUserAttentionExtractor attentionExtractor : attentionExtractors) {
-            attentionExtractor.loadModel();
-        }
-    }
-
-    /**
-     * 按优先级处理逻辑
-     *
-     * @param configInfoList
-     * @param extractorConfig
-     */
-    private static void addConfigByPriority(List<UserAttentionExtractorConfig> configInfoList, UserAttentionExtractorConfig extractorConfig) {
-        int pos = 0;
-        for (; pos < configInfoList.size(); pos++) {
-            if (configInfoList.get(pos).getPriority() <= extractorConfig.getPriority()) {
-                break;
-            }
-        }
-        configInfoList.add(pos, extractorConfig);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/FixedThreadPoolHelper.java

@@ -1,11 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.utils;
-
-public class FixedThreadPoolHelper extends ThreadPoolHelper {
-    public FixedThreadPoolHelper(int nThread) {
-        super(ThreadPoolUtils.newFixedThreadPool(nThread));
-    }
-
-    public FixedThreadPoolHelper(int nThread, String name) {
-        super(ThreadPoolUtils.newFixedThreadPool(nThread, name), "FixedPool_" + name);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/RedisSmartClient.java

@@ -1,64 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.utils;
-
-
-import com.google.common.base.Function;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.stereotype.Service;
-import redis.clients.jedis.JedisCluster;
-import redis.clients.jedis.Tuple;
-
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-@Service
-public class RedisSmartClient {
-
-    private final static ExecutorService executorService = Executors.newCachedThreadPool();
-    @Qualifier("redisTemplate")
-    @Autowired
-    protected RedisTemplate<String, String> redisTemplate;
-
-
-    public RedisSmartClient() {
-    }
-    public void set(final String key, final String value) {
-        redisTemplate.opsForValue().set(key, value);
-    }
-
-    public String get(final String key) {
-        return redisTemplate.opsForValue().get(key);
-    }
-
-    // hget
-    public String hget(final String key, final String field) {
-        return redisTemplate.opsForHash().get(key, field).toString();
-    }
-
-
-    // getRandomValueFromSet
-    public List<String> getRandomList(final String key, long num) {
-        return redisTemplate.opsForSet().randomMembers(key, num);
-    }
-
-
-    // hgetall
-    public Map<Object, Object> hgetall(String key) {
-        return redisTemplate.opsForHash().entries(key);
-    }
-
-    // TODO 异步写入redis 作为用户的分发历史
-    public <T> T write(String key, String value) {
-        T t = null;
-        return t;
-    }
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/ThreadPoolHelper.java

@@ -1,64 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.utils;
-
-
-import lombok.Data;
-
-import java.util.Timer;
-import java.util.concurrent.ThreadPoolExecutor;
-
-/**
- * 线程池Helper
- *   可以命名线程池; 保存ThreadPoolExecutor实例, 方便监控线程池信息
- */
-@Data
-public class ThreadPoolHelper {
-    private String name;
-    private ThreadPoolExecutor threadPoolExecutor;
-    private Timer counterTimer;
-
-    ThreadPoolHelper(ThreadPoolExecutor threadPoolExecutor) {
-        this.threadPoolExecutor = threadPoolExecutor;
-        this.name = "";
-    }
-
-    ThreadPoolHelper(ThreadPoolExecutor threadPoolExecutor, String name) {
-        this.threadPoolExecutor = threadPoolExecutor;
-        this.name = name;
-    }
-
-
-    public long getTaskCount() {
-        return this.threadPoolExecutor.getTaskCount();
-    }
-
-    public long getActiveCount() {
-        return this.threadPoolExecutor.getActiveCount();
-    }
-
-    public long getCorePoolSize() {
-        return this.threadPoolExecutor.getCorePoolSize();
-    }
-
-    public long getPoolSize() {
-        return this.threadPoolExecutor.getPoolSize();
-    }
-
-    public long getMaximumPoolSize() {
-        return this.threadPoolExecutor.getMaximumPoolSize();
-    }
-
-    public long getLargestPoolSize() {
-        return this.threadPoolExecutor.getLargestPoolSize();
-    }
-
-    public long getTaskQueueSize() {
-        return this.threadPoolExecutor.getQueue().size();
-    }
-
-    public void cancelCounter() {
-        if (counterTimer != null) {
-            counterTimer.cancel();
-            counterTimer.purge();
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/utils/ThreadPoolUtils.java

@@ -1,46 +0,0 @@
-package com.tzld.piaoquan.recommend.server.framework.utils;
-
-
-import java.util.concurrent.*;
-
-/**
- * 命名线程池工场类
- * 最后创建的线程名称:
- *   name_{counter}
- */
-class NamedThreadFactory implements ThreadFactory {
-    private int counter = 0;
-    private String name;
-    NamedThreadFactory(String name) {
-        this.name = name;
-    }
-
-    @Override
-    public Thread newThread(Runnable r) {
-        counter += 1;
-        return new Thread(r, name + "_" + counter);
-    }
-}
-
-
-public class ThreadPoolUtils {
-    public static ThreadPoolExecutor newFixedThreadPool(int nThreads, String poolName) {
-        return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
-                new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("Fixed_" + poolName));
-    }
-    public static ThreadPoolExecutor newFixedThreadPool(int nThreads) {
-        return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
-                new LinkedBlockingQueue<Runnable>());
-    }
-
-    public static ThreadPoolExecutor newCachedThreadPool(String poolName) {
-        return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
-                new SynchronousQueue<Runnable>(), new NamedThreadFactory("Cached_" + poolName));
-    }
-
-    public static ThreadPoolExecutor newCachedThreadPool() {
-        return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
-                new SynchronousQueue<Runnable>());
-    }
-}
-

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/grpcservice/RecommendGrpcService.java

@@ -3,7 +3,7 @@ package com.tzld.piaoquan.recommend.server.grpcservice;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendResponse;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendServiceGrpc;
-import com.tzld.piaoquan.recommend.server.service.VideoRecommendService;
+import com.tzld.piaoquan.recommend.server.service.RecommendService;
 import io.grpc.stub.StreamObserver;
 import net.devh.boot.grpc.server.service.GrpcService;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -14,18 +14,18 @@ import org.springframework.beans.factory.annotation.Autowired;
 @GrpcService
 public class RecommendGrpcService extends RecommendServiceGrpc.RecommendServiceImplBase {
     @Autowired
-    private VideoRecommendService videoRecommendService;
+    private RecommendService recommendService;
 
     @Override
     public void homepageRecommend(RecommendRequest request, StreamObserver<RecommendResponse> responseObserver) {
-        RecommendResponse response = videoRecommendService.homepageRecommend(request);
+        RecommendResponse response = recommendService.homepageRecommend(request);
         responseObserver.onNext(response);
         responseObserver.onCompleted();
     }
 
     @Override
     public void relevantRecommend(RecommendRequest request, StreamObserver<RecommendResponse> responseObserver) {
-        RecommendResponse response = videoRecommendService.relevantRecommend(request);
+        RecommendResponse response = recommendService.relevantRecommend(request);
         responseObserver.onNext(response);
         responseObserver.onCompleted();
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/FlowPoolRecommendPipeline.java

@@ -1,211 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement;
-
-
-import com.alibaba.fastjson.JSONObject;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.MachineInfo;
-import com.tzld.piaoquan.recommend.server.model.RecommendParam;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.RecommendService;
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.FlowPoolWithLevelRecallStrategyTomson;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.FlowPoolWithLevelScoreRecallStrategy;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.FlowPoolWithScoreRecallStrategy;
-import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import org.apache.commons.collections4.CollectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeansException;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.PostConstruct;
-import javax.annotation.Resource;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-@Service
-public class FlowPoolRecommendPipeline implements ApplicationContextAware {
-
-    private final Logger log = LoggerFactory.getLogger(FlowPoolRecommendPipeline.class);
-
-    private final static Map<String, RecallStrategy> strategyMap = new HashMap<>();
-
-    @Resource
-    private RecommendService recommendService;
-
-    private ApplicationContext applicationContext;
-
-    private final ExecutorService pool = ThreadPoolFactory.recallPool();
-
-    @PostConstruct
-    public void init() {
-        Map<String, RecallStrategy> type = applicationContext.getBeansOfType(RecallStrategy.class);
-        for (Map.Entry<String, RecallStrategy> entry : type.entrySet()) {
-            RecallStrategy value = entry.getValue();
-            strategyMap.put(value.getClass().getSimpleName(), value);
-        }
-    }
-
-
-    public List<Video> feedByRec(final RecommendRequest requestData,
-                                 final int recommendType, Boolean logPrint) {
-        List<RecallStrategy> strategies = new ArrayList<>();
-
-        RecommendParam param = recommendService.genRecommendParam(requestData, recommendType);
-        RecallParam recallParam = recommendService.convertToRecallParam(param);
-        RecallResult recallResult = getRecallResult(strategies, recallParam);
-        if (logPrint) {
-            log.info("traceId = {}, recallResult [{}]", requestData.getRequestId(),
-                    JSONObject.toJSONString(recallResult));
-        }
-
-        if (recallResult == null || CollectionUtils.isEmpty(recallResult.getData())) {
-            return Collections.emptyList();
-        }
-
-        RankParam rankParam = recommendService.convertToRankParam(param, recallResult);
-        List<Video> videoList = mergeAndRankFlowPoolRecall(rankParam);
-        if (logPrint) {
-            log.info("traceId = {}, videoList [{}]", requestData.getRequestId(),
-                    JSONObject.toJSONString(videoList));
-        }
-        return videoList;
-    }
-
-    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
-        return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
-    }
-
-    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
-
-        //初始化 userid
-        UserFeature userFeature = new UserFeature();
-        userFeature.setMid(param.getMid());
-
-        // 初始化RankItem
-        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
-                .filter(d -> d.getPushFrom().equals(pushFrom))
-                .findFirst();
-        if (!data.isPresent()) {
-            return Collections.emptyList();
-        }
-        List<Video> videoList = data.get().getVideos();
-        if (videoList == null) {
-            return Collections.emptyList();
-        }
-        List<RankItem> rankItems = new ArrayList<>();
-        for (int i = 0; i < videoList.size(); i++) {
-            RankItem rankItem = new RankItem(videoList.get(i));
-            rankItems.add(rankItem);
-        }
-
-        // 初始化上下文参数
-        ScoreParam scoreParam = convert(param);
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
-                .scoring(scoreParam, userFeature, rankItems);
-
-        if (rovRecallScore == null) {
-            return Collections.emptyList();
-        }
-
-        return CommonCollectionUtils.toList(rovRecallScore, i -> {
-            // hard code 将排序分数 赋值给video的sortScore
-            Video v = i.getVideo();
-            v.setSortScore(i.getScore());
-            return v;
-        });
-    }
-
-    protected ScoreParam convert(RankParam param) {
-        ScoreParam scoreParam = new ScoreParam();
-
-        scoreParam.setMid(param.getMid());
-
-        // TODO hardcode 为了兼容写入逻辑
-        RequestContext context = new RequestContext();
-        context.setApptype(param.getAppType() + "");
-
-        // TODO 地域转换
-        context.setRegion(param.getProvince());
-        context.setCity(param.getCity());
-
-        Calendar calendar = Calendar.getInstance();
-        context.setWeek((calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        context.setDay(new SimpleDateFormat("yyyyMMdd").format(calendar.getTime()));
-        context.setHour(new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        MachineInfo machineInfo = param.getMachineInfo();
-        if (machineInfo != null) {
-            context.setMachineinfo_brand(machineInfo.getBrand());
-            context.setMachineinfo_model(machineInfo.getModel());
-            context.setMachineinfo_platform(machineInfo.getPlatform());
-            context.setMachineinfo_sdkversion(machineInfo.getSdkVersion());
-            context.setMachineinfo_system(machineInfo.getSystem());
-            context.setMachineinfo_wechatversion(machineInfo.getWechatVersion());
-        }
-
-        scoreParam.setRequestContext(context);
-        return scoreParam;
-    }
-
-    private RecallResult getRecallResult(List<RecallStrategy> strategies, RecallParam param) {
-        if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL)) {
-            strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategyTomson.class.getSimpleName()));
-        } else if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL_SCORE)) {
-            strategies.add(strategyMap.get(FlowPoolWithLevelScoreRecallStrategy.class.getSimpleName()));
-        } else {
-            strategies.add(strategyMap.get(FlowPoolWithScoreRecallStrategy.class.getSimpleName()));
-        }
-
-        CountDownLatch cdl = new CountDownLatch(strategies.size());
-        List<Future<RecallResult.RecallData>> recallResultFutures = new ArrayList<>();
-        for (final RecallStrategy strategy : strategies) {
-            Future<RecallResult.RecallData> future = pool.submit(() -> {
-                List<Video> result = strategy.recall(param);
-                cdl.countDown();
-                return new RecallResult.RecallData(strategy.pushFrom(), result);
-            });
-            recallResultFutures.add(future);
-        }
-        try {
-            cdl.await(3000, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            log.error("recall error", e);
-            return null;
-        }
-
-        List<RecallResult.RecallData> results = new ArrayList<>();
-        for (Future<RecallResult.RecallData> f : recallResultFutures) {
-            try {
-                RecallResult.RecallData data = f.get();
-                results.add(data);
-            } catch (Exception e) {
-                log.error("future get error ", e);
-            }
-        }
-
-        return new RecallResult(results);
-    }
-
-
-    @Override
-    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
-        this.applicationContext = applicationContext;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/TopRecommendPipeline.java

@@ -1,627 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement;
-
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.base.Stopwatch;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.CandidateInfo;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.MergeUtils;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueue;
-import com.tzld.piaoquan.recommend.server.framework.recaller.BaseRecaller;
-import com.tzld.piaoquan.recommend.server.framework.recaller.provider.RedisBackedQueue;
-import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import org.apache.commons.collections4.CollectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.PostConstruct;
-import javax.annotation.Resource;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-@Service
-public class TopRecommendPipeline {
-
-    private static final Logger log = LoggerFactory.getLogger(TopRecommendPipeline.class);
-
-    public static final String MERGE_CONF = "merge_config.conf";
-
-    @Value("${recommend.recall.num:500}")
-    private int recallNum;
-    @ApolloJsonValue("${rank.score.merge.weightv547:}")
-    private Map<String, Double> mergeWeight;
-    @Resource
-    private RedisSmartClient client;
-    @Resource
-    private RedisTemplate<String, String> redisTemplate;
-
-    @Qualifier("featureRedisTemplate")
-    @Autowired
-    private RedisTemplate<String, String> featureRedisTemplate;
-
-    private RedisBackedQueue queueProvider;
-
-    @PostConstruct
-    public void init() {
-        queueProvider = new RedisBackedQueue(client, 15 * 60 * 1000L);
-        mergeWeight = mergeWeight == null ? new HashMap<>() : mergeWeight;
-    }
-
-    public List<Video> feeds(final RecommendRequest requestData,
-                             final int requestIndex,
-                             final User userInfo, Boolean logPrint,
-                             Map<String, String> timeLogMap) {
-        // Step 1: Attention extraction
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        stopwatch.reset().start();
-        timeLogMap.put("uid", userInfo.getUid());
-        timeLogMap.put("mid", userInfo.getMid());
-        timeLogMap.put("requestId", requestData.getRequestId());
-
-        List<RankItem> rankItems = feedByRec(requestData, requestIndex, userInfo, logPrint, timeLogMap);
-        timeLogMap.put("feedByRec", stopwatch.elapsed().toMillis() + "");
-        if (rankItems == null || rankItems.isEmpty()) {
-            return new ArrayList<>();
-        }
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, feeds rankItems = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(rankItems));
-        }
-        stopwatch.reset().start();
-        List<Video> videos = rankItem2Video(rankItems);
-        timeLogMap.put("rankItem2Video", stopwatch.elapsed().toMillis() + "");
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, videos = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(videos));
-        }
-        return videos;
-    }
-
-    private List<Video> rankItem2Video(List<RankItem> rankItems) {
-        List<Video> videos = new ArrayList<>();
-        for (RankItem item : rankItems) {
-            Video video = new Video();
-            video.setVideoId(Long.parseLong(item.getId()));
-            video.setPushFrom(item.getQueue());
-            video.setScore(item.getScore());
-            video.setSortScore(item.getScore());
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-
-            Map<String, List<String>> pushFromIndex = new HashMap<>();
-            pushFromIndex.put(item.getQueue(), item.getCandidateInfoList().stream()
-                    .map(CandidateInfo::getCandidateQueueName).collect(Collectors.toList()));
-            video.setPushFromIndex(pushFromIndex);
-            videos.add(video);
-        }
-        videos.sort(Comparator.comparing(Video::getScore).reversed());
-        return videos;
-    }
-
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
-
-    public Double calScoreWeight(List<Double> data){
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i=0; i<data.size(); ++i){
-            up += 1.0 / (i + 1) * data.get(i);
-            down += 1.0 / (i + 1);
-        }
-        return down > 1E-8? up / down: 0.0;
-    }
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
-        List<Double> data = new LinkedList<>();
-        for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
-        }
-        return data;
-    }
-
-    public List<RankItem> feedByRec(final RecommendRequest requestData,
-                                    final int requestIndex,
-                                    final User userInfo, Boolean logPrint,
-                                    Map<String, String> timeLogMap) {
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        // Step 2: create top queue
-        stopwatch.reset().start();
-        StrategyQueue topQueue = MergeUtils.createTopQueue(MERGE_CONF, "top-queue");
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, topQueue = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(topQueue));
-        }
-        timeLogMap.put("createTopQueue", stopwatch.elapsed().toMillis() + "");
-
-        // Step 3: Candidate
-        stopwatch.reset().start();
-        Map<String, Candidate> candidates = new HashMap<String, Candidate>();
-        topQueue.candidate(candidates, recallNum, userInfo, requestData, 0, 0);
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, candidates = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(candidates));
-        }
-        timeLogMap.put("topQueue-candidate-cost", stopwatch.elapsed().toMillis() + "");
-
-
-        // Step 4: Recalling & Basic Scoring
-        stopwatch.reset().start();
-        BaseRecaller recaller = new BaseRecaller(queueProvider);
-        List<RankItem> items = recaller.recalling(requestData, userInfo, new ArrayList<>(candidates.values()));
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, items = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(items));
-        }
-        timeLogMap.put("recalling-cost", stopwatch.elapsed().toMillis() + "");
-        timeLogMap.put("recalling-size", items == null ? "0" : items.size() + "");
-
-        if (CollectionUtils.isEmpty(items)) {
-            return new ArrayList<>();
-        }
-
-        // Step 4: Advance Scoring
-        stopwatch.reset().start();
-        List<RankItem> rankItemList = videoScoredByFeature(items, requestData);
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, rankItemList = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(rankItemList));
-        }
-
-        stopwatch.reset().start();
-        // Step 5: Merger
-        MergeUtils.distributeItemsToMultiQueues(topQueue, rankItemList);
-        topQueue.merge(recallNum * 3, userInfo, requestData, requestIndex, 0);
-
-        // 多样性融合
-        List<RankItem> mergeItems = topQueue.getItems();
-        if (CollectionUtils.isEmpty(mergeItems)) {
-            return new ArrayList<>();
-        }
-        duplicate(mergeItems);
-
-        timeLogMap.put("mergeItems-cost", stopwatch.elapsed().toMillis() + "");
-        timeLogMap.put("mergeItems-size", mergeItems.size() + "");
-
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, mergeItems = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(mergeItems));
-        }
-//        MergeUtils.diversityRerank(mergeItems, SimilarityUtils.getIsSameUserTagOrCategoryFunc(), recallNum, 6, 2);
-
-        // Step 6: Global Rank & subList
-        return mergeItems;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    private List<RankItem> videoScoredByFeature(List<RankItem> items, RecommendRequest recommendRequest) {
-        // 1 模型分
-        List<RankItem> rankItemList = model(items, recommendRequest);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分 3H
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 3; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : rankItemList) {
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // 平台回流ROV
-            List<Double> view2PreReturns = getRateData(preturns, views, 0.0, 0.0);
-            Double view2PreReturnsScore = calScoreWeightNoTimeDecay(view2PreReturns);
-            item.scoresMap.put("view2PreReturnsScore", view2PreReturnsScore);
-
-            // 平台回流ROS
-            List<Double> share2PreReturns = getRateData(preturns, shares, 1.0, 10.0);
-            Double share2PreReturnsScore = calScoreWeightNoTimeDecay(share2PreReturns);
-            item.scoresMap.put("share2PreReturnsScore", share2PreReturnsScore);
-
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-//        double a = mergeWeight.getOrDefault("a", 0.1);
-//        double b = mergeWeight.getOrDefault("b", 0.0);
-//        double c = mergeWeight.getOrDefault("c", 0.000001);
-//        double d = mergeWeight.getOrDefault("d", 1.0);
-//        double e = mergeWeight.getOrDefault("e", 1.0);
-//        double f = mergeWeight.getOrDefault("f", 0.8);
-//        double g = mergeWeight.getOrDefault("g", 2.0);
-//        double h = mergeWeight.getOrDefault("h", 240.0);
-//        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : rankItemList) {
-//            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-//                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-//            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-//                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-//            double strScore = item.getScoreStr();
-//            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-//            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-//            Double view2PreReturnsScore = item.scoresMap.getOrDefault("view2PreReturnsScore", 0.0);
-//            Double share2PreReturnsScore = item.scoresMap.getOrDefault("share2PreReturnsScore", 0.0);
-            // if NaN set 0
-            if (Double.isNaN(share2allreturnScore)) {
-                share2allreturnScore = 0.0;
-            }
-            if (Double.isNaN(view2allreturnScore)) {
-                view2allreturnScore = 0.0;
-            }
-            double score = share2allreturnScore + view2allreturnScore;
-//            if (ifAdd < 0.5) {
-//                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-//                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-//            } else {
-//                score = a * strScore + b * rosScore + c * preturnsScore +
-//                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-//
-//            }
-//            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-//            if (allreturnsScore > h) {
-//                score += (f * share2allreturnScore + g * view2allreturnScore);
-//            }
-            // 设置计算好的分数
-            item.setScore(score);
-        }
-        return rankItemList;
-    }
-
-    private List<RankItem> model(List<RankItem> items, RecommendRequest param) {
-        if (items.isEmpty()) {
-            return items;
-        }
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = featureRedisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.getClass().getSimpleName(), e));
-                }
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<String> videoIds = CommonCollectionUtils.toListDistinct(items, RankItem::getId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = featureRedisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    items.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    items.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.getClass().getSimpleName(), e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : items) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}",
-                            this.getClass().getSimpleName(), e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item : items) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}",
-                            this.getClass().getSimpleName(), e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, items);
-        return rovRecallScore;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    private void duplicate(List<RankItem> items) {
-        Set<String> ids = new HashSet<>();
-        List<RankItem> result = new ArrayList<>();
-        for (RankItem item : items) {
-            if (ids.contains(item.getId())) {
-                continue;
-            }
-            ids.add(item.getId());
-            result.add(item);
-        }
-        items.clear();
-        items.addAll(result);
-    }
-
-    private Map<String, String> getSceneFeature(RecommendRequest param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ){
-        }else{
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/attention/SimpleAttentionExtractor.java

@@ -1,20 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.attention;
-
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.userattention.AbstractUserAttentionExtractor;
-import com.tzld.piaoquan.recommend.server.framework.userattention.UserAttentionExtractorConfig;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-public class SimpleAttentionExtractor extends AbstractUserAttentionExtractor {
-
-    public SimpleAttentionExtractor(UserAttentionExtractorConfig config) {
-        super(config);
-    }
-
-    @Override
-    public void extractAttention(RecommendRequest recommendRequestData, User user) {
-        //TODO
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global1hHotCandidate.java

@@ -1,71 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Global1hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Global1hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
-
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-
-
-        QueueName queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "global1h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "global1h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_rov")
-                .addMatch("type", "global1h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "global1h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "combination")
-                .addMatch("type", "global1h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum, getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-
-    }
-
-    @Override
-    public String toString() {
-        return "Global24hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global24hHotCandidate.java

@@ -1,64 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Global24hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Global24hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<>();
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-
-        QueueName queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "combination")
-                .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-
-    }
-
-    @Override
-    public String toString() {
-        return "Global24hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global3hHotCandidate.java

@@ -1,66 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Global3hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Global3hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
-
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-
-
-        QueueName queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "global3h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_rov")
-                .addMatch("type", "global3h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "global3h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum, getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "global3h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-
-    }
-
-    @Override
-    public String toString() {
-        return "Global24hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/HotCandidateQueue.java

@@ -1,39 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.util.Map;
-
-/**
- * 对应 SimHotVideo 和 ReturnVideo
- *
- * @author sunxy
- */
-public class HotCandidateQueue extends IndexCandidateQueue {
-
-    public HotCandidateQueue(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user,
-                               RecommendRequest requestData, int requestIndex, int expId) {
-
-        return 0;
-    }
-
-    @Override
-    public String toString() {
-        return "HotCandidateQueue{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region1hHotCandidate.java

@@ -1,81 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Region1hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Region1hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        String region = user.getRegion();
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-
-
-        QueueName queueName = new QueueName(ItemType, "short_rov")
-                .addMatch("type", "region1h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum,
-                getStrategyQueueInfo().getQueueName());
-
-
-        queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "region1h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum,
-                getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "region1h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
-                getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "combination")
-                .addMatch("type", "region1h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum,
-                getStrategyQueueInfo().getQueueName());
-
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "region1h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
-                getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-    }
-
-    @Override
-    public String toString() {
-        return "Region1hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region24hHotCandidate.java

@@ -1,77 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Region24hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Region24hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        String region = user.getRegion();
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
-
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
-        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-
-        // index key  video:queue:type=region24h:region=北京:ordering=rov
-        QueueName queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "region24h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
-
-        // index key  video:queue:type=region24h:region=北京:ordering=str
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "region24h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
-                getStrategyQueueInfo().getQueueName());
-
-        // index key  video:queue:type=region24h:region=北京:ordering=combination
-        queueName = new QueueName(ItemType, "combination")
-                .addMatch("type", "region24h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum,
-                getStrategyQueueInfo().getQueueName());
-
-        // index key  video:queue:type=region24h:region=北京:ordering=str
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "region24h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
-                getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-
-    }
-
-    @Override
-    public String toString() {
-        return "Region24hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region3hHotCandidate.java

@@ -1,77 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.candidate;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
-import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class Region3hHotCandidate extends IndexCandidateQueue {
-
-    public static final String ItemType = "video";
-
-    public Region3hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
-        super(strategyQueueInfo, strategyQueueConfig);
-    }
-
-    @Override
-    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
-        int currRecall = 0;
-        String region = user.getRegion();
-        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<>();
-
-        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-
-
-        // index key video:queue:type=region3h:region=北京:ordering=short_rov
-        QueueName queueName = new QueueName(ItemType, "short_rov")
-                .addMatch("type", "region3h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum,
-                getStrategyQueueInfo().getQueueName());
-
-        // index key video:queue:type=region3h:region=北京:ordering=rov
-        queueName = new QueueName(ItemType, "rov")
-                .addMatch("type", "region3h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum,
-                getStrategyQueueInfo().getQueueName());
-
-        // index key video:queue:type=region3h:region=北京:ordering=str
-        queueName = new QueueName(ItemType, "short_ros")
-                .addMatch("type", "region3h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
-                getStrategyQueueInfo().getQueueName());
-
-        // index key video:queue:type=region3h:region=北京:ordering=str
-        queueName = new QueueName(ItemType, "str")
-                .addMatch("type", "region3h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
-                getStrategyQueueInfo().getQueueName());
-
-        candidates.putAll(simplifiedCandidates);
-        return currRecall;
-    }
-
-    @Override
-    public String toString() {
-        return "Region3hHotCandidate{" +
-                "myCandidates=" + myCandidates +
-                ", rankerItemsList=" + rankerItemsList +
-                ", children=" + children +
-                ", items=" + items +
-                '}';
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/AllowListFilter.java

@@ -1,113 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.google.common.collect.Lists;
-import com.google.common.hash.Hashing;
-import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.PreViewedService;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.data.redis.core.RedisTemplate;
-
-import java.util.*;
-
-public class AllowListFilter extends AbstractFilter<Video> {
-
-    @Autowired
-    @Qualifier("longVideoRedisTemplate")
-    private RedisTemplate<String, String> redisTemplate;
-
-    private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX = "movie:videoid:allowSet:";
-    private static final String RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY = "mp:religion:allowlist:videoid:bitmap";
-
-    public AllowListFilter(FilterConfigInfo filterConfigInfo,
-                           RecommendRequest recommendRequest,
-                           User user) {
-        super(filterConfigInfo, recommendRequest, user);
-
-    }
-
-
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> list) {
-        if (CollectionUtils.isEmpty(list)) {
-            return;
-        }
-        List<Video> result = Lists.newArrayList();
-        for (Video video : list) {
-            if (!isAllowList(user, video)) {
-                result.add(video);
-            }
-        }
-        list.clear();
-        list.addAll(result);
-    }
-
-
-    public boolean isAllowList(User user, Video video) {
-        //不是新小程序 在白名单则不显示
-        Set<Long> retainVideoIds = new LinkedHashSet<>();
-        int appType = requestContext.getAppType();
-        if ( appType != AppTypeEnum.WAN_NENG_VIDEO.getCode()
-                && appType != AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
-                && appType != AppTypeEnum.ZUI_JING_QI.getCode()
-                && appType!= AppTypeEnum.H5.getCode()) {
-            // 如果不在新小程序白名单，则允许
-            return !isMemberOfVideoAllowList(video.getVideoId());
-
-        } else if (appType == AppTypeEnum.WAN_NENG_VIDEO.getCode()
-                || appType == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
-                || appType == AppTypeEnum.ZUI_JING_QI.getCode()
-                || appType == AppTypeEnum.H5.getCode()) {
-            return !isMemberOfReligionVideoAllowList(video.getVideoId());
-
-        }
-        return true;
-    }
-
-    private boolean isMemberOfVideoAllowList(Long videoId) {
-        if (Objects.isNull(videoId)) {
-            return false;
-        }
-        try {
-            int newIdx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 100;
-            String newPrefix = VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX + newIdx;
-            Boolean result = redisTemplate.opsForSet().isMember(newPrefix, String.valueOf(videoId));
-            return result;
-        } catch (Exception e) {
-            LOGGER.error("isMemberOfVideoAllowList error {}", videoId, e);
-        }
-        return false;
-    }
-
-    public boolean isMemberOfReligionVideoAllowList(Long videoId) {
-        if (Objects.isNull(videoId)) {
-            return false;
-        }
-        try {
-            return redisTemplate.opsForValue().getBit(RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY, videoId);
-        } catch (Exception e) {
-            LOGGER.error("isMemberOfReligionVideoAllowList error {}", e, videoId);
-        }
-        return false;
-    }
-
-
-
-
-
-}
-
-
-
-

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/HistoryLongPeriodFilter.java

@@ -1,59 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.RecommendParam;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.RecommendService;
-import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
-import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
-import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * TODO 有性能问题，暂时废弃
- *
- * @author sunxy
- */
-public class HistoryLongPeriodFilter extends AbstractFilter<Video> {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(HistoryLongPeriodFilter.class);
-
-    private final RegionFilterService filterService;
-    private final RecommendService recommendService;
-
-    public HistoryLongPeriodFilter(FilterConfigInfo filterConfigInfo, RecommendRequest requestContext, User user) {
-        super(filterConfigInfo, requestContext, user);
-        filterService = ServiceBeanFactory.getBean(RegionFilterService.class);
-        recommendService = ServiceBeanFactory.getBean(RecommendService.class);
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> list) {
-        if (list == null || list.isEmpty()) {
-            return;
-        }
-        RecommendParam recommendParam = recommendService.genRecommendParam(requestContext, 1);
-        RecallParam recallParam = recommendService.convertToRecallParam(recommendParam);
-        FilterParam filterParam = FilterParamFactory.create(recallParam,
-                list.stream().map(Video::getVideoId).collect(Collectors.toList()));
-        filterParam.setForceTruncation(10000);
-        filterParam.setConcurrent(true);
-        filterParam.setNotUsePreView(false);
-        FilterResult filterResult = filterService.filter(filterParam);
-        if (filterResult != null && filterResult.getVideoIds() != null) {
-            list.removeIf(video -> !filterResult.getVideoIds().contains(video.getVideoId()));
-        }
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/PreViewedFilter.java

@@ -1,54 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.PreViewedService;
-import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-public class PreViewedFilter extends AbstractFilter<Video> {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(PreViewedFilter.class);
-
-    private final Set<Long> preViewedVideoIds;
-
-    public PreViewedFilter(FilterConfigInfo filterConfigInfo,
-                           RecommendRequest recommendRequest,
-                           User user) {
-        super(filterConfigInfo, recommendRequest, user);
-        PreViewedService preViewedService = ServiceBeanFactory.getBean(PreViewedService.class);
-        preViewedVideoIds = preViewedService.getVideoIds(this.requestContext.getAppType(), user.getMid());
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> list) {
-        if (StringUtils.isBlank(user.getMid())
-                || CollectionUtils.isEmpty(list)) {
-            return;
-        }
-
-        if (CollectionUtils.isEmpty(preViewedVideoIds)) {
-            return;
-        }
-        list.removeIf(video -> preViewedVideoIds.contains(video.getVideoId()));
-    }
-
-
-}
-
-
-
-

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/RecommendStatusFilter.java

@@ -1,47 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.SpringContextHolder;
-import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
-import com.tzld.piaoquan.recommend.server.service.filter.strategy.RecommendStatusStrategy;
-import org.apache.commons.collections4.CollectionUtils;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-public class RecommendStatusFilter extends AbstractFilter<Video> {
-
-    private final RecommendStatusStrategy recommendStatusStrategy;
-
-    public RecommendStatusFilter(FilterConfigInfo filterConfigInfo,
-                                 RecommendRequest recommendRequest,
-                                 User user) {
-        super(filterConfigInfo, recommendRequest, user);
-        recommendStatusStrategy = SpringContextHolder.getBean(RecommendStatusStrategy.class);
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> videos) {
-        if (CollectionUtils.isEmpty(videos)) {
-            return;
-        }
-        FilterParam filterParam = new FilterParam();
-        filterParam.setVideoIds(videos.stream().map(Video::getVideoId).collect(Collectors.toList()));
-        List<Long> videoIdList = recommendStatusStrategy.filter(filterParam);
-        Set<Long> videoIdSet = new HashSet<>(videoIdList);
-        videos.removeIf(video -> !videoIdSet.contains(video.getVideoId()));
-    }
-
-}
-
-
-
-

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/RiskVideoFilter.java

@@ -1,28 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-
-import java.util.List;
-
-public class RiskVideoFilter extends AbstractFilter<Video> {
-
-
-    public RiskVideoFilter(FilterConfigInfo filterConfigInfo,
-                           RecommendRequest recommendRequest,
-                           User user) {
-        super(filterConfigInfo, recommendRequest, user);
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> t) {
-
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/TagFilter.java

@@ -1,105 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Lists;
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRel;
-import com.tzld.piaoquan.recommend.server.repository.WxVideoTagRelRepository;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.math.NumberUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-public class TagFilter extends AbstractFilter<Video> {
-
-    @Autowired
-    private WxVideoTagRelRepository repository;
-
-    @Value("${video.filter.tagids:}")
-    private String videoFilterTagIds;
-
-    private LoadingCache<Long, Set<Long>> videoTagCache = CacheBuilder.newBuilder()
-            .maximumSize(100)
-            .refreshAfterWrite(60, TimeUnit.SECONDS)
-            .expireAfterWrite(60, TimeUnit.SECONDS)
-            .expireAfterAccess(60, TimeUnit.SECONDS)
-            .build(new CacheLoader<Long, Set<Long>>() {
-                @Override
-                public Set<Long> load(Long tagId) {
-                    List<WxVideoTagRel> rels = repository.findAllByTagId(tagId);
-                    return CommonCollectionUtils.toSet(rels, WxVideoTagRel::getVideoId);
-                }
-            });
-
-
-    public TagFilter(FilterConfigInfo filterConfigInfo,
-                     RecommendRequest recommendRequest,
-                     User user) {
-        super(filterConfigInfo, recommendRequest, user);
-
-        if (org.apache.commons.lang.StringUtils.isNotBlank(this.videoFilterTagIds)) {
-            String[] tags = this.videoFilterTagIds.split(",");
-            for (String tag : tags) {
-                if (org.apache.commons.lang.StringUtils.isBlank(tag)) {
-                    continue;
-                }
-                this.videoTagCache.getUnchecked(NumberUtils.toLong(tag));
-            }
-        }
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> t) {
-        if (CollectionUtils.isEmpty(t)) {
-            return;
-        }
-        List<Video> videos = Lists.newArrayList();
-        for (Video video : t) {
-            if (!hasVideoRelTagId(video.getVideoId(), videoFilterTagIds)) {
-                videos.add(video);
-            }
-        }
-        t.removeAll(videos);
-    }
-
-    private boolean hasVideoRelTagId(Long videoId, String videoFilterTagIds) {
-        // TODO 主要是涉政标签
-        List<Long> tagIds = new ArrayList<>();
-
-        String[] tags = videoFilterTagIds.split(",");
-        for (String tag : tags) {
-            if (Objects.isNull(tag) || Objects.equals("", tag)) {
-                continue;
-            }
-            tagIds.add(Long.parseLong(tag));
-        }
-
-        for (Long tagId : tagIds) {
-            if (Objects.isNull(videoId) || videoId <= 0L || Objects.isNull(tagId) || tagId <= 0L) {
-                return false;
-            }
-            Set<Long> videos = this.videoTagCache.getUnchecked(tagId);
-            if (CollectionUtils.isEmpty(videos)) {
-                return false;
-            }
-            return videos.contains(videoId);
-        }
-        return false;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/recall/ViewedHistoryFilter.java

@@ -1,115 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.recall;
-
-
-import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.framework.recaller.AbstractFilter;
-import com.tzld.piaoquan.recommend.server.framework.recaller.FilterConfigInfo;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
-import com.tzld.piaoquan.recommend.server.service.SpringContextHolder;
-import com.tzld.piaoquan.recommend.server.service.filter.VideoCityFilterService;
-import com.tzld.piaoquan.recommend.server.service.filter.strategy.VideoView;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.time.StopWatch;
-import org.springframework.data.mongodb.core.MongoTemplate;
-import org.springframework.data.mongodb.core.query.Criteria;
-import org.springframework.data.mongodb.core.query.Query;
-import org.springframework.data.redis.core.RedisTemplate;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-public class ViewedHistoryFilter extends AbstractFilter<Video> {
-
-    private final VideoCityFilterService videoCityFilterService;
-
-    protected Set<String> historySet;
-
-    private final RedisTemplate<String, String> redisTemplate;
-
-    private final MongoTemplate mongoTemplate;
-
-    private final String keyFormat = "user:exclude:videoidset:%s";
-
-    public ViewedHistoryFilter(FilterConfigInfo filterConfigInfo,
-                               RecommendRequest recommendRequest,
-                               User user) {
-        super(filterConfigInfo, recommendRequest, user);
-        mongoTemplate = ServiceBeanFactory.getBean(MongoTemplate.class);
-        redisTemplate = SpringContextHolder.getBean("filterRedisTemplate", RedisTemplate.class);
-        historySet = this.loadUserHistory(user);
-        if (historySet == null) {
-            historySet = new HashSet<>();
-        }
-        videoCityFilterService = SpringContextHolder.getBean(VideoCityFilterService.class);
-
-    }
-
-    @Override
-    public void doFilter(Candidate candidate, List<Video> videoList) {
-        if (CollectionUtils.isEmpty(videoList)) {
-            return;
-        }
-        videoList.removeIf(video -> this.historySet.contains(String.valueOf(video.getVideoId())));
-        if (CollectionUtils.isEmpty(videoList)) {
-            return;
-        }
-        List<Long> filterVideosByCity = videoCityFilterService.filterVideosByCity(videoList.stream().map(Video::getVideoId).collect(Collectors.toList()),
-                new HashSet<>(requestContext.getAbExpCodeList()), requestContext.getHotSceneType(), requestContext.getCityCode());
-        Set<Long> filterVideosSet = new HashSet<>(filterVideosByCity);
-        videoList.removeIf(video -> !filterVideosSet.contains(video.getVideoId()));
-    }
-
-
-    public Set<String> loadUserHistory(User user) {
-        String userid = StringUtils.isNotBlank(user.getUid()) ? user.getUid() : user.getMid();
-        if (StringUtils.isBlank(userid)) {
-            return new HashSet<>();
-        }
-
-        String key = String.format(keyFormat, userid);
-        Set<String> viewedVideoIds = redisTemplate.opsForSet().members(key);
-        if (CollectionUtils.isEmpty(viewedVideoIds)) {
-            viewedVideoIds = new HashSet<>();
-            StopWatch stopWatch = new StopWatch();
-            stopWatch.start();
-            // 从mongodb中取曝光记录
-            Criteria criteriatime = new Criteria();
-            criteriatime.and("uid").is(userid);
-            Query query = new Query();
-            query.addCriteria(criteriatime);
-            //查新库
-            List<VideoView> list = mongoTemplate.find(query, VideoView.class);
-            //限制最多10000条
-            if (Objects.nonNull(list) && list.size() > 10000) {
-                list = list.subList(list.size() - 10000, list.size());
-            }
-            if (CollectionUtils.isNotEmpty(list)) {
-                viewedVideoIds.addAll(list.stream().map(VideoView::getVideoId).map(String::valueOf).collect(Collectors.toSet()));
-            } else {
-                // 避免缓存击穿
-                viewedVideoIds.add("-1");
-            }
-            stopWatch.stop();
-
-            redisTemplate.opsForSet().add(key, viewedVideoIds.toArray(new String[0]));
-            redisTemplate.expire(key, 1, TimeUnit.DAYS);
-        }
-
-        return viewedVideoIds;
-    }
-
-
-
-
-
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/BaseLRModelScorer.java

@@ -1,22 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.score;
-
-import com.tzld.piaoquan.recommend.server.framework.score.AbstractScorer;
-import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
-import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public abstract class BaseLRModelScorer extends AbstractScorer {
-
-    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRModelScorer.class);
-
-    public BaseLRModelScorer(ScorerConfigInfo scorerConfigInfo) {
-        super(scorerConfigInfo);
-    }
-
-    @Override
-    public void loadModel() {
-        doLoadModel(LRModel.class);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogShareLRScorer.java

@@ -1,328 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.score;
-
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
-import com.tzld.piaoquan.recommend.feature.domain.video.feature.VlogShareLRFeatureExtractor;
-import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
-import com.tzld.piaoquan.recommend.server.service.rank.strategy.OfflineVlogShareLRFeatureExtractor;
-import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-
-public class VlogShareLRScorer extends BaseLRModelScorer {
-
-    private final static int CORE_POOL_SIZE = 64;
-
-    private static final int LOCAL_TIME_OUT = 150;
-    private final static Logger LOGGER = LoggerFactory.getLogger(VlogShareLRScorer.class);
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
-    private static final double defaultUserCtrGroupNumber = 10.0;
-    private static final int enterFeedsScoreRatio = 10;
-    private static final int enterFeedsScoreNum = 20;
-
-
-    public VlogShareLRScorer(ScorerConfigInfo configInfo) {
-        super(configInfo);
-    }
-
-
-    @Override
-    public List<RankItem> scoring(final ScoreParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
-
-        if (CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<RankItem> result = rankItems;
-        result = rankByJava(rankItems, param.getRequestContext(),
-                userFeature == null ? UserFeature.defaultInstance(param.getMid()) : userFeature);
-
-        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<RankItem> rankByJava(final List<RankItem> items,
-                                      final RequestContext requestContext,
-                                      final UserFeature user) {
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        // userBytes
-        UserBytesFeature userInfoBytes = null;
-        userInfoBytes = new UserBytesFeature(user);
-
-        // 所有都参与打分，按照ctr排序
-        multipleCtrScore(items, userInfoBytes, requestContext, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
-            }
-        }
-
-        Collections.sort(items);
-
-        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-
-    /**
-     * 计算 predict ctr
-     */
-    public double calcScore(final LRModel lrModel,
-                            final RankItem item,
-                            final UserBytesFeature userInfoBytes,
-                            final RequestContext requestContext) {
-
-        LRSamples lrSamples = null;
-        VlogShareLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new VlogShareLRFeatureExtractor();
-
-        try {
-            VideoBytesFeature newsInfoBytes = new VideoBytesFeature(item.getItemFeature() == null
-                    ? ItemFeature.defaultInstance(item.getVideoId() + "")
-                    : item.getItemFeature());
-            lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
-                    new RequestContextBytesFeature(requestContext));
-        } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e));
-        }
-
-
-        double pro = 0.0;
-        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
-            try {
-                pro = lrModel.score(lrSamples);
-            } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
-            }
-            // 增加实时特征后打开在线存储日志逻辑
-            //
-            // CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
-            // samples.setLr_samples(lrSamples);
-            // item.setSamples(samples);
-            //
-        }
-        item.setScore(pro);
-        return pro;
-    }
-
-
-    /**
-     * 并行打分
-     *
-     * @param items
-     * @param userInfoBytes
-     * @param requestContext
-     * @param model
-     */
-    private void multipleCtrScore(final List<RankItem> items,
-                                  final UserBytesFeature userInfoBytes,
-                                  final RequestContext requestContext,
-                                  final LRModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
-                    } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", requestContext.getRequest_id(),
-                            ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
-    }
-    @Override
-    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
-                                           final Map<String, String> userFeatureMap,
-                                           final List<RankItem> rankItems){
-        if (CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<RankItem> result = rankItems;
-        result = rankByJava(
-                sceneFeatureMap, userFeatureMap, rankItems
-        );
-
-        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
-                                      final Map<String, String> userFeatureMap,
-                                      final List<RankItem> items) {
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-        // userBytes
-        Map<String, byte[]> userFeatureMapByte = new HashMap<>();
-        for(Map.Entry<String, String> entry: userFeatureMap.entrySet()){
-            userFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
-        }
-        //sceneBytes
-        Map<String, byte[]> sceneFeatureMapByte = new HashMap<>();
-        for(Map.Entry<String, String> entry: sceneFeatureMap.entrySet()){
-            sceneFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
-        }
-
-        // 所有都参与打分，按照ctr排序
-        multipleCtrScore(items, userFeatureMapByte, sceneFeatureMapByte, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
-            }
-        }
-
-        Collections.sort(items);
-
-        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-    private void multipleCtrScore(final List<RankItem> items,
-                                  final Map<String, byte[]> userFeatureMapByte,
-                                  final Map<String, byte[]> sceneFeatureMapByte,
-                                  final LRModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-            // items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex), userFeatureMapByte, sceneFeatureMapByte);
-                    } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", sceneFeatureMapByte.size(),
-                            ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
-    }
-
-    public double calcScore(final LRModel lrModel,
-                            final RankItem item,
-                            final Map<String, byte[]> userFeatureMapByte,
-                            final Map<String, byte[]> sceneFeatureMapByte) {
-
-        LRSamples lrSamples = null;
-        OfflineVlogShareLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new OfflineVlogShareLRFeatureExtractor();
-
-        try {
-
-            Map<String, byte[]> itemFeatureByte = new HashMap<>();
-            for (Map.Entry<String, String> entry: item.getFeatureMap().entrySet()){
-                itemFeatureByte.put(entry.getKey(), entry.getValue().getBytes());
-            }
-            lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
-        } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e));
-        }
-
-
-        double pro = 0.0;
-        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
-            try {
-                pro = lrModel.score(lrSamples);
-            } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
-            }
-        }
-        item.setScoreStr(pro);
-        return pro;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogShareLRScorer4Ros.java

@@ -1,328 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.score;
-
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
-import com.tzld.piaoquan.recommend.feature.domain.video.feature.VlogShareLRFeatureExtractor;
-import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
-import com.tzld.piaoquan.recommend.server.service.rank.strategy.OfflineVlogShareLRFeatureExtractor;
-import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-
-public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
-
-    private final static int CORE_POOL_SIZE = 64;
-
-    private static final int LOCAL_TIME_OUT = 150;
-    private final static Logger LOGGER = LoggerFactory.getLogger(VlogShareLRScorer4Ros.class);
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
-    private static final double defaultUserCtrGroupNumber = 10.0;
-    private static final int enterFeedsScoreRatio = 10;
-    private static final int enterFeedsScoreNum = 20;
-
-
-    public VlogShareLRScorer4Ros(ScorerConfigInfo configInfo) {
-        super(configInfo);
-    }
-
-
-    @Override
-    public List<RankItem> scoring(final ScoreParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
-
-        if (CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<RankItem> result = rankItems;
-        result = rankByJava(rankItems, param.getRequestContext(),
-                userFeature == null ? UserFeature.defaultInstance(param.getMid()) : userFeature);
-
-        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<RankItem> rankByJava(final List<RankItem> items,
-                                      final RequestContext requestContext,
-                                      final UserFeature user) {
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        // userBytes
-        UserBytesFeature userInfoBytes = null;
-        userInfoBytes = new UserBytesFeature(user);
-
-        // 所有都参与打分，按照ctr排序
-        multipleCtrScore(items, userInfoBytes, requestContext, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
-            }
-        }
-
-        Collections.sort(items);
-
-        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-
-    /**
-     * 计算 predict ctr
-     */
-    public double calcScore(final LRModel lrModel,
-                            final RankItem item,
-                            final UserBytesFeature userInfoBytes,
-                            final RequestContext requestContext) {
-
-        LRSamples lrSamples = null;
-        VlogShareLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new VlogShareLRFeatureExtractor();
-
-        try {
-            VideoBytesFeature newsInfoBytes = new VideoBytesFeature(item.getItemFeature() == null
-                    ? ItemFeature.defaultInstance(item.getVideoId() + "")
-                    : item.getItemFeature());
-            lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
-                    new RequestContextBytesFeature(requestContext));
-        } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e));
-        }
-
-
-        double pro = 0.0;
-        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
-            try {
-                pro = lrModel.score(lrSamples);
-            } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
-            }
-            // 增加实时特征后打开在线存储日志逻辑
-            //
-            // CtrSamples.Builder samples =  com.tzld.piaoquan.recommend.server.gen.recommend.CtrSamples.newBuilder();
-            // samples.setLr_samples(lrSamples);
-            // item.setSamples(samples);
-            //
-        }
-        item.setScore(pro);
-        return pro;
-    }
-
-
-    /**
-     * 并行打分
-     *
-     * @param items
-     * @param userInfoBytes
-     * @param requestContext
-     * @param model
-     */
-    private void multipleCtrScore(final List<RankItem> items,
-                                  final UserBytesFeature userInfoBytes,
-                                  final RequestContext requestContext,
-                                  final LRModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex), userInfoBytes, requestContext);
-                    } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", requestContext.getRequest_id(),
-                            ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
-    }
-    @Override
-    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
-                                  final Map<String, String> userFeatureMap,
-                                  final List<RankItem> rankItems){
-        if (CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<RankItem> result = rankItems;
-        result = rankByJava(
-                sceneFeatureMap, userFeatureMap, rankItems
-        );
-
-        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
-                                      final Map<String, String> userFeatureMap,
-                                      final List<RankItem> items) {
-        long startTime = System.currentTimeMillis();
-        LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-        // userBytes
-        Map<String, byte[]> userFeatureMapByte = new HashMap<>();
-        for(Map.Entry<String, String> entry: userFeatureMap.entrySet()){
-            userFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
-        }
-        //sceneBytes
-        Map<String, byte[]> sceneFeatureMapByte = new HashMap<>();
-        for(Map.Entry<String, String> entry: sceneFeatureMap.entrySet()){
-            sceneFeatureMapByte.put(entry.getKey(), entry.getValue().getBytes());
-        }
-
-        // 所有都参与打分，按照ctr排序
-        multipleCtrScore(items, userFeatureMapByte, sceneFeatureMapByte, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
-            }
-        }
-
-        Collections.sort(items);
-
-        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-    private void multipleCtrScore(final List<RankItem> items,
-                                  final Map<String, byte[]> userFeatureMapByte,
-                                  final Map<String, byte[]> sceneFeatureMapByte,
-                                  final LRModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-//            items.get(fIndex).setScore(0.0);   //原始分为 cube中的粗打分，如果超时，为原始值存在问题， 需要置0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex), userFeatureMapByte, sceneFeatureMapByte);
-                    } catch (Exception e) {
-                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", sceneFeatureMapByte.size(),
-                            ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
-    }
-
-    public double calcScore(final LRModel lrModel,
-                            final RankItem item,
-                            final Map<String, byte[]> userFeatureMapByte,
-                            final Map<String, byte[]> sceneFeatureMapByte) {
-
-        LRSamples lrSamples = null;
-        OfflineVlogShareLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new OfflineVlogShareLRFeatureExtractor();
-
-        try {
-
-            Map<String, byte[]> itemFeatureByte = new HashMap<>();
-            for (Map.Entry<String, String> entry: item.getFeatureMap().entrySet()){
-                itemFeatureByte.put(entry.getKey(), entry.getValue().getBytes());
-            }
-            lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
-        } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e));
-        }
-
-
-        double pro = 0.0;
-        if (lrSamples != null && lrSamples.getFeaturesList() != null) {
-            try {
-                pro = lrModel.score(lrSamples);
-            } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
-            }
-        }
-        item.setScoreRos(pro);
-        return pro;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogThompsonScorer.java

@@ -1,143 +0,0 @@
-package com.tzld.piaoquan.recommend.server.implement.score;
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.framework.score.BaseThompsonSamplingScorer;
-import com.tzld.piaoquan.recommend.server.framework.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.framework.score.ScorerConfigInfo;
-import com.tzld.piaoquan.recommend.server.service.score.model.ThompsonSamplingModel;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.*;
-
-
-//@Service
-public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
-
-    private static final int LOCAL_TIME_OUT = 150;
-    private final static Logger LOGGER = LoggerFactory.getLogger(VlogThompsonScorer.class);
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
-
-    public VlogThompsonScorer(ScorerConfigInfo configInfo) {
-        super(configInfo);
-    }
-
-    @Override
-    public List<RankItem> scoring(final ScoreParam param,
-                                  final UserFeature userFeature,
-                                  final List<RankItem> rankItems) {
-
-        if (userFeature == null || CollectionUtils.isEmpty(rankItems)) {
-            return rankItems;
-        }
-
-        long startTime = System.currentTimeMillis();
-        ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        List<RankItem> result = rankItems;
-        result = rankByJava(rankItems, param.getRequestContext(), userFeature);
-
-        LOGGER.debug("thompson sampling ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
-                System.currentTimeMillis() - startTime);
-
-        return result;
-    }
-
-    private List<RankItem> rankByJava(final List<RankItem> items,
-                                      final RequestContext requestContext,
-                                      final UserFeature user) {
-        long startTime = System.currentTimeMillis();
-        ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
-
-        // 所有都参与打分，按照ROV Thompson排序
-        multipleCtrScore(items, model);
-
-        // debug log
-        if (LOGGER.isDebugEnabled()) {
-            for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
-            }
-        }
-
-        LOGGER.debug("thompson ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
-        LOGGER.debug("[thompson ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
-                System.currentTimeMillis() - startTime);
-        return items;
-    }
-
-
-    /**
-     * 计算 predict ROV
-     */
-    public double calcScore(final ThompsonSamplingModel model,
-                            final RankItem item) {
-        double score = 0d;
-        try {
-            score = model.score(item);
-        } catch (Exception e) {
-            LOGGER.error("score error for doc={} exception={}", item.getVideo(), ExceptionUtils.getFullStackTrace(e));
-        }
-        item.setScore(score);
-        return score;
-    }
-
-
-    /**
-     * 并行打分 Thompson ROV
-     *
-     * @param items
-     * @param model
-     */
-    private void multipleCtrScore(final List<RankItem> items,
-                                  final ThompsonSamplingModel model) {
-
-        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
-        for (int index = 0; index < items.size(); index++) {
-            final int fIndex = index;
-            items.get(fIndex).setScore(0.0);   //设置为原始值为0
-            calls.add(new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    try {
-                        calcScore(model, items.get(fIndex));
-                    } catch (Exception e) {
-                        LOGGER.error("thompson exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
-                    }
-                    return new Object();
-                }
-            });
-        }
-
-        List<Future<Object>> futures = null;
-        try {
-            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
-        }
-
-        //等待所有请求的结果返回, 超时也返回
-        int cancel = 0;
-        if (futures != null) {
-            for (Future<Object> future : futures) {
-                try {
-                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
-                        cancel++;
-                    }
-                } catch (InterruptedException e) {
-                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
-                } catch (ExecutionException e) {
-                    LOGGER.error("ExecutionException {},{}", ExceptionUtils.getFullStackTrace(e));
-                }
-            }
-        }
-        LOGGER.debug("ROV-Thompson Score {}, Total: {}, Cancel: {}", items.size(), cancel);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/RecommendParam.java

@@ -1,12 +1,9 @@
 package com.tzld.piaoquan.recommend.server.model;
 
-import com.alibaba.fastjson.annotation.JSONField;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 
-import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 /**
@@ -19,17 +16,9 @@ import java.util.Set;
 public class RecommendParam {
     private int topK;
     private double flowPoolP;
-    private String abCode;
     private String ruleKey;
     private String dataKey;
 
-    // hardcode   兼容旧服务的518实验
-    private String hDataKey;
-    private String hRuleKey;
-
-    private int flowPoolId;
-    private int lastDigit;
-    private String flowPoolAbtestGroup;
     private String rankKeyPrefix;
     private int appType;
     private Long videoId;
@@ -39,22 +28,12 @@ public class RecommendParam {
     private int size;
     private String uid;
 
-    // 风险过滤
-    private int expansionFactor;
-    private Boolean riskFilterFlag;
-    private Map<Integer, List<String>> appRegionFiltered;
-
-    @JSONField(serialize = false)
-    private List<Long> videosWithRisk;
-
     private String regionCode;
     private MachineInfo machineInfo;
     private String province;
     private String city;
     private Set<String> abExpCodes;
-
-    // 层 - 实验
-    private Map<String, String> expIdMap;
+    private boolean riskUser;
 
     private Integer categoryId;
 
@@ -62,5 +41,11 @@ public class RecommendParam {
 
     private String clientIp;
 
+    private Integer versionCode;
+
+    private String rootSourceId;
+    private Integer userShareDepth;
+
+    private int pageNum;
 }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/Video.java

@@ -2,10 +2,7 @@ package com.tzld.piaoquan.recommend.server.model;
 
 import lombok.Data;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 /**
  * @author dyp
@@ -16,12 +13,10 @@ public class Video {
     private double rovScore;
     private double sortScore;
     private String pushFrom;
-    private String abCode;
 
     // 流量池相关 start
     private String flowPool;
     private String level;
-    private String flowPoolAbtestGroup;
     private boolean inFlowPool;
     // 流量池相关 end
 
@@ -31,6 +26,9 @@ public class Video {
     // video的特征 tag
     private List<String> tags = new ArrayList<>();
 
+    // 视频的品类
+    private Set<String> mergeCateList = new HashSet<>();
+
     // video的模型打分
     public double scoreRos = 0.0D;
     public double scoreStr = 0.0D;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/remote/ABTestRemoteService.java

@@ -1,36 +0,0 @@
-package com.tzld.piaoquan.recommend.server.remote;
-
-import com.tzld.piaoquan.abtest.client.ABTestClient;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.util.Map;
-
-
-/**
- * @author dyp
- */
-@Component
-@Slf4j
-public class ABTestRemoteService {
-
-    @Autowired
-    private ABTestClient client;
-
-//    public String getGroup(String mid, String uid, int appType, String group) {
-//        if (StringUtils.isBlank(mid)) {
-//            return null;
-//        }
-//        return client.getGroup(mid, uid, appType, group);
-//    }
-
-    public Map<String, String> getExp(String mid, String uid, int appType, String group) {
-        if (StringUtils.isBlank(mid)) {
-            return null;
-        }
-        return client.getExp(mid, uid, appType, group);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/remote/FeatureRemoteService.java

@@ -1,143 +0,0 @@
-package com.tzld.piaoquan.recommend.server.remote;
-
-import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserActionFeature;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.feature.model.feature.UserActionFeatureProto;
-import com.tzld.piaoquan.recommend.feature.model.feature.UserFeatureProto;
-import com.tzld.piaoquan.recommend.feature.model.feature.VideoFeatureProto;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.math.NumberUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-
-/**
- * @author dyp
- */
-@Component
-@Slf4j
-public class FeatureRemoteService {
-
-    @Autowired
-    private FeatureClient client;
-
-    // 有性能问题再增加localCache
-    public UserFeature getUserFeature(String mid) {
-        if (StringUtils.isBlank(mid)) {
-            return null;
-        }
-        UserFeatureProto proto = client.getUserFeature(mid);
-        return convert(proto);
-    }
-
-    private UserFeature convert(UserFeatureProto proto) {
-        if (proto == null) {
-            return null;
-        }
-        UserFeature feature = new UserFeature();
-        feature.setUid(proto.getUid());
-        feature.setMid(proto.getMid());
-        if (proto.hasDay1CntFeature()) {
-            feature.setDay1_cnt_features(convert(proto.getDay1CntFeature()));
-        } else {
-            feature.setDay1_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasDay3CntFeature()) {
-            feature.setDay3_cnt_features(convert(proto.getDay3CntFeature()));
-        } else {
-            feature.setDay3_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasDay7CntFeature()) {
-            feature.setDay7_cnt_features(convert(proto.getDay7CntFeature()));
-        } else {
-            feature.setDay7_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasMonth3CntFeature()) {
-            feature.setMonth3_cnt_features(convert(proto.getMonth3CntFeature()));
-        } else {
-            feature.setMonth3_cnt_features(new UserActionFeature());
-        }
-        feature.setUser_cycle_bucket_7days(proto.getUserCycleBucket7Day());
-        feature.setUser_cycle_bucket_30days(proto.getUserCycleBucket30Day());
-        feature.setUser_share_bucket_30days(proto.getUserShareBucket30Day());
-
-        return feature;
-    }
-
-    private UserActionFeature convert(UserActionFeatureProto proto) {
-        UserActionFeature feature = new UserActionFeature();
-        feature.setClick_cnt(proto.getClickCnt());
-        feature.setCtr(proto.getCtr());
-        feature.setExp_cnt(proto.getExpCnt());
-        feature.setReturn_cnt(proto.getReturnCnt());
-        feature.setRov(proto.getRov());
-        feature.setShare_cnt(proto.getShareCnt());
-        feature.setStr(proto.getStr());
-        return feature;
-    }
-
-    public ItemFeature getVideoFeature(Long videoId) {
-        if (videoId == null) {
-            return null;
-        }
-        VideoFeatureProto proto = client.getVideoFeature(videoId);
-        return convert(proto);
-    }
-
-    public Map<Long, ItemFeature> getVideoFeatureMap(List<Long> videoIds) {
-        if (CollectionUtils.isEmpty(videoIds)) {
-            return Collections.emptyMap();
-        }
-        List<VideoFeatureProto> protoList = client.getAllVideoFeature(videoIds);
-        return CommonCollectionUtils.toMap(protoList, p -> NumberUtils.toLong(p.getVideoId(), 0L), this::convert);
-    }
-
-
-    private ItemFeature convert(VideoFeatureProto proto) {
-        if (proto == null) {
-            return null;
-        }
-        ItemFeature feature = new ItemFeature();
-
-        feature.setPlayLength(proto.getPlayLength());
-        feature.setTags(proto.getTags());
-        feature.setTotalTime(proto.getTotalTime());
-        feature.setUpId(proto.getUpId());
-        feature.setVideoId(proto.getVideoId());
-        feature.setDaysSinceUpload(proto.getDaysSinceUpload());
-
-
-        if (proto.hasVideoDay1CntFeature()) {
-            feature.setDay1_cnt_features(convert(proto.getVideoDay1CntFeature()));
-        } else {
-            feature.setDay1_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasVideoDay3CntFeature()) {
-            feature.setDay3_cnt_features(convert(proto.getVideoDay3CntFeature()));
-        } else {
-            feature.setDay3_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasVideoDay7CntFeature()) {
-            feature.setDay7_cnt_features(convert(proto.getVideoDay7CntFeature()));
-        } else {
-            feature.setDay7_cnt_features(new UserActionFeature());
-        }
-        if (proto.hasVideoMonth3CntFeature()) {
-            feature.setMonth3_cnt_features(convert(proto.getVideoMonth3CntFeature()));
-        } else {
-            feature.setMonth3_cnt_features(new UserActionFeature());
-        }
-        return feature;
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/FeatureService.java

@@ -1,11 +1,17 @@
 package com.tzld.piaoquan.recommend.server.service;
 
+import com.alibaba.fastjson.JSON;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
+import com.tzld.piaoquan.recommend.server.model.MachineInfo;
 import com.tzld.piaoquan.recommend.server.remote.FeatureV2RemoteService;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.bo.UserShareReturnProfile;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -22,6 +28,9 @@ public class FeatureService {
     @Autowired
     private FeatureV2RemoteService remoteService;
 
+    private static final Set<String> hotSceneSet = new HashSet<>(Arrays.asList("1008", "1007", "1058", "1074", "1010"));
+    private static final Set<String> hotSceneTypeSet = new HashSet<>(Arrays.asList("1008", "1007"));
+
     /**
      * @return k1:视频、k2:表、k3:特征、v:特征值
      */
@@ -67,7 +76,7 @@ public class FeatureService {
 
             // vid + headvid
             protos.add(genWithVidAndHeadVid("alg_recsys_feature_cf_i2i_new", vid, headVid));
-            protos.add(genWithVidAndHeadVid("alg_recsys_feature_cf_i2i_new_v2", vid, headVid));
+            //protos.add(genWithVidAndHeadVid("alg_recsys_feature_cf_i2i_new_v2", vid, headVid));
         }
 
 
@@ -120,8 +129,330 @@ public class FeatureService {
         return feature;
     }
 
+
+    public Map<String, Map<String, Map<String, String>>> getVideoBaseInfo(String headVid, List<String> vidList) {
+        List<FeatureKeyProto> protos = new ArrayList<>();
+        if (null != headVid && !headVid.isEmpty()) {
+            protos.add(genWithKeyMap("alg_vid_feature_basic_info", headVid, ImmutableMap.of("vid", headVid)));
+        }
+        for (String vid : vidList) {
+            protos.add(genWithKeyMap("alg_vid_feature_basic_info", vid, ImmutableMap.of("vid", vid)));
+        }
+        Feature feature = getFeatureByProto(protos);
+        return feature.getVideoFeature();
+    }
+
+    public Feature getNewFeature(String province, String mid, String sceneType, String headVid,
+                                 Map<String, Map<String, Map<String, String>>> videoBaseInfoMap, List<String> vidList) {
+        List<FeatureKeyProto> protos = new ArrayList<>();
+        String i2iSceneType = "other";
+        if (hotSceneSet.contains(sceneType)) {
+            i2iSceneType = sceneType;
+        }
+        for (String vid : vidList) {
+            // ********************* old vid ******************
+            protos.add(genWithVid("alg_vid_feature_all_exp_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_all_share", vid));
+            protos.add(genWithVid("alg_vid_feature_all_return", vid));
+            protos.add(genWithVid("alg_vid_feature_exp2share_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_share2return", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_exp_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_root_share_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_root_return_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_exp_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_root_share_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_root_return_v2", vid));
+
+            // vid + province
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_exp_v2", vid, province));
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_root_share_v2", vid, province));
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_root_return_v2", vid, province));
+
+            // headvid + vid
+            // protos.add(genWithKeyMap("alg_recsys_feature_cf_i2i_new_v2", vid, ImmutableMap.of("vid_a", headVid, "vid_b", vid)));
+
+            // ********************* new vid ******************
+            protos.add(genWithKeyMap("alg_vid_feature_day", vid, ImmutableMap.of("vid", vid)));
+            protos.add(genWithKeyMap("alg_sence_type_feature", vid, ImmutableMap.of("sence_type", sceneType, "videoid", vid)));
+            protos.add(genWithKeyMap("alg_videoid_feature", vid, ImmutableMap.of("videoid", vid)));
+            // protos.add(genWithKeyMap("alg_recsys_feature_cf_i2i_scene_rov", vid, ImmutableMap.of("sence_type", i2iSceneType, "vid_a", headVid, "vid_b", vid)));
+            // protos.add(genWithKeyMap("alg_recsys_feature_cf_i2i_scene_ros", vid, ImmutableMap.of("sence_type", i2iSceneType, "vid_a", headVid, "vid_b", vid)));
+            // protos.add(genWithKeyMap("alg_recsys_feature_weak_cf_i2i_scene_rov", vid, ImmutableMap.of("sence_type", i2iSceneType, "vid_a", headVid, "vid_b", vid)));
+            // protos.add(genWithKeyMap("alg_recsys_feature_weak_cf_i2i_scene_ros", vid, ImmutableMap.of("sence_type", i2iSceneType, "vid_a", headVid, "vid_b", vid)));
+            if (null != videoBaseInfoMap && videoBaseInfoMap.containsKey(vid)) {
+                Map<String, Map<String, String>> videoInfo = videoBaseInfoMap.get(vid);
+                if (null != videoInfo && videoInfo.containsKey("alg_vid_feature_basic_info")) {
+                    Map<String, String> baseInfo = videoInfo.get("alg_vid_feature_basic_info");
+                    if (null != baseInfo) {
+                        String cate1 = baseInfo.get("cate1_list");
+                        if (null != cate1 && !cate1.isEmpty()) {
+                            protos.add(genWithKeyMap("alg_cate1_feature", vid, ImmutableMap.of("cate1", cate1)));
+                            protos.add(genWithKeyMap("alg_cate1_feature_day", vid, ImmutableMap.of("cate1", cate1)));
+                        }
+                        String cate2 = baseInfo.get("cate2");
+                        if (null != cate2 && !cate2.isEmpty()) {
+                            protos.add(genWithKeyMap("alg_cate2_feature", vid, ImmutableMap.of("cate2", cate2)));
+                            protos.add(genWithKeyMap("alg_cate2_feature_day", vid, ImmutableMap.of("cate2", cate2)));
+                        }
+                        String vidSource = baseInfo.get("vid_source");
+                        if (null != vidSource && !vidSource.isEmpty()) {
+                            protos.add(genWithKeyMap("alg_vid_source_feature", vid, ImmutableMap.of("vid_source", vidSource)));
+                            protos.add(genWithKeyMap("alg_video_source_feature_day", vid, ImmutableMap.of("video_source", vidSource)));
+                        }
+                        String videoUnionid = baseInfo.get("title_time_w_h_unionid");
+                        if (null != videoUnionid && !videoUnionid.isEmpty()) {
+                            protos.add(genWithKeyMap("alg_video_unionid_feature_day", vid, ImmutableMap.of("video_unionid", videoUnionid)));
+                        }
+                    }
+                }
+            }
+        }
+
+        // user
+        protos.add(genWithMid("alg_mid_feature_play", mid));
+        protos.add(genWithMid("alg_mid_feature_share_and_return", mid));
+        protos.add(genWithMid("alg_mid_feature_play_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_return_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_share_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_feed_exp_share_tags_v2", mid));
+        protos.add(genWithMid("alg_mid_feature_feed_exp_return_tags_v2", mid));
+        protos.add(genWithMid("alg_mid_feature_sharecf", mid));
+        protos.add(genWithMid("alg_mid_feature_returncf", mid));
+
+        return getFeatureByProto(protos);
+    }
+
+    public Feature getFeatureV3(RankParam param, Map<String, Map<String, Map<String, String>>> videoBaseInfoMap, List<String> vidList) {
+        String mid = param.getMid();
+        String mergeMid = getMergeMid(param.getUid(), mid);
+        String headVid = String.valueOf(param.getHeadVid());
+        String province = param.getProvince().replaceAll("省$", "");
+        String apptype = param.getAppType() + "";
+        String orgHotScene = String.valueOf(param.getHotSceneType());
+        String brand = "";
+        if (null != param.getMachineInfo()) {
+            MachineInfo machineInfo = param.getMachineInfo();
+            if (null != machineInfo.getBrand()) {
+                brand = machineInfo.getBrand();
+            }
+        }
+        String hotSceneType = orgHotScene;
+        if (!hotSceneTypeSet.contains(hotSceneType)) {
+            hotSceneType = "other";
+        }
+        String senceType = orgHotScene;
+        if (!hotSceneSet.contains(senceType)) {
+            senceType = "other";
+        }
+
+        List<FeatureKeyProto> protos = new ArrayList<>();
+        for (String vid : vidList) {
+            // ********************* old vid ******************
+            protos.add(genWithVid("alg_vid_feature_all_exp_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_all_share", vid));
+            protos.add(genWithVid("alg_vid_feature_all_return", vid));
+            protos.add(genWithVid("alg_vid_feature_exp2share_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_share2return", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_exp_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_root_share_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_root_return_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_exp_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_root_share_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_root_return_v2", vid));
+
+            // vid + province
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_exp_v2", vid, province));
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_root_share_v2", vid, province));
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_root_return_v2", vid, province));
+
+            // ********************* new vid ******************
+            protos.add(genWithKeyMap("alg_recsys_feature_video_clean_stat", vid, ImmutableMap.of("vid", vid)));
+            protos.add(genWithKeyMap("alg_vid_global_feature_20250212", vid, ImmutableMap.of("vid", vid)));
+            protos.add(genWithKeyMap("alg_vid_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid)));
+            protos.add(genWithKeyMap("alg_vid_recommend_flowpool_exp_feature_20250212", vid, ImmutableMap.of("vid", vid)));
+            // protos.add(genWithKeyMap("alg_vid_long_period_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid)));
+            protos.add(genWithKeyMap("alg_vid_apptype_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid, "apptype", apptype)));
+            protos.add(genWithKeyMap("alg_vid_province_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid, "province", province)));
+            protos.add(genWithKeyMap("alg_vid_brand_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid, "brand", brand)));
+            protos.add(genWithKeyMap("alg_vid_hotsencetype_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid, "hotsencetype", hotSceneType)));
+
+            protos.add(genWithKeyMap("scene_type_vid_cf_feature_20250212", vid, ImmutableMap.of("sence_type", senceType, "vid_a", headVid, "vid_b", vid)));
+            protos.add(genWithKeyMap("vid_click_cf_feature_20250212", vid, ImmutableMap.of("vid_a", headVid, "vid_b", vid)));
+            protos.add(genWithKeyMap("alg_recsys_feature_cf_i2i_v2", vid, ImmutableMap.of("vid_a", headVid, "vid_b", vid)));
+            if (null != videoBaseInfoMap) {
+                Map<String, Map<String, String>> videoInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>());
+                Map<String, String> baseInfo = videoInfo.getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+                String merge_cate1 = baseInfo.getOrDefault("merge_first_level_cate", "unknown").trim();
+                protos.add(genWithKeyMap("alg_merge_cate1_recommend_exp_feature_20250212", vid, ImmutableMap.of("merge_cate1", merge_cate1)));
+                // 特殊情况
+                //protos.add(genWithKeyMap("mid_merge_cate1_feature_20250212", vid, ImmutableMap.of("mid", mid, "merge_cate1", merge_cate1)));
+
+                String merge_cate2 = baseInfo.getOrDefault("merge_second_level_cate", "unknown").trim();
+                protos.add(genWithKeyMap("alg_merge_cate2_recommend_exp_feature_20250212", vid, ImmutableMap.of("merge_cate2", merge_cate2)));
+                // 特殊情况
+                //protos.add(genWithKeyMap("mid_merge_cate2_feature_20250212", vid, ImmutableMap.of("mid", mid, "merge_cate2", merge_cate2)));
+
+                String channel = baseInfo.getOrDefault("channel", "unknown").trim();
+                protos.add(genWithKeyMap("alg_channel_recommend_exp_feature_20250212", vid, ImmutableMap.of("channel", channel)));
+
+                String festive = baseInfo.getOrDefault("festive_label2", "unknown").trim();
+                protos.add(genWithKeyMap("alg_festive_recommend_exp_feature_20250212", vid, ImmutableMap.of("festive", festive)));
+
+                String videoUnionid = baseInfo.getOrDefault("title_time_w_h_unionid", "unknown");
+                protos.add(genWithKeyMap("alg_video_unionid_recommend_exp_feature_20250212", vid, ImmutableMap.of("video_unionid", videoUnionid)));
+            }
+        }
+
+        // user
+        protos.add(genWithMid("alg_mid_feature_play", mid));
+        protos.add(genWithMid("alg_mid_feature_share_and_return", mid));
+        protos.add(genWithMid("alg_mid_feature_play_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_return_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_share_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_feed_exp_share_tags_v2", mid));
+        protos.add(genWithMid("alg_mid_feature_feed_exp_return_tags_v2", mid));
+        protos.add(genWithMid("alg_mid_feature_sharecf", mid));
+        protos.add(genWithMid("alg_mid_feature_returncf", mid));
+
+        // ********************* new mid ******************
+        protos.add(genWithMid("mid_global_feature_20250212", mid));
+        protos.add(genWithMid("mid_u2u_friend_index_feature_20250212", mid));
+        protos.add(genWithMid("alg_recsys_feature_user_share_return_stat", mergeMid));
+
+        return getFeatureByProto(protos);
+    }
+
+    public Feature getFeatureByNewLabel(String appType, String hotSceneType, String province, String brand, String mid, String headVideoId, List<String> vidList, Map<String, Map<String, Map<String, String>>> videoBaseInfoMap) {
+
+        List<FeatureKeyProto> protos = new ArrayList<>();
+        // 视频维度的特征
+        for (String vid : vidList) {
+            protos.add(this.genWithKeyMap("alg_vid_global_feature_20250212", vid, ImmutableMap.of("vid", vid)));
+            protos.add(this.genWithKeyMap("alg_vid_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid)));
+            protos.add(this.genWithKeyMap("alg_vid_recommend_flowpool_exp_feature_20250212", vid, ImmutableMap.of("vid", vid)));
+            protos.add(this.genWithKeyMap("alg_vid_apptype_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid, "apptype", appType)));
+            protos.add(this.genWithKeyMap("alg_vid_province_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid, "province", province)));
+            protos.add(this.genWithKeyMap("alg_vid_brand_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid, "brand", brand)));
+            protos.add(this.genWithKeyMap("alg_vid_hotsencetype_recommend_exp_feature_20250212", vid, ImmutableMap.of("vid", vid, "hotsencetype", hotSceneType)));
+            protos.add(this.genWithKeyMap("scene_type_vid_cf_feature_20250212", vid, ImmutableMap.of("vid_a", headVideoId, "vid_b", vid, "sence_type", hotSceneType)));
+            protos.add(this.genWithKeyMap("vid_click_cf_feature_20250212", vid, ImmutableMap.of("vid_a", headVideoId, "vid_b", vid)));
+            protos.add(this.genWithKeyMap("alg_recsys_feature_cf_i2i_v2", vid, ImmutableMap.of("vid_a", headVideoId, "vid_b", vid)));
+
+            if (MapUtils.isNotEmpty(videoBaseInfoMap)) {
+                Map<String, Map<String, String>> baseInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>());
+                Map<String, String> featureMap = baseInfo.getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+                protos.add(this.genWithKeyMap("alg_channel_recommend_exp_feature_20250212", vid, ImmutableMap.of("channel", featureMap.getOrDefault("channel", "unknown").trim())));
+                protos.add(this.genWithKeyMap("alg_merge_cate1_recommend_exp_feature_20250212", vid, ImmutableMap.of("merge_cate1", featureMap.getOrDefault("merge_first_level_cate", "unknown").trim())));
+                protos.add(this.genWithKeyMap("alg_merge_cate2_recommend_exp_feature_20250212", vid, ImmutableMap.of("merge_cate2", featureMap.getOrDefault("merge_second_level_cate", "unknown").trim())));
+                protos.add(this.genWithKeyMap("alg_video_unionid_recommend_exp_feature_20250212", vid, ImmutableMap.of("video_unionid", featureMap.getOrDefault("title_time_w_h_unionid", "unknown"))));
+
+                protos.add(this.genWithKeyMap("mid_merge_cate1_feature_20250212", vid, ImmutableMap.of("mid", mid, "merge_cate1", featureMap.getOrDefault("merge_first_level_cate", "unknown").trim())));
+                protos.add(this.genWithKeyMap("mid_merge_cate2_feature_20250212", vid, ImmutableMap.of("mid", mid, "merge_cate2", featureMap.getOrDefault("merge_second_level_cate", "unknown").trim())));
+
+            }
+        }
+
+
+        // 用户维度特征
+        protos.add(this.genWithMid("mid_global_feature_20250212", mid));
+        protos.add(this.genWithMid("mid_u2u_friend_index_feature_20250212", mid));
+        protos.add(this.genWithMid("alg_mid_feature_return_tags", mid));
+        protos.add(this.genWithMid("alg_mid_feature_share_tags", mid));
+        protos.add(this.genWithMid("alg_mid_feature_sharecf", mid));
+        protos.add(this.genWithMid("alg_mid_feature_returncf", mid));
+
+        return this.getFeatureByProto(protos);
+    }
+
+    public UserShareReturnProfile getUserProfile(String uid, String mid) {
+        try {
+            String mergeMid = getMergeMid(uid, mid);
+            String table = "alg_recsys_feature_user_share_return_stat";
+            List<FeatureKeyProto> protos = new ArrayList<>();
+            protos.add(genWithMid(table, mergeMid));
+            Feature feature = getFeatureByProto(protos);
+            Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+            if (null != userFeature) {
+                Map<String, String> c9 = userFeature.get(table);
+                if (null != c9 && !c9.isEmpty()) {
+                    String c9Str = JSONUtils.toJson(c9);
+                    if (!c9Str.isEmpty()) {
+                        return JSON.parseObject(c9Str, UserShareReturnProfile.class);
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("get user profile error! value=[{}]", mid, e);
+        }
+        return null;
+    }
+
+    public Map<String, String> getUserInfo(String table, String id) {
+        try {
+            Feature feature = getFeatureByProto(Collections.singletonList(genWithMid(table, id)));
+            Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+            if (null != userFeature) {
+                return userFeature.get(table);
+            }
+        } catch (Exception e) {
+            log.error("get user info error! value=[{}]", id, e);
+        }
+        return null;
+    }
+
+    public Map<String, String> getHeadVideoInfo(String headVid) {
+        try {
+            if (null != headVid && !headVid.isEmpty()) {
+                Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = getVideoBaseInfo(headVid, new ArrayList<>());
+                return videoBaseInfoMap.getOrDefault(headVid, new HashMap<>()).get("alg_vid_feature_basic_info");
+            }
+        } catch (Exception e) {
+            log.error("get head info error! value=[{}]", headVid, e);
+        }
+        return null;
+    }
+
+    private Feature getFeatureByProto(List<FeatureKeyProto> protos) {
+        Map<String, String> result = remoteService.getFeature(protos);
+        Feature feature = new Feature();
+        result.entrySet().forEach(e -> {
+            String[] uk = StringUtils.split(e.getKey(), ":");
+            String prefix = uk[0];
+            String table = uk[1];
+            Map<String, String> colMap = JSONUtils.fromJson(e.getValue(), new TypeToken<Map<String, String>>() {
+            }, Collections.emptyMap());
+
+            String featureStr = colMap.get("feature");
+            switch (prefix) {
+                case "v":
+                    String vid = uk[2];
+                    Map<String, Map<String, String>> tableFeatureMap = feature.getVideoFeature().getOrDefault(vid, new HashMap<>());
+                    tableFeatureMap.put(table, JSONUtils.fromJson(featureStr, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    feature.getVideoFeature().put(vid, tableFeatureMap);
+                    break;
+                case "u":
+                    feature.getUserFeature().put(table, JSONUtils.fromJson(featureStr, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    break;
+                default:
+                    break;
+            }
+        });
+        return feature;
+    }
+
+    private String getMergeMid(String uid, String mid) {
+        if (null != uid && !uid.isEmpty() && !uid.equals("null")) {
+            return uid;
+        } else {
+            return mid;
+        }
+    }
+
     private final String videoUkFormat = "v:%s:%s";
     private final String userUkFormat = "u:%s";
+    private final String kvUkFormat = "v:%s:%s";
 
     private FeatureKeyProto genWithVid(String table, String vid) {
         return FeatureKeyProto.newBuilder()
@@ -166,6 +497,16 @@ public class FeatureService {
                 .build();
     }
 
+    private FeatureKeyProto genWithKeyMap(String table, String uniqKey, Map<String, String> map) {
+        FeatureKeyProto.Builder builder = FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(kvUkFormat, table, uniqKey))
+                .setTableName(table);
+        for (Map.Entry<String, String> entry : map.entrySet()) {
+            builder.putFieldValue(entry.getKey(), entry.getValue());
+        }
+        return builder.build();
+    }
+
     @Data
     public static class Feature {
         // k1:视频、k2:表、k3:特征、v:特征值

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -3,15 +3,15 @@ package com.tzld.piaoquan.recommend.server.service;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.base.Stopwatch;
 import com.google.common.base.Strings;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.Constant;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.tzld.piaoquan.recommend.server.common.RedisKeyConstants;
 import com.tzld.piaoquan.recommend.server.gen.common.Result;
 import com.tzld.piaoquan.recommend.server.gen.recommend.*;
 import com.tzld.piaoquan.recommend.server.model.MachineInfo;
 import com.tzld.piaoquan.recommend.server.model.RecommendParam;
 import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.remote.ABTestRemoteService;
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConfigService;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
@@ -23,12 +23,10 @@ import com.tzld.piaoquan.recommend.server.service.recall.RecallService;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import com.tzld.piaoquan.recommend.server.util.ParserUtils;
 import com.tzld.piaoquan.recommend.server.util.TraceUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.RandomUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -53,31 +51,8 @@ public class RecommendService {
     @Qualifier("redisTemplate")
     private RedisTemplate<String, String> redisTemplate;
 
-
-    private final Map<Integer, String> ab_initial_config_map = new HashMap<>();
-
-    @ApolloJsonValue("${ab_exp_code:{}}")
-    private Map<String, Map<String, String>> abExpCodeMap;
-
-    @ApolloJsonValue("${app_ab_code:{}}")
-    private Map<String, Map<String, Map<String, String>>> appAbCodeMap;
-
-    @ApolloJsonValue("${flow.pool.ids:[7, 8, 9, 10, 11, 12, 13, 14, 15, 16]}")
-    private List<Integer> flowPoolIds;
     @ApolloJsonValue("${city_code:[]}")
     private Set<String> cityCodes;
-
-    @ApolloJsonValue("${new.exp.list:[6]}")
-    private Set<Integer> newExpList;
-
-    @ApolloJsonValue("${new.exp,config.v2:[]}")
-    private List<ExpConfig> expConfigs;
-
-    @Value("${new.exp.abtest.service.switch:true}")
-    private boolean newExpABTestServiceSwitch;
-
-    @Autowired
-    private FlowPoolConfigService flowPoolConfigService;
     @Autowired
     private RecallService recallService;
     @Autowired
@@ -91,13 +66,34 @@ public class RecommendService {
     @Autowired
     private TimerLogService timerLogService;
 
-    @Autowired
-    private ABTestRemoteService abTestRemoteService;
-
     private ThreadLocal<Map<String, Object>> timerLogMapTL = ThreadLocal.withInitial(HashMap::new);
 
+    @Value("${risk.video.filter.switch:true}")
+    private boolean riskVideoFilterSwitch;
+    @ApolloJsonValue("${risk.scene:[]}")
+    private Set<Long> riskScenes;
+
+    private LoadingCache<String, Set<String>> riskUserCache = CacheBuilder.newBuilder()
+            .maximumSize(5)
+            .refreshAfterWrite(300, TimeUnit.SECONDS)
+            .expireAfterWrite(300, TimeUnit.SECONDS)
+            .expireAfterAccess(300, TimeUnit.SECONDS)
+            .build(new CacheLoader<String, Set<String>>() {
+                @Override
+                public Set<String> load(String key) {
+                    Set<String> result = redisTemplate.opsForSet().members(key);
+                    if (CollectionUtils.isEmpty(result)) {
+                        return Collections.emptySet();
+                    }
+                    return result;
+                }
+            });
+
     @PostConstruct
     public void init() {
+        Set<String> data1 = riskUserCache.getUnchecked(RedisKeyConstants.Recommend.riskUserUid);
+        Set<String> data2 = riskUserCache.getUnchecked(RedisKeyConstants.Recommend.riskUserMid);
+        log.info("riskUserCache uid size {}  mid size {}", data1.size(), data2.size());
     }
 
     public RecommendResponse homepageRecommend(RecommendRequest request) {
@@ -158,7 +154,6 @@ public class RecommendService {
                 vps.add(VideoProto.newBuilder()
                         .setPosition(i + 1)
                         .setPushFrom(Strings.nullToEmpty(videos.get(i).getPushFrom()))
-                        .setAbCode(Strings.nullToEmpty(videos.get(i).getAbCode()))
                         .setVideoId(videos.get(i).getVideoId())
                         .setRovScore(videos.get(i).getRovScore())
                         .setSortScore(videos.get(i).getSortScore())
@@ -213,7 +208,6 @@ public class RecommendService {
 
                 map.put("flowPool", String.valueOf(v.getFlowPool()));
                 map.put("level", String.valueOf(v.getLevel()));
-                map.put("flowPoolAbtestGroup", String.valueOf(v.getFlowPoolAbtestGroup()));
                 map.put("inFlowPool", String.valueOf(v.isInFlowPool()));
 
                 map.put("scoreRos", String.valueOf(v.getScoreRos()));
@@ -283,7 +277,6 @@ public class RecommendService {
                 .setResult(Result.newBuilder().setCode(1).setMessage("success"))
                 .addAllVideo(CommonCollectionUtils.toList(videos, v -> VideoProto.newBuilder()
                         .setPushFrom(Strings.nullToEmpty(v.getPushFrom()))
-                        .setAbCode(Strings.nullToEmpty(v.getAbCode()))
                         .setVideoId(v.getVideoId())
                         .setRovScore(v.getRovScore())
                         .build()))
@@ -305,11 +298,8 @@ public class RecommendService {
         param.setCity(request.getCity());
 
 
-        String abInitialConfig = ab_initial_config_map.getOrDefault(request.getAppType(), "095-1");
-        Map<String, String> abExpCode = abExpCodeMap.getOrDefault(abInitialConfig, Collections.emptyMap());
-        param.setAbCode(abExpCode.get("ab_code"));
-        param.setRuleKey(abExpCode.get("rule_key"));
-        param.setDataKey(abExpCode.get("data_key"));
+        param.setRuleKey("rule66");
+        param.setDataKey("data66");
         param.setRankKeyPrefix("rank:score1:");
 
         // 实验配置
@@ -330,111 +320,16 @@ public class RecommendService {
             } else if (abExpCodes.contains("339")) {
                 param.setFlowPoolP(0);
             }
-
-            for (Map.Entry<String, Map<String, String>> entry : abExpCodeMap.entrySet()) {
-                if (abExpCodes.contains(entry.getKey())) {
-                    param.setAbCode(entry.getValue().get("ab_code"));
-                    param.setRuleKey(entry.getValue().get("rule_key"));
-                    param.setDataKey(entry.getValue().get("data_key"));
-
-                    param.setHDataKey(entry.getValue().get("h_data_key"));
-                    param.setHRuleKey(entry.getValue().get("h_rule_key"));
-
-                    param.setRankKeyPrefix(StringUtils.isNotBlank(entry.getValue().get("rank_key_prefix"))
-                            ? entry.getValue().get("rank_key_prefix")
-                            : "rank:score1:");
-                    break;
-                }
-            }
-        }
-
-        // app实验组
-        if (request.getEventIdCount() > 0) {
-            if (recommendType == 0) {
-                String eventId = request.getEventIdMap().get("10003");
-                Map<String, Map<String, String>> appAbCode = appAbCodeMap.get("10003");
-                if (StringUtils.isNotBlank(eventId)
-                        && appAbCode != null
-                        && appAbCode.containsKey(eventId)) {
-                    param.setAbCode(appAbCode.get(eventId).get("ab_code"));
-                    param.setRuleKey(appAbCode.get(eventId).get("rule_key"));
-                    param.setDataKey(appAbCode.get(eventId).get("data_key"));
-                }
-            }
-        }
-
-        // 流量池分发实验组划分
-        int flowPoolIdChoice = flowPoolIds.get(RandomUtils.nextInt(0, flowPoolIds.size()));
-        Map<String, List<Integer>> flowPoolConfig = flowPoolConfigService.getFlowPoolConfig();
-        Set<String> abExpCodes = param.getAbExpCodes();
-        if (Objects.nonNull(abExpCodes) && abExpCodes.contains(Constant.SUPPLY_AB_CODE_ID)) {
-//        if (Objects.equals(Constant.SUPPLY_AB_CODE, param.getAbCode())) {
-            List<Integer> supplyFlowPoolIdList = flowPoolConfig.get(FlowPoolConstants.SUPPLY_FLOW_SET_LEVEL);
-            if (Objects.nonNull(supplyFlowPoolIdList) && !supplyFlowPoolIdList.isEmpty()) {
-                flowPoolIdChoice = supplyFlowPoolIdList.get(0);
-                param.setFlowPoolId(flowPoolIdChoice);
-                param.setFlowPoolAbtestGroup(FlowPoolConstants.SUPPLY_FLOW_SET_LEVEL);
-            }
-        } else {
-            param.setFlowPoolId(flowPoolIdChoice);
-            param.setFlowPoolAbtestGroup("control_group");
-            for (Map.Entry<String, List<Integer>> entry : flowPoolConfig.entrySet()) {
-                if (entry.getValue().contains(flowPoolIdChoice)) {
-                    param.setFlowPoolAbtestGroup(entry.getKey());
-                }
-            }
         }
-        // @desc 新的流量池分发实验组划分，每个尾号不同策略分组。 @time 20240318 @author 张博
-        int lastDigit = RandomUtils.nextInt(0, 10);
-        param.setLastDigit(lastDigit);
 
         // 风险过滤
-        List<String> keysRisk = new ArrayList<>();
-        keysRisk.add("RISK_SHIELD_FILTER_RULE_V1_JSON");
-        keysRisk.add("RISK_SHIELD_FILTER_VIDEO_V1_STR");
-        keysRisk.add("RISK_SHIELD_FILTER_EXPANSION_FACTOR_INT");
-        keysRisk.add("RISK_SHIELD_FILTER_FLAG_BOOL");
-        List<String> valuesRisk = redisTemplate.opsForValue().multiGet(keysRisk);
-        String RISK_SHIELD_FILTER_RULE_V1_JSON = "";
-        String RISK_SHIELD_FILTER_VIDEO_V1_STR = "";
-        String RISK_SHIELD_FILTER_EXPANSION_FACTOR_INT = "";
-        String RISK_SHIELD_FILTER_FLAG_BOOL = "";
-        if (valuesRisk != null && valuesRisk.size() == 4) {
-            RISK_SHIELD_FILTER_RULE_V1_JSON = valuesRisk.get(0);
-            RISK_SHIELD_FILTER_VIDEO_V1_STR = valuesRisk.get(1);
-            RISK_SHIELD_FILTER_EXPANSION_FACTOR_INT = valuesRisk.get(2);
-            RISK_SHIELD_FILTER_FLAG_BOOL = valuesRisk.get(3);
-        } else {
-            log.error("risk rule read redis is wrong");
-        }
-        Map<Integer, List<String>> appRegionFiltered = ParserUtils.parseJsonForRiskRule(RISK_SHIELD_FILTER_RULE_V1_JSON);
-        List<Long> videosWithRisk = ParserUtils.parseJsonForRiskVideos(RISK_SHIELD_FILTER_VIDEO_V1_STR);
-        int expansionFactor = 5;
-        if (RISK_SHIELD_FILTER_EXPANSION_FACTOR_INT != null) {
-            try {
-                expansionFactor = Integer.parseInt(RISK_SHIELD_FILTER_EXPANSION_FACTOR_INT);
-            } catch (NumberFormatException e) {
-                expansionFactor = 5;
-            }
-        }
-        // 容灾
-        if (expansionFactor < 5) {
-            expansionFactor = 5;
-        } else if (expansionFactor > 25) {
-            expansionFactor = 25;
+        if (riskVideoFilterSwitch) {
+            boolean riskUser = riskScenes.contains(request.getHotSceneType())
+                    || riskUserCache.getUnchecked(RedisKeyConstants.Recommend.riskUserUid).contains(param.getUid())
+                    || riskUserCache.getUnchecked(RedisKeyConstants.Recommend.riskUserMid).contains(param.getMid());
+            param.setRiskUser(riskUser);
         }
-        Boolean riskFilterFlag = false;
-        if (RISK_SHIELD_FILTER_FLAG_BOOL != null) {
-            try {
-                riskFilterFlag = Boolean.parseBoolean(RISK_SHIELD_FILTER_FLAG_BOOL.toLowerCase());
-            } catch (Exception e) {
-                riskFilterFlag = false;
-            }
-        }
-        param.setAppRegionFiltered(appRegionFiltered);
-        param.setVideosWithRisk(videosWithRisk);
-        param.setExpansionFactor(expansionFactor);
-        param.setRiskFilterFlag(riskFilterFlag);
+
 
         if (request.hasMachineInfo()) {
             MachineInfoProto machineInfoProto = request.getMachineInfo();
@@ -448,64 +343,33 @@ public class RecommendService {
             param.setMachineInfo(machineInfo);
         }
 
-
-        // TODO：hard code 为了快速做AB验证，应该由AB系统支持
-        if (newExpList.contains(param.getAppType())) {
-            if (newExpABTestServiceSwitch) {
-                param.setExpIdMap(abTestRemoteService.getExp(param.getMid(), param.getUid(), param.getAppType(),
-                        request.getNewExpGroup()));
-            } else {
-                Map<String, List<Layer>> layerMap = CommonCollectionUtils.toMap(expConfigs, c -> c.getAppType(),
-                        c -> c.getLayers());
-                if (MapUtils.isNotEmpty(layerMap)) {
-                    Map<String, Integer> bucketMap = JSONUtils.fromJson(request.getNewExpGroup(),
-                            new TypeToken<Map<String, Integer>>() {
-                            }, Collections.emptyMap());
-                    if (MapUtils.isNotEmpty(bucketMap)) {
-                        List<Layer> layers = layerMap.get(param.getAppType() + "");
-
-                        if (CollectionUtils.isNotEmpty(layers)) {
-                            Map<String, String> expIdMap = new HashMap<>();
-                            for (Layer layer : layers) {
-                                for (Exp exp : layer.getExps()) {
-                                    if (bucketMap.containsKey(layer.getLayerId())
-                                            && exp.getRange().length == 2
-                                            && exp.getRange()[0] <= bucketMap.get(layer.getLayerId())
-                                            && exp.getRange()[1] >= bucketMap.get(layer.getLayerId())) {
-                                        expIdMap.put(layer.getLayerId(), exp.getExpId());
-                                    }
-                                }
-                            }
-                            param.setExpIdMap(expIdMap);
-                        }
-                    }
-                }
-            }
-        }
         if (StringUtils.isNotBlank(request.getCategoryId()) && StringUtils.isNumeric(request.getCategoryId())) {
             param.setCategoryId(Integer.parseInt(request.getCategoryId()));
         }
 
         param.setHotSceneType(request.getHotSceneType());
         param.setClientIp(request.getClientIp());
-
+        param.setVersionCode(request.getVersionCode());
+        param.setRootSourceId(request.getRootSourceId());
+        param.setUserShareDepth(request.getUserShareDepth());
+        param.setPageNum(request.getPageNum());
         return param;
     }
 
     private List<Video> videoRecommend(RecommendParam param) {
         Stopwatch stopwatch = Stopwatch.createStarted();
         RecallResult recallResult = recallService.recall(convertToRecallParam(param));
-        
+
         long recallTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
         timerLogMapTL.get().put("recallTime", recallTime);
-        log.info("recallResult={}, videoRecommend recallResult cost={}", recallResult, recallTime);
+        // log.info("recallResult={}, videoRecommend recallResult cost={}", recallResult, recallTime);
         stopwatch.reset().start();
 
         RankResult rankResult = rankRouter.rank(convertToRankParam(param, recallResult));
 
         long rankTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
         timerLogMapTL.get().put("rankTime", rankTime);
-        log.info("rankResult={}, videoRecommend rank cost={}", rankResult, rankTime);
+        // log.info("rankResult={}, videoRecommend rank cost={}", rankResult, rankTime);
 
 
         if (rankResult == null || CollectionUtils.isEmpty(rankResult.getVideos())) {
@@ -517,14 +381,6 @@ public class RecommendService {
         if (param.getSize() < rankResult.getVideos().size()) {
             videos = rankResult.getVideos().subList(0, param.getSize());
         }
-
-
-        // mark video in flow pool
-        // TODO 业务上有什么用处
-        // 需要被标记的视频满足什么条件
-        // 逻辑已经废弃
-        // markFlowPoolVideo(param, videos);
-
         return videos;
     }
 
@@ -533,28 +389,10 @@ public class RecommendService {
         recallParam.setAppType(param.getAppType());
         // hard code 算法实验配置化之前，复用abcode做AB验证
         // note 避免非实验产品被覆盖
-        recallParam.setAbCode(param.getAbCode());
         recallParam.setRuleKey(param.getRuleKey());
         recallParam.setDataKey(param.getDataKey());
-        recallParam.setHRuleKey(param.getHRuleKey());
-        recallParam.setHDataKey(param.getHDataKey());
-        if (newExpList.contains(param.getAppType())) {
-            recallParam.setAbCode("");
-            if (MapUtils.isNotEmpty(param.getExpIdMap()) && param.getExpIdMap().containsKey("recall")) {
-                String expId = param.getExpIdMap().get("recall");
-                if (abExpCodeMap.containsKey(expId)) {
-                    recallParam.setAbCode(abExpCodeMap.get(expId).get("ab_code"));
-                    recallParam.setRuleKey(abExpCodeMap.get(expId).get("rule_key"));
-                    recallParam.setDataKey(abExpCodeMap.get(expId).get("data_key"));
-                    recallParam.setHDataKey(abExpCodeMap.get(expId).get("h_data_key"));
-                    recallParam.setHRuleKey(abExpCodeMap.get(expId).get("h_rule_key"));
-                }
-            }
-        }
 
         recallParam.setVideoId(param.getVideoId());
-        recallParam.setFlowPoolAbtestGroup(param.getFlowPoolAbtestGroup());
-        recallParam.setLastDigit(param.getLastDigit());
 
         String provinceCode = StringUtils.isNotBlank(param.getProvinceCode())
                 ? param.getProvinceCode()
@@ -566,22 +404,19 @@ public class RecommendService {
         recallParam.setMid(param.getMid());
         recallParam.setSize(param.getSize());
         recallParam.setUid(param.getUid());
-        //风险过滤
-        recallParam.setExpansionFactor(param.getExpansionFactor());
-        recallParam.setRiskFilterFlag(param.getRiskFilterFlag());
-        recallParam.setVideosWithRisk(param.getVideosWithRisk());
-        recallParam.setAppRegionFiltered(param.getAppRegionFiltered());
+        // 风险过滤
+        recallParam.setRiskUser(param.isRiskUser());
         recallParam.setAbExpCodes(param.getAbExpCodes());
 
         recallParam.setProvince(param.getProvince());
-        recallParam.setExpIdMap(param.getExpIdMap());
         recallParam.setCategoryId(param.getCategoryId());
 
         recallParam.setCityCode(param.getCityCode());
-        // recallParam.setAbCode(param.getAbCode());
 
         recallParam.setHotSceneType(param.getHotSceneType());
         recallParam.setClientIp(param.getClientIp());
+        recallParam.setRootSourceId(param.getRootSourceId());
+        recallParam.setUserShareDepth(param.getUserShareDepth());
 
         return recallParam;
     }
@@ -590,72 +425,28 @@ public class RecommendService {
     public RankParam convertToRankParam(RecommendParam param, RecallResult recallResult) {
         RankParam rankParam = new RankParam();
         rankParam.setRecallResult(recallResult);
-        // hard code 算法实验配置化之前，复用abcode做AB验证
-        // note 避免非实验产品被覆盖
-        rankParam.setAbCode(param.getAbCode());
         rankParam.setRankKeyPrefix(param.getRankKeyPrefix());
-        if (newExpList.contains(param.getAppType())) {
-            rankParam.setAbCode("");
-//            if (MapUtils.isNotEmpty(param.getExpIdMap())
-//                    && param.getExpIdMap().containsKey("rank")) {
-//                String expId = param.getExpIdMap().get("rank");
-            // TODO hard code  MVP版本 为了快速验证，对rank强制染色，不做实验编排
-            if (MapUtils.isNotEmpty(param.getExpIdMap())
-                    && param.getExpIdMap().containsKey("recall")) {
-                String expId = param.getExpIdMap().get("recall");
-                if (abExpCodeMap.containsKey(expId)) {
-                    rankParam.setAbCode(abExpCodeMap.get(expId).get("ab_code"));
-                    rankParam.setRankKeyPrefix(StringUtils.isNotBlank(abExpCodeMap.get(expId).get("rank_key_prefix"))
-                            ? abExpCodeMap.get(expId).get("rank_key_prefix")
-                            : "rank:score1:");
-                }
-            }
-        }
 
         rankParam.setSize(param.getSize());
         rankParam.setFlowPoolP(param.getFlowPoolP());
         rankParam.setTopK(param.getTopK());
         rankParam.setAppType(param.getAppType());
         rankParam.setMid(param.getMid());
+        rankParam.setUid(param.getUid());
         rankParam.setProvince(param.getProvince());
         rankParam.setCity(param.getCity());
         rankParam.setMachineInfo(param.getMachineInfo());
         rankParam.setAbExpCodes(param.getAbExpCodes());
-        rankParam.setExpIdMap(param.getExpIdMap());
         rankParam.setCategoryId(param.getCategoryId());
         rankParam.setHeadVid(param.getVideoId());
+        rankParam.setHotSceneType(param.getHotSceneType());
+        rankParam.setVersionCode(param.getVersionCode());
+        rankParam.setRootSourceId(param.getRootSourceId());
+        rankParam.setUserShareDepth(param.getUserShareDepth());
+        rankParam.setPageNum(param.getPageNum());
         return rankParam;
     }
 
-    @Deprecated
-    private void markFlowPoolVideo(RecommendParam param, List<Video> videos) {
-        // TODO
-        for (Video data : videos) {
-            try {
-                Long videoId = data.getVideoId();
-                String quick_flow_pool_isin_flow_pool_key =
-                        String.format("flow:pool:quick:video:ids:%s:3", param.getAppType());
-                String quick_flow_pool_flow_pool_key =
-                        String.format("flow:pool:quick:video:%s:3:%s", param.getAppType(), videoId);
-                if (redisTemplate.opsForSet().isMember(quick_flow_pool_isin_flow_pool_key, String.valueOf(videoId))) {
-                    data.setFlowPool(redisTemplate.opsForSet().randomMember(quick_flow_pool_flow_pool_key));
-                    data.setInFlowPool(true);
-                } else {
-                    String isIn_flow_pool_key =
-                            String.format("flow:pool:video:ids:%s", param.getAppType());
-                    String flow_pool_key =
-                            String.format("flow:pool:video:%s:%s", param.getAppType(), videoId);
-                    if (redisTemplate.opsForSet().isMember(isIn_flow_pool_key, String.valueOf(videoId))) {
-                        data.setFlowPool(redisTemplate.opsForSet().randomMember(flow_pool_key));
-                        data.setInFlowPool(true);
-                    }
-                }
-            } catch (Exception e) {
-                log.error("markFlowPoolVideo error data={}", JSONUtils.toJson(data));
-            }
-        }
-    }
-
     public void updateCache(RecommendRequest request, RecommendParam param, List<Video> videos) {
         if (StringUtils.isBlank(request.getMid())
                 || CollectionUtils.isEmpty(videos)) {
@@ -685,21 +476,7 @@ public class RecommendService {
         if (CollectionUtils.isEmpty(flowPoolVideos)) {
             return;
         }
-
-        switch (param.getFlowPoolAbtestGroup()) {
-            case FlowPoolConstants.SUPPLY_FLOW_SET_LEVEL:
-                flowPoolService.updateSupplyDistributeCountWithLevel(flowPoolVideos);
-                break;
-            case FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL:
-                flowPoolService.updateDistributeCountWithLevel(flowPoolVideos);
-                break;
-            case FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL_SCORE:
-                flowPoolService.updateDistributeCountWithLevelScore(flowPoolVideos);
-                break;
-            default:
-                flowPoolService.updateDistributeCountWithScore(flowPoolVideos);
-                break;
-        }
+        flowPoolService.updateDistributeCountWithLevel(flowPoolVideos);
     }
 
     private void updateLastVideoCache(List<Video> videos) {
@@ -723,88 +500,6 @@ public class RecommendService {
         consumer.accept(Region24HRecallStrategy.PUSH_FORM);
         consumer.accept(RegionRelative24HRecallStrategy.PUSH_FORM);
         consumer.accept(RegionRelative24HDupRecallStrategy.PUSH_FORM);
-        consumer.accept(RegionHWithoutDupRecallStrategy.PUSH_FORM);
-    }
-
-    public static class ExpConfig {
-        private String appType;
-        private List<Layer> layers;
-
-        public String getAppType() {
-            return appType;
-        }
-
-        public void setAppType(String appType) {
-            this.appType = appType;
-        }
-
-        public List<Layer> getLayers() {
-            return layers;
-        }
-
-        public void setLayers(List<Layer> layers) {
-            this.layers = layers;
-        }
-    }
-
-    public static class Layer {
-        private String layerId;
-        private int bucketNum;
-        private List<Exp> exps;
-        private Map<String, String> groupRule;
-
-        public String getLayerId() {
-            return layerId;
-        }
-
-        public void setLayerId(String layerId) {
-            this.layerId = layerId;
-        }
-
-        public int getBucketNum() {
-            return bucketNum;
-        }
-
-        public void setBucketNum(int bucketNum) {
-            this.bucketNum = bucketNum;
-        }
-
-        public List<Exp> getExps() {
-            return exps;
-        }
-
-        public void setExps(List<Exp> exps) {
-            this.exps = exps;
-        }
-
-        public Map<String, String> getGroupRule() {
-            return groupRule;
-        }
-
-        public void setGroupRule(Map<String, String> groupRule) {
-            this.groupRule = groupRule;
-        }
-    }
-
-    public static class Exp {
-        private String expId;
-        private int[] range;
-
-        public String getExpId() {
-            return expId;
-        }
-
-        public void setExpId(String expId) {
-            this.expId = expId;
-        }
-
-        public int[] getRange() {
-            return range;
-        }
-
-        public void setRange(int[] range) {
-            this.range = range;
-        }
     }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/SpringContextHolder.java

@@ -1,33 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service;
-
-import org.springframework.beans.BeansException;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
-import org.springframework.stereotype.Service;
-
-@Service
-public class SpringContextHolder implements ApplicationContextAware {
-
-    private static ApplicationContext applicationContext;
-
-    @Override
-    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
-        SpringContextHolder.applicationContext = applicationContext;
-    }
-
-    public static ApplicationContext getApplicationContext() {
-        return applicationContext;
-    }
-
-    public static Object getBean(String beanName) {
-        return applicationContext.getBean(beanName);
-    }
-
-    public static <T> T getBean(String beanName, Class<T> clazz) {
-        return applicationContext.getBean(beanName, clazz);
-    }
-
-    public static <T> T getBean(Class<T> clazz) {
-        return applicationContext.getBean(clazz);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/VideoRecommendService.java

@@ -1,222 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service;
-
-import com.alibaba.fastjson.JSONObject;
-import com.google.common.base.Stopwatch;
-import com.google.common.base.Strings;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
-import com.tzld.piaoquan.recommend.server.framework.common.User;
-import com.tzld.piaoquan.recommend.server.gen.common.Result;
-import com.tzld.piaoquan.recommend.server.gen.recommend.PushFromIndex;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
-import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendResponse;
-import com.tzld.piaoquan.recommend.server.gen.recommend.VideoProto;
-import com.tzld.piaoquan.recommend.server.implement.FlowPoolRecommendPipeline;
-import com.tzld.piaoquan.recommend.server.implement.TopRecommendPipeline;
-import com.tzld.piaoquan.recommend.server.model.RecommendParam;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.strategy.RankStrategy4RegionMergeModelV547;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.Resource;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author sunxy
- */
-@Service
-@Slf4j
-public class VideoRecommendService {
-
-    @Value("${recommend.new.log.print:true}")
-    private Boolean newLogPrint;
-    @Resource
-    private RecommendService recommendService;
-    @Resource
-    private TopRecommendPipeline topRecommendPipeline;
-    @Resource
-    private FlowPoolRecommendPipeline flowPoolRecommendPipeline;
-    @Resource
-    private RankStrategy4RegionMergeModelV547 rankStrategy4RegionMergeModelV547;
-    @Resource
-    private TimerLogService timerLogService;
-    @Autowired
-    @Qualifier("redisTemplate")
-    private RedisTemplate<String, String> redisTemplate;
-
-    @Value("${recall.framework.exp:code:610}")
-    private String recallFrameworkExpCode;
-
-
-    public RecommendResponse homepageRecommend(RecommendRequest request) {
-        boolean hit = CommonCollectionUtils.contains(request.getAbExpCodeList(), recallFrameworkExpCode);
-        if (hit) {
-            RecommendParam recommendParam = recommendService.genRecommendParam(request, 0);
-            return recommendNew(request, recommendParam, 0);
-        }
-        return recommend(request, 0);
-    }
-
-
-    public RecommendResponse relevantRecommend(RecommendRequest request) {
-        boolean hit = CommonCollectionUtils.contains(request.getAbExpCodeList(), recallFrameworkExpCode);
-        if (hit) {
-            RecommendParam recommendParam = recommendService.genRecommendParam(request, 1);
-            return recommendNew(request, recommendParam, 1);
-        }
-        return recommend(request, 1);
-    }
-
-    private RecommendResponse recommendNew(RecommendRequest request, RecommendParam param, int recommendType) {
-        if (request == null) {
-            return RecommendResponse.newBuilder()
-                    .setResult(Result.newBuilder().setCode(1).setMessage("success"))
-                    .build();
-        }
-        if (request.getVersionAuditStatus() == 1) {
-            return recommendService.specialMidRecommend(request);
-        }
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        if (StringUtils.isNotBlank(request.getMid())
-                && redisTemplate.opsForSet().isMember("special:mid", request.getMid())) {
-            return recommendService.specialMidRecommend(request);
-        }
-        stopwatch.reset().start();
-
-        log.info("genRecommendParam={},genRecommendParam cost={}", JSONUtils.toJson(param),
-                stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
-        stopwatch.reset().start();
-        List<Video> videos = videoRecommend(request, param, recommendType);
-        recommendService.updateCache(request, param, videos);
-
-        // 更新position
-        List<VideoProto> vps = new ArrayList<>();
-        for (int i = 0; i < videos.size(); i++) {
-            Map<String, List<String>> pushFromIndex = videos.get(i).getPushFromIndex();
-            List<PushFromIndex> pushFromIndexList = new ArrayList<>();
-            if (MapUtils.isNotEmpty(pushFromIndex)) {
-                pushFromIndex.forEach((k, v) -> {
-                    PushFromIndex.Builder builder = PushFromIndex.newBuilder().setPushFrom(k);
-                    v.forEach(builder::addIndex);
-                    pushFromIndexList.add(builder.build());
-                });
-            }
-            vps.add(VideoProto.newBuilder()
-                    .setPosition(i + 1)
-                    .setPushFrom(Strings.nullToEmpty(videos.get(i).getPushFrom()))
-                    .setAbCode(Strings.nullToEmpty(videos.get(i).getAbCode()))
-                    .setVideoId(videos.get(i).getVideoId())
-                    .setRovScore(videos.get(i).getRovScore())
-                    .setSortScore(videos.get(i).getSortScore())
-                    .setFlowPool(Strings.nullToEmpty(videos.get(i).getFlowPool()))
-                    .setIsInFlowPool(videos.get(i).isInFlowPool() ? 1 : 0)
-                    .setRand(videos.get(i).getRand())
-                    .addAllPushFromIndex(pushFromIndexList)
-                    .build());
-        }
-
-        return RecommendResponse.newBuilder()
-                .setResult(Result.newBuilder().setCode(1).setMessage("success"))
-                .addAllVideo(vps)
-                .build();
-    }
-
-    private List<Video> videoRecommend(RecommendRequest request, RecommendParam param, int recommendType) {
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        User user = getUser(request);
-        List<Video> topRecommendVideoList = new ArrayList<>();
-        List<Video> flowPoolRecommendVideoList = new ArrayList<>();
-        Map<String, String> timeLogMap = new HashMap<>(32);
-        try {
-            topRecommendVideoList = topRecommendPipeline.feeds(request, 0, user, newLogPrint, timeLogMap);
-            if (newLogPrint) {
-                log.info("traceId = {}, cost = {}, topRecommendVideoList [{}]", request.getRequestId(),
-                        stopwatch.elapsed().toMillis(), JSONObject.toJSONString(topRecommendVideoList));
-            }
-
-        } catch (Exception e) {
-            log.error("traceId = {}, topRecommendVideoList error", request.getRequestId(), e);
-        } finally {
-        }
-        try {
-            stopwatch.reset().start();
-            flowPoolRecommendVideoList = flowPoolRecommendPipeline.feedByRec(request, recommendType, newLogPrint);
-            if (newLogPrint) {
-                log.info("traceId = {}, cost = {}, flowPoolRecommendVideoList [{}]", request.getRequestId(),
-                        stopwatch.elapsed().toMillis(), JSONObject.toJSONString(flowPoolRecommendVideoList));
-            }
-        } catch (Exception e) {
-            log.error("traceId = {}, flowPoolRecommendVideoList error", request.getRequestId(), e);
-        }
-
-        stopwatch.reset().start();
-        RankParam rankParam = recommendService.convertToRankParam(param, null);
-        RankResult rankResult = rankStrategy4RegionMergeModelV547.mergeAndSort(rankParam, topRecommendVideoList,
-                flowPoolRecommendVideoList);
-        if (newLogPrint) {
-            log.info("traceId = {}, cost = {}, final rankResult [{}]", request.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONObject.toJSONString(rankResult));
-        }
-
-        // 只返回size条数据
-        List<Video> videos = rankResult.getVideos();
-        if (param.getSize() < rankResult.getVideos().size()) {
-            videos = rankResult.getVideos().subList(0, param.getSize());
-        }
-
-        // 更新position
-        List<VideoProto> vps = new ArrayList<>();
-        for (int i = 0; i < videos.size(); i++) {
-            Map<String, List<String>> pushFromIndex = videos.get(i).getPushFromIndex();
-            List<PushFromIndex> pushFromIndexList = new ArrayList<>();
-            if (MapUtils.isNotEmpty(pushFromIndex)) {
-                pushFromIndex.forEach((k, v) -> {
-                    PushFromIndex.Builder builder = PushFromIndex.newBuilder().setPushFrom(k);
-                    v.forEach(builder::addIndex);
-                    pushFromIndexList.add(builder.build());
-                });
-            }
-            vps.add(VideoProto.newBuilder()
-                    .setPosition(i + 1)
-                    .setPushFrom(Strings.nullToEmpty(videos.get(i).getPushFrom()))
-                    .setAbCode(Strings.nullToEmpty(videos.get(i).getAbCode()))
-                    .setVideoId(videos.get(i).getVideoId())
-                    .setRovScore(videos.get(i).getRovScore())
-                    .setSortScore(videos.get(i).getSortScore())
-                    .setFlowPool(Strings.nullToEmpty(videos.get(i).getFlowPool()))
-                    .setIsInFlowPool(videos.get(i).isInFlowPool() ? 1 : 0)
-                    .setRand(videos.get(i).getRand())
-                    .addAllPushFromIndex(pushFromIndexList)
-                    .build());
-        }
-        return videos;
-    }
-
-
-    private User getUser(RecommendRequest request) {
-        User user = new User();
-        user.setMid(request.getMid());
-        user.setUid(request.getUid());
-        user.setRegion(request.getProvince());
-        return user;
-    }
-
-    private RecommendResponse recommend(RecommendRequest request, int recommendType) {
-        return recommendService.recommend(request, recommendType);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/ViewedService.java

@@ -1,115 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service;
-
-import com.google.common.collect.Lists;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.HttpClientFactory;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.PostConstruct;
-import java.util.*;
-
-/**
- * @author dyp
- */
-@Service
-@Slf4j
-public class ViewedService {
-    private final Map<Integer, List<Integer>> viewedTypesMap = new HashMap<>();
-    private final List<Integer> defaultViewedTypes = Lists.newArrayList(1, 6, 7, 5, 9);
-
-    @Value("${video.filter.url:}")
-    private String videoFilterUrl;
-    @Value("${securityAbExpCode:}")
-    private String securityAbExpCode;
-
-    private final CloseableHttpClient client = HttpClientFactory.create(1000, 1000, 200, 200, 0, 1000);
-
-
-    @PostConstruct
-    public void init() {
-//        1-已观看 2-视频状态 3-是否进入老年人社区 4-话题状态 5-推荐状态 6-白名单过滤 7-涉政视频过滤
-        viewedTypesMap.put(0, Lists.newArrayList(1, 6, 5, 9));
-        viewedTypesMap.put(4, Lists.newArrayList(1, 6, 5, 9));
-        viewedTypesMap.put(13, Lists.newArrayList(1, 5, 9));
-    }
-
-
-    // TODO 如果过滤失败，那么认为所有视频都被过滤掉
-    public List<Long> filterViewedVideo(int appType, String mid, String uid, List<Long> videoIds, String cityCode,
-                                        Set<String> abExpCodes, Long hotSceneType, String clientIp) {
-
-        List<Integer> viewedTypes = new ArrayList<>(viewedTypesMap.getOrDefault(appType, defaultViewedTypes));
-        if (CommonCollectionUtils.contains(abExpCodes, securityAbExpCode)) {
-            viewedTypes.add(8);
-        }
-        CloseableHttpResponse chr = null;
-        try {
-            HttpPost post = new HttpPost(videoFilterUrl);
-            post.addHeader("Connection", "close");
-            post.addHeader("content-type", "application/json");
-            Map<String, Object> param = new HashMap<>();
-            param.put("appType", appType);
-            param.put("mid", mid);
-            param.put("uid", uid);
-            param.put("types", viewedTypes);
-            param.put("videoIds", videoIds);
-            param.put("cityCode", cityCode);
-            param.put("hotSenceType", hotSceneType);
-            param.put("clientIp", clientIp);
-            param.put("abExpCodes", abExpCodes);
-            List<Integer> recommendStatus = new ArrayList<>();
-            recommendStatus.add(-6);
-            param.put("recommendStatus", recommendStatus);
-            post.setEntity(new StringEntity(JSONUtils.toJson(param)));
-
-            chr = client.execute(post);
-            if (chr == null
-                    || chr.getStatusLine() == null
-                    || chr.getStatusLine().getStatusCode() != 200) {
-                log.error("filterViewedVideo failed http status exception!");
-                return Collections.emptyList();
-            }
-            HttpEntity entity = chr.getEntity();
-            if (entity == null) {
-                return Collections.emptyList();
-            }
-            String content = EntityUtils.toString(entity);
-            Map<String, String> data = JSONUtils.fromJson(content,
-                    new TypeToken<Map<String, String>>() {
-                    },
-                    Collections.emptyMap());
-            if (data.get("code") == null
-                    || !data.get("code").equals("0")) {
-                log.error("filterViewedVideo failed videoFilterUrl return code exception! data={}",
-                        JSONUtils.toJson(data));
-                return Collections.emptyList();
-            }
-            return JSONUtils.fromJson(data.get("data"),
-                    new TypeToken<List<Long>>() {
-                    }, Collections.emptyList());
-
-        } catch (Exception e) {
-            log.error("invoke http filterViewedVideo error", e);
-        } finally {
-            try {
-                if (chr != null) {
-                    chr.close();
-                }
-            } catch (Exception ex) {
-                log.error("close CloseableHttpResponse error", ex);
-            }
-        }
-        return Collections.emptyList();
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/WarmUpService.java

@@ -2,7 +2,6 @@ package com.tzld.piaoquan.recommend.server.service;
 
 import com.tzld.piaoquan.recommend.server.repository.WxVideoStatusRepository;
 import com.tzld.piaoquan.recommend.server.util.SimilarityUtils;
-import com.tzld.piaoquan.recommend.similarity.word2vec.Segment;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.core.annotation.Order;
@@ -35,7 +34,6 @@ public class WarmUpService {
         featureRedisTemplate.opsForValue().get("");
         longVideoRedisTemplate.opsForValue().get("");
         com.tzld.piaoquan.recommend.server.service.score.ScorerUtils.warmUp();
-        com.tzld.piaoquan.recommend.server.framework.score.ScorerUtils.warmUp();
         wxVideoStatusRepository.count();
         SimilarityUtils.init();
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -1,136 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.filter;
-
-import com.google.common.collect.Lists;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
-import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
-import com.tzld.piaoquan.recommend.server.service.filter.strategy.*;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author dyp
- */
-@Slf4j
-public abstract class AbstractFilterService {
-
-    private final ExecutorService pool = ThreadPoolFactory.filterPool();
-
-
-    private List<Long> filterWithRiskVideo(boolean riskFlag,
-                                           int appType,
-                                           String regionCode,
-                                           Map<Integer, List<String>> rules,
-                                           List<Long> videosWithRisk,
-                                           List<Long> videoIds,
-                                           int forceTruncation) {
-        if (!riskFlag) {
-            return this.truncation(videoIds, forceTruncation);
-        }
-        // 1 判断是否过滤,不展示的app+区域列表。
-        boolean filterFlag;
-        if (rules.containsKey(appType)) {
-            filterFlag = rules.get(appType).contains(regionCode);
-        } else {
-            filterFlag = true;
-        }
-        if (!filterFlag) {
-            return this.truncation(videoIds, forceTruncation);
-        }
-        // 2 开始过滤。
-        List<Long> videoIdNew = new ArrayList<>();
-        for (Long videoId : videoIds) {
-            if (!videosWithRisk.contains(videoId)) {
-                videoIdNew.add(videoId);
-            }
-        }
-        return this.truncation(videoIdNew, forceTruncation);
-    }
-
-    private List<Long> truncation(List<Long> videoIds, int forceTruncation) {
-        if (forceTruncation == 0) {
-            return videoIds;
-        } else {
-            return videoIds.subList(0, Math.min(forceTruncation, videoIds.size()));
-        }
-    }
-
-    protected List<Long> viewFilter(FilterParam param) {
-        // hardcode : 风险过滤会做截断，所以先同步调用
-        List<Long> riskVideoIds = filterWithRiskVideo(param.getRiskFilterFlag(),
-                param.getAppType(), param.getRegionCode(), param.getAppRegionFiltered(), param.getVideosWithRisk(),
-                param.getVideoIds(), param.getForceTruncation());
-        param.setVideoIds(riskVideoIds);
-
-        List<FilterStrategy> strategies = getStrategies(param);
-        CountDownLatch cdl = new CountDownLatch(strategies.size());
-        List<Future<List<Long>>> futures = new ArrayList<>();
-        for (final FilterStrategy strategy : strategies) {
-            Future<List<Long>> future = pool.submit(() -> {
-                List<Long> result = strategy.filter(param);
-                cdl.countDown();
-                return result;
-            });
-            futures.add(future);
-        }
-        try {
-            cdl.await(1000, TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            log.error("filter error", e);
-            return null;
-        }
-
-        List<List<Long>> videoIds = new ArrayList<>();
-        for (Future<List<Long>> f : futures) {
-            try {
-                videoIds.add(f.get());
-            } catch (Exception e) {
-                log.error("future get error ", e);
-            }
-        }
-        if (CollectionUtils.isEmpty(videoIds)) {
-            return Collections.emptyList();
-        }
-        List<Long> result = Lists.newArrayList(param.getVideoIds());
-        for (int i = 0; i < videoIds.size(); ++i) {
-            result.retainAll(videoIds.get(i));
-        }
-        return result;
-    }
-
-    private List<FilterStrategy> getStrategies(FilterParam param) {
-        List<FilterStrategy> strategies = new ArrayList<>();
-        strategies.add(ServiceBeanFactory.getBean(SecurityStrategy.class));
-        strategies.add(ServiceBeanFactory.getBean(PreViewedStrategy.class));
-        strategies.add(ServiceBeanFactory.getBean(ViewedStrategy.class));
-        strategies.add(ServiceBeanFactory.getBean(RecommendStatusStrategy.class));
-        strategies.add(ServiceBeanFactory.getBean(SupplyExpStrategy.class));
-        strategies.add(ServiceBeanFactory.getBean(AppletVideoStatusStrategy.class));
-        switch (param.getAppType()) {
-            case 0:
-            case 4:
-                strategies.add(ServiceBeanFactory.getBean(AllowListStrategy.class));
-                break;
-            case 13:
-                break;
-            default:
-                strategies.add(ServiceBeanFactory.getBean(AllowListStrategy.class));
-                strategies.add(ServiceBeanFactory.getBean(TagStrategy.class));
-                break;
-        }
-        if (CollectionUtils.isNotEmpty(param.getAbExpCodes()) && param.getAbExpCodes().contains("697")) {
-            strategies.add(ServiceBeanFactory.getBean(VideoSourceTypeStrategy.class));
-        }
-
-        return strategies;
-    }
-
-}