Merge branch 'master' into vlog_merge_refactor_smz

recommend-server-client/pom.xml

@@ -10,7 +10,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>recommend-server-client</artifactId>
-    <version>1.0.2</version>
+    <version>1.0.3</version>
 
     <dependencies>
         <dependency>

recommend-server-client/src/main/java/com/tzld/piaoquan/recommend/server/gen/recommend/Recommend.java

@@ -51,7 +51,7 @@ public final class Recommend {
       "\n2com/tzld/piaoquan/recommend/server/rec" +
       "ommend.proto\032\031google/protobuf/any.proto\032" +
       "/com/tzld/piaoquan/recommend/server/comm" +
-      "on.proto\"\255\003\n\020RecommendRequest\022\022\n\nrequest" +
+      "on.proto\"\356\003\n\020RecommendRequest\022\022\n\nrequest" +
       "_id\030\001 \001(\t\022\013\n\003mid\030\002 \001(\t\022\013\n\003uid\030\003 \001(\t\022\014\n\004s" +
       "ize\030\004 \001(\005\022\020\n\010app_type\030\005 \001(\005\022\021\n\tcity_code" +
       "\030\006 \001(\t\022\025\n\rprovince_code\030\007 \001(\t\022\023\n\013ab_exp_" +
@@ -60,23 +60,25 @@ public final class Recommend {
       "atus\030\n \001(\005\022\032\n\022recommend_trace_id\030\013 \001(\t\022\020" +
       "\n\010video_id\030\014 \001(\003\022\014\n\004city\030\r \001(\t\022\020\n\010provin" +
       "ce\030\016 \001(\t\022\'\n\014machine_info\030\017 \001(\0132\021.Machine" +
-      "InfoProto\022\025\n\rnew_exp_group\030\020 \001(\t\032.\n\014Even" +
-      "tIdEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001" +
-      "\"\177\n\020MachineInfoProto\022\r\n\005brand\030\001 \001(\t\022\r\n\005m" +
-      "odel\030\002 \001(\t\022\020\n\010platform\030\003 \001(\t\022\023\n\013sdk_vers" +
-      "ion\030\004 \001(\t\022\016\n\006system\030\005 \001(\t\022\026\n\016wechat_vers" +
-      "ion\030\006 \001(\t\"H\n\021RecommendResponse\022\027\n\006result" +
-      "\030\001 \001(\0132\007.Result\022\032\n\005video\030\002 \003(\0132\013.VideoPr" +
-      "oto\"\265\001\n\nVideoProto\022\020\n\010video_id\030\001 \001(\003\022\021\n\t" +
-      "rov_score\030\002 \001(\001\022\021\n\tpush_from\030\003 \001(\t\022\017\n\007ab" +
-      "_code\030\004 \001(\t\022\022\n\nsort_score\030\005 \001(\001\022\020\n\010posit" +
-      "ion\030\006 \001(\005\022\021\n\tflow_pool\030\007 \001(\t\022\027\n\017is_in_fl" +
-      "ow_pool\030\010 \001(\005\022\014\n\004rand\030\t \001(\0012\212\001\n\020Recommen" +
-      "dService\022:\n\021HomepageRecommend\022\021.Recommen" +
-      "dRequest\032\022.RecommendResponse\022:\n\021Relevant" +
-      "Recommend\022\021.RecommendRequest\032\022.Recommend" +
-      "ResponseB7\n0com.tzld.piaoquan.recommend." +
-      "server.gen.recommendP\001\210\001\001b\006proto3"
+      "InfoProto\022\025\n\rnew_exp_group\030\020 \001(\t\022\022\n\nsess" +
+      "ion_id\030\021 \001(\t\022\026\n\016sub_session_id\030\022 \001(\t\022\023\n\013" +
+      "page_source\030\023 \001(\t\032.\n\014EventIdEntry\022\013\n\003key" +
+      "\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\"\177\n\020MachineInfo" +
+      "Proto\022\r\n\005brand\030\001 \001(\t\022\r\n\005model\030\002 \001(\t\022\020\n\010p" +
+      "latform\030\003 \001(\t\022\023\n\013sdk_version\030\004 \001(\t\022\016\n\006sy" +
+      "stem\030\005 \001(\t\022\026\n\016wechat_version\030\006 \001(\t\"H\n\021Re" +
+      "commendResponse\022\027\n\006result\030\001 \001(\0132\007.Result" +
+      "\022\032\n\005video\030\002 \003(\0132\013.VideoProto\"\265\001\n\nVideoPr" +
+      "oto\022\020\n\010video_id\030\001 \001(\003\022\021\n\trov_score\030\002 \001(\001" +
+      "\022\021\n\tpush_from\030\003 \001(\t\022\017\n\007ab_code\030\004 \001(\t\022\022\n\n" +
+      "sort_score\030\005 \001(\001\022\020\n\010position\030\006 \001(\005\022\021\n\tfl" +
+      "ow_pool\030\007 \001(\t\022\027\n\017is_in_flow_pool\030\010 \001(\005\022\014" +
+      "\n\004rand\030\t \001(\0012\212\001\n\020RecommendService\022:\n\021Hom" +
+      "epageRecommend\022\021.RecommendRequest\032\022.Reco" +
+      "mmendResponse\022:\n\021RelevantRecommend\022\021.Rec" +
+      "ommendRequest\032\022.RecommendResponseB7\n0com" +
+      ".tzld.piaoquan.recommend.server.gen.reco" +
+      "mmendP\001\210\001\001b\006proto3"
     };
     descriptor = com.google.protobuf.Descriptors.FileDescriptor
       .internalBuildGeneratedFileFrom(descriptorData,
@@ -89,7 +91,7 @@ public final class Recommend {
     internal_static_RecommendRequest_fieldAccessorTable = new
       com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
         internal_static_RecommendRequest_descriptor,
-        new java.lang.String[] { "RequestId", "Mid", "Uid", "Size", "AppType", "CityCode", "ProvinceCode", "AbExpCode", "EventId", "VersionAuditStatus", "RecommendTraceId", "VideoId", "City", "Province", "MachineInfo", "NewExpGroup", });
+        new java.lang.String[] { "RequestId", "Mid", "Uid", "Size", "AppType", "CityCode", "ProvinceCode", "AbExpCode", "EventId", "VersionAuditStatus", "RecommendTraceId", "VideoId", "City", "Province", "MachineInfo", "NewExpGroup", "SessionId", "SubSessionId", "PageSource", });
     internal_static_RecommendRequest_EventIdEntry_descriptor =
       internal_static_RecommendRequest_descriptor.getNestedTypes().get(0);
     internal_static_RecommendRequest_EventIdEntry_fieldAccessorTable = new

recommend-server-client/src/main/java/com/tzld/piaoquan/recommend/server/gen/recommend/RecommendRequest.java

@@ -26,6 +26,9 @@ private static final long serialVersionUID = 0L;
     city_ = "";
     province_ = "";
     newExpGroup_ = "";
+    sessionId_ = "";
+    subSessionId_ = "";
+    pageSource_ = "";
   }
 
   @java.lang.Override
@@ -168,6 +171,24 @@ private static final long serialVersionUID = 0L;
             newExpGroup_ = s;
             break;
           }
+          case 138: {
+            java.lang.String s = input.readStringRequireUtf8();
+
+            sessionId_ = s;
+            break;
+          }
+          case 146: {
+            java.lang.String s = input.readStringRequireUtf8();
+
+            subSessionId_ = s;
+            break;
+          }
+          case 154: {
+            java.lang.String s = input.readStringRequireUtf8();
+
+            pageSource_ = s;
+            break;
+          }
           default: {
             if (!parseUnknownField(
                 input, unknownFields, extensionRegistry, tag)) {
@@ -755,6 +776,120 @@ private static final long serialVersionUID = 0L;
     }
   }
 
+  public static final int SESSION_ID_FIELD_NUMBER = 17;
+  private volatile java.lang.Object sessionId_;
+  /**
+   * <code>string session_id = 17;</code>
+   * @return The sessionId.
+   */
+  @java.lang.Override
+  public java.lang.String getSessionId() {
+    java.lang.Object ref = sessionId_;
+    if (ref instanceof java.lang.String) {
+      return (java.lang.String) ref;
+    } else {
+      com.google.protobuf.ByteString bs = 
+          (com.google.protobuf.ByteString) ref;
+      java.lang.String s = bs.toStringUtf8();
+      sessionId_ = s;
+      return s;
+    }
+  }
+  /**
+   * <code>string session_id = 17;</code>
+   * @return The bytes for sessionId.
+   */
+  @java.lang.Override
+  public com.google.protobuf.ByteString
+      getSessionIdBytes() {
+    java.lang.Object ref = sessionId_;
+    if (ref instanceof java.lang.String) {
+      com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString.copyFromUtf8(
+              (java.lang.String) ref);
+      sessionId_ = b;
+      return b;
+    } else {
+      return (com.google.protobuf.ByteString) ref;
+    }
+  }
+
+  public static final int SUB_SESSION_ID_FIELD_NUMBER = 18;
+  private volatile java.lang.Object subSessionId_;
+  /**
+   * <code>string sub_session_id = 18;</code>
+   * @return The subSessionId.
+   */
+  @java.lang.Override
+  public java.lang.String getSubSessionId() {
+    java.lang.Object ref = subSessionId_;
+    if (ref instanceof java.lang.String) {
+      return (java.lang.String) ref;
+    } else {
+      com.google.protobuf.ByteString bs = 
+          (com.google.protobuf.ByteString) ref;
+      java.lang.String s = bs.toStringUtf8();
+      subSessionId_ = s;
+      return s;
+    }
+  }
+  /**
+   * <code>string sub_session_id = 18;</code>
+   * @return The bytes for subSessionId.
+   */
+  @java.lang.Override
+  public com.google.protobuf.ByteString
+      getSubSessionIdBytes() {
+    java.lang.Object ref = subSessionId_;
+    if (ref instanceof java.lang.String) {
+      com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString.copyFromUtf8(
+              (java.lang.String) ref);
+      subSessionId_ = b;
+      return b;
+    } else {
+      return (com.google.protobuf.ByteString) ref;
+    }
+  }
+
+  public static final int PAGE_SOURCE_FIELD_NUMBER = 19;
+  private volatile java.lang.Object pageSource_;
+  /**
+   * <code>string page_source = 19;</code>
+   * @return The pageSource.
+   */
+  @java.lang.Override
+  public java.lang.String getPageSource() {
+    java.lang.Object ref = pageSource_;
+    if (ref instanceof java.lang.String) {
+      return (java.lang.String) ref;
+    } else {
+      com.google.protobuf.ByteString bs = 
+          (com.google.protobuf.ByteString) ref;
+      java.lang.String s = bs.toStringUtf8();
+      pageSource_ = s;
+      return s;
+    }
+  }
+  /**
+   * <code>string page_source = 19;</code>
+   * @return The bytes for pageSource.
+   */
+  @java.lang.Override
+  public com.google.protobuf.ByteString
+      getPageSourceBytes() {
+    java.lang.Object ref = pageSource_;
+    if (ref instanceof java.lang.String) {
+      com.google.protobuf.ByteString b = 
+          com.google.protobuf.ByteString.copyFromUtf8(
+              (java.lang.String) ref);
+      pageSource_ = b;
+      return b;
+    } else {
+      return (com.google.protobuf.ByteString) ref;
+    }
+  }
+
   private byte memoizedIsInitialized = -1;
   @java.lang.Override
   public final boolean isInitialized() {
@@ -820,6 +955,15 @@ private static final long serialVersionUID = 0L;
     if (!getNewExpGroupBytes().isEmpty()) {
       com.google.protobuf.GeneratedMessageV3.writeString(output, 16, newExpGroup_);
     }
+    if (!getSessionIdBytes().isEmpty()) {
+      com.google.protobuf.GeneratedMessageV3.writeString(output, 17, sessionId_);
+    }
+    if (!getSubSessionIdBytes().isEmpty()) {
+      com.google.protobuf.GeneratedMessageV3.writeString(output, 18, subSessionId_);
+    }
+    if (!getPageSourceBytes().isEmpty()) {
+      com.google.protobuf.GeneratedMessageV3.writeString(output, 19, pageSource_);
+    }
     unknownFields.writeTo(output);
   }
 
@@ -894,6 +1038,15 @@ private static final long serialVersionUID = 0L;
     if (!getNewExpGroupBytes().isEmpty()) {
       size += com.google.protobuf.GeneratedMessageV3.computeStringSize(16, newExpGroup_);
     }
+    if (!getSessionIdBytes().isEmpty()) {
+      size += com.google.protobuf.GeneratedMessageV3.computeStringSize(17, sessionId_);
+    }
+    if (!getSubSessionIdBytes().isEmpty()) {
+      size += com.google.protobuf.GeneratedMessageV3.computeStringSize(18, subSessionId_);
+    }
+    if (!getPageSourceBytes().isEmpty()) {
+      size += com.google.protobuf.GeneratedMessageV3.computeStringSize(19, pageSource_);
+    }
     size += unknownFields.getSerializedSize();
     memoizedSize = size;
     return size;
@@ -944,6 +1097,12 @@ private static final long serialVersionUID = 0L;
     }
     if (!getNewExpGroup()
         .equals(other.getNewExpGroup())) return false;
+    if (!getSessionId()
+        .equals(other.getSessionId())) return false;
+    if (!getSubSessionId()
+        .equals(other.getSubSessionId())) return false;
+    if (!getPageSource()
+        .equals(other.getPageSource())) return false;
     if (!unknownFields.equals(other.unknownFields)) return false;
     return true;
   }
@@ -994,6 +1153,12 @@ private static final long serialVersionUID = 0L;
     }
     hash = (37 * hash) + NEW_EXP_GROUP_FIELD_NUMBER;
     hash = (53 * hash) + getNewExpGroup().hashCode();
+    hash = (37 * hash) + SESSION_ID_FIELD_NUMBER;
+    hash = (53 * hash) + getSessionId().hashCode();
+    hash = (37 * hash) + SUB_SESSION_ID_FIELD_NUMBER;
+    hash = (53 * hash) + getSubSessionId().hashCode();
+    hash = (37 * hash) + PAGE_SOURCE_FIELD_NUMBER;
+    hash = (53 * hash) + getPageSource().hashCode();
     hash = (29 * hash) + unknownFields.hashCode();
     memoizedHashCode = hash;
     return hash;
@@ -1184,6 +1349,12 @@ private static final long serialVersionUID = 0L;
       }
       newExpGroup_ = "";
 
+      sessionId_ = "";
+
+      subSessionId_ = "";
+
+      pageSource_ = "";
+
       return this;
     }
 
@@ -1236,6 +1407,9 @@ private static final long serialVersionUID = 0L;
         result.machineInfo_ = machineInfoBuilder_.build();
       }
       result.newExpGroup_ = newExpGroup_;
+      result.sessionId_ = sessionId_;
+      result.subSessionId_ = subSessionId_;
+      result.pageSource_ = pageSource_;
       onBuilt();
       return result;
     }
@@ -1347,6 +1521,18 @@ private static final long serialVersionUID = 0L;
         newExpGroup_ = other.newExpGroup_;
         onChanged();
       }
+      if (!other.getSessionId().isEmpty()) {
+        sessionId_ = other.sessionId_;
+        onChanged();
+      }
+      if (!other.getSubSessionId().isEmpty()) {
+        subSessionId_ = other.subSessionId_;
+        onChanged();
+      }
+      if (!other.getPageSource().isEmpty()) {
+        pageSource_ = other.pageSource_;
+        onChanged();
+      }
       this.mergeUnknownFields(other.unknownFields);
       onChanged();
       return this;
@@ -2577,6 +2763,234 @@ private static final long serialVersionUID = 0L;
       onChanged();
       return this;
     }
+
+    private java.lang.Object sessionId_ = "";
+    /**
+     * <code>string session_id = 17;</code>
+     * @return The sessionId.
+     */
+    public java.lang.String getSessionId() {
+      java.lang.Object ref = sessionId_;
+      if (!(ref instanceof java.lang.String)) {
+        com.google.protobuf.ByteString bs =
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        sessionId_ = s;
+        return s;
+      } else {
+        return (java.lang.String) ref;
+      }
+    }
+    /**
+     * <code>string session_id = 17;</code>
+     * @return The bytes for sessionId.
+     */
+    public com.google.protobuf.ByteString
+        getSessionIdBytes() {
+      java.lang.Object ref = sessionId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        sessionId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    /**
+     * <code>string session_id = 17;</code>
+     * @param value The sessionId to set.
+     * @return This builder for chaining.
+     */
+    public Builder setSessionId(
+        java.lang.String value) {
+      if (value == null) {
+    throw new NullPointerException();
+  }
+  
+      sessionId_ = value;
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>string session_id = 17;</code>
+     * @return This builder for chaining.
+     */
+    public Builder clearSessionId() {
+      
+      sessionId_ = getDefaultInstance().getSessionId();
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>string session_id = 17;</code>
+     * @param value The bytes for sessionId to set.
+     * @return This builder for chaining.
+     */
+    public Builder setSessionIdBytes(
+        com.google.protobuf.ByteString value) {
+      if (value == null) {
+    throw new NullPointerException();
+  }
+  checkByteStringIsUtf8(value);
+      
+      sessionId_ = value;
+      onChanged();
+      return this;
+    }
+
+    private java.lang.Object subSessionId_ = "";
+    /**
+     * <code>string sub_session_id = 18;</code>
+     * @return The subSessionId.
+     */
+    public java.lang.String getSubSessionId() {
+      java.lang.Object ref = subSessionId_;
+      if (!(ref instanceof java.lang.String)) {
+        com.google.protobuf.ByteString bs =
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        subSessionId_ = s;
+        return s;
+      } else {
+        return (java.lang.String) ref;
+      }
+    }
+    /**
+     * <code>string sub_session_id = 18;</code>
+     * @return The bytes for subSessionId.
+     */
+    public com.google.protobuf.ByteString
+        getSubSessionIdBytes() {
+      java.lang.Object ref = subSessionId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        subSessionId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    /**
+     * <code>string sub_session_id = 18;</code>
+     * @param value The subSessionId to set.
+     * @return This builder for chaining.
+     */
+    public Builder setSubSessionId(
+        java.lang.String value) {
+      if (value == null) {
+    throw new NullPointerException();
+  }
+  
+      subSessionId_ = value;
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>string sub_session_id = 18;</code>
+     * @return This builder for chaining.
+     */
+    public Builder clearSubSessionId() {
+      
+      subSessionId_ = getDefaultInstance().getSubSessionId();
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>string sub_session_id = 18;</code>
+     * @param value The bytes for subSessionId to set.
+     * @return This builder for chaining.
+     */
+    public Builder setSubSessionIdBytes(
+        com.google.protobuf.ByteString value) {
+      if (value == null) {
+    throw new NullPointerException();
+  }
+  checkByteStringIsUtf8(value);
+      
+      subSessionId_ = value;
+      onChanged();
+      return this;
+    }
+
+    private java.lang.Object pageSource_ = "";
+    /**
+     * <code>string page_source = 19;</code>
+     * @return The pageSource.
+     */
+    public java.lang.String getPageSource() {
+      java.lang.Object ref = pageSource_;
+      if (!(ref instanceof java.lang.String)) {
+        com.google.protobuf.ByteString bs =
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        pageSource_ = s;
+        return s;
+      } else {
+        return (java.lang.String) ref;
+      }
+    }
+    /**
+     * <code>string page_source = 19;</code>
+     * @return The bytes for pageSource.
+     */
+    public com.google.protobuf.ByteString
+        getPageSourceBytes() {
+      java.lang.Object ref = pageSource_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        pageSource_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    /**
+     * <code>string page_source = 19;</code>
+     * @param value The pageSource to set.
+     * @return This builder for chaining.
+     */
+    public Builder setPageSource(
+        java.lang.String value) {
+      if (value == null) {
+    throw new NullPointerException();
+  }
+  
+      pageSource_ = value;
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>string page_source = 19;</code>
+     * @return This builder for chaining.
+     */
+    public Builder clearPageSource() {
+      
+      pageSource_ = getDefaultInstance().getPageSource();
+      onChanged();
+      return this;
+    }
+    /**
+     * <code>string page_source = 19;</code>
+     * @param value The bytes for pageSource to set.
+     * @return This builder for chaining.
+     */
+    public Builder setPageSourceBytes(
+        com.google.protobuf.ByteString value) {
+      if (value == null) {
+    throw new NullPointerException();
+  }
+  checkByteStringIsUtf8(value);
+      
+      pageSource_ = value;
+      onChanged();
+      return this;
+    }
     @java.lang.Override
     public final Builder setUnknownFields(
         final com.google.protobuf.UnknownFieldSet unknownFields) {

recommend-server-client/src/main/java/com/tzld/piaoquan/recommend/server/gen/recommend/RecommendRequestOrBuilder.java

@@ -224,4 +224,40 @@ public interface RecommendRequestOrBuilder extends
    */
   com.google.protobuf.ByteString
       getNewExpGroupBytes();
+
+  /**
+   * <code>string session_id = 17;</code>
+   * @return The sessionId.
+   */
+  java.lang.String getSessionId();
+  /**
+   * <code>string session_id = 17;</code>
+   * @return The bytes for sessionId.
+   */
+  com.google.protobuf.ByteString
+      getSessionIdBytes();
+
+  /**
+   * <code>string sub_session_id = 18;</code>
+   * @return The subSessionId.
+   */
+  java.lang.String getSubSessionId();
+  /**
+   * <code>string sub_session_id = 18;</code>
+   * @return The bytes for subSessionId.
+   */
+  com.google.protobuf.ByteString
+      getSubSessionIdBytes();
+
+  /**
+   * <code>string page_source = 19;</code>
+   * @return The pageSource.
+   */
+  java.lang.String getPageSource();
+  /**
+   * <code>string page_source = 19;</code>
+   * @return The bytes for pageSource.
+   */
+  com.google.protobuf.ByteString
+      getPageSourceBytes();
 }

recommend-server-client/src/main/proto/com/tzld/piaoquan/recommend/server/recommend.proto

@@ -25,6 +25,9 @@ message RecommendRequest {
   string province = 14;
   MachineInfoProto machine_info = 15;
   string new_exp_group = 16;
+  string session_id = 17;
+  string sub_session_id = 18;
+  string page_source = 19;
 }
 
 message MachineInfoProto {

recommend-server-service/pom.xml

@@ -126,6 +126,17 @@
             <groupId>com.aliyun.openservices</groupId>
             <artifactId>aliyun-log-logback-appender</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.aliyun.openservices</groupId>
+            <artifactId>aliyun-log-producer</artifactId>
+            <version>0.3.10</version>
+        </dependency>
+        <dependency>
+            <groupId>com.aliyun.openservices</groupId>
+            <artifactId>aliyun-log</artifactId>
+            <version>0.6.35</version>
+            <classifier>jar-with-dependencies</classifier>
+        </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>
@@ -142,7 +153,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-server-client</artifactId>
-            <version>1.0.2</version>
+            <version>1.0.3</version>
         </dependency>
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/CustomMongoProperties.java

@@ -11,8 +11,11 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
 )
 @Data
 public class CustomMongoProperties {
-    private String host;
-    private Integer port;
+    private String clusterHost;
+    private Integer clusterPort;
+    private String secondaryHost;
+    private Integer secondaryPort;
+
     private String uri;
     private String database;
     private String username;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/FeatureRedisTemplateConfig.java

@@ -31,11 +31,11 @@ public class FeatureRedisTemplateConfig {
 
     @Bean("featureRedisFactory")
     @Primary
-    public LettuceConnectionFactory factory(@Qualifier("featureRedisPool") GenericObjectPoolConfig<LettucePoolingClientConfiguration> tairPool,
-                                            @Qualifier("featureRedisConfig") RedisStandaloneConfiguration tairConfig) {
+    public LettuceConnectionFactory factory(@Qualifier("featureRedisPool") GenericObjectPoolConfig<LettucePoolingClientConfiguration> featureRedisPool,
+                                            @Qualifier("featureRedisConfig") RedisStandaloneConfiguration featureRedisConfig) {
         LettuceClientConfiguration lettuceClientConfiguration =
-                LettucePoolingClientConfiguration.builder().poolConfig(tairPool).build();
-        return new LettuceConnectionFactory(tairConfig, lettuceClientConfiguration);
+                LettucePoolingClientConfiguration.builder().poolConfig(featureRedisPool).build();
+        return new LettuceConnectionFactory(featureRedisConfig, lettuceClientConfiguration);
     }
 
     @Bean(name = "featureRedisTemplate")

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/FilterRedisTemplateConfig.java

@@ -0,0 +1,63 @@
+package com.tzld.piaoquan.recommend.server.config;
+
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.connection.lettuce.LettucePoolingClientConfiguration;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+
+@Configuration
+public class FilterRedisTemplateConfig {
+
+    @Bean("filterRedisPool")
+    @ConfigurationProperties(prefix = "spring.filter-redis.lettuce.pool")
+    public GenericObjectPoolConfig<LettucePoolingClientConfiguration> filterRedisPool() {
+        return new GenericObjectPoolConfig<>();
+    }
+
+    @Bean("filterRedisConfig")
+    @ConfigurationProperties(prefix = "spring.filter-redis")
+    public RedisStandaloneConfiguration filterRedisConfig() {
+        return new RedisStandaloneConfiguration();
+    }
+
+    @Bean("filterRedisFactory")
+    @Primary
+    public LettuceConnectionFactory factory(@Qualifier("filterRedisPool") GenericObjectPoolConfig<LettucePoolingClientConfiguration> filterRedisPool,
+                                            @Qualifier("filterRedisConfig") RedisStandaloneConfiguration filterRedisConfig) {
+        LettuceClientConfiguration lettuceClientConfiguration =
+                LettucePoolingClientConfiguration.builder().poolConfig(filterRedisPool).build();
+        return new LettuceConnectionFactory(filterRedisConfig, lettuceClientConfiguration);
+    }
+
+    @Bean(name = "filterRedisTemplate")
+    public RedisTemplate<String, String> getRedisTemplate(@Qualifier("filterRedisFactory") RedisConnectionFactory factory) {
+        return buildRedisTemplateByString(factory);
+    }
+
+    /**
+     * 构建redisTemplate 使用string序列化
+     *
+     * @param factory
+     * @return
+     */
+    private RedisTemplate<String, String> buildRedisTemplateByString(RedisConnectionFactory factory) {
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(factory);
+        // key的序列化类型 保证可读性
+        redisTemplate.setKeySerializer(new StringRedisSerializer());
+        redisTemplate.setValueSerializer(new StringRedisSerializer());
+        redisTemplate.setHashKeySerializer(new StringRedisSerializer());
+        redisTemplate.setHashValueSerializer(new StringRedisSerializer());
+        return redisTemplate;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/LongVideoRedisTemplateConfig.java

@@ -19,23 +19,23 @@ public class LongVideoRedisTemplateConfig {
 
     @Bean("longVideoRedisPool")
     @ConfigurationProperties(prefix = "spring.long-video-redis.lettuce.pool")
-    public GenericObjectPoolConfig<LettucePoolingClientConfiguration> redisPool() {
+    public GenericObjectPoolConfig<LettucePoolingClientConfiguration> longVideoRedisPool() {
         return new GenericObjectPoolConfig<>();
     }
 
     @Bean("longVideoRedisConfig")
     @ConfigurationProperties(prefix = "spring.long-video-redis")
-    public RedisStandaloneConfiguration tairConfig() {
+    public RedisStandaloneConfiguration longVideoRedisConfig() {
         return new RedisStandaloneConfiguration();
     }
 
     @Bean("longVideoRedisFactory")
     @Primary
-    public LettuceConnectionFactory factory(GenericObjectPoolConfig<LettucePoolingClientConfiguration> redisPool,
-                                            RedisStandaloneConfiguration redisConfig) {
+    public LettuceConnectionFactory factory(@Qualifier("longVideoRedisPool") GenericObjectPoolConfig<LettucePoolingClientConfiguration> longVideoRedisPool,
+                                            @Qualifier("longVideoRedisConfig") RedisStandaloneConfiguration longVideoRedisConfig) {
         LettuceClientConfiguration lettuceClientConfiguration =
-                LettucePoolingClientConfiguration.builder().poolConfig(redisPool).build();
-        return new LettuceConnectionFactory(redisConfig, lettuceClientConfiguration);
+                LettucePoolingClientConfiguration.builder().poolConfig(longVideoRedisPool).build();
+        return new LettuceConnectionFactory(longVideoRedisConfig, lettuceClientConfiguration);
     }
 
     @Bean(name = "longVideoRedisTemplate")

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/MongoTemplateConfig.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.recommend.server.config;
 
+import com.google.common.collect.Lists;
 import com.mongodb.MongoClientSettings;
 import com.mongodb.MongoCredential;
 import com.mongodb.ServerAddress;
@@ -32,8 +33,9 @@ public class MongoTemplateConfig {
         MongoClientSettings.Builder builder = MongoClientSettings.builder();
 
         builder.applyToClusterSettings(settings -> {
-            settings.hosts(singletonList(
-                    new ServerAddress(properties.getHost(), properties.getPort())));
+            settings.hosts(Lists.newArrayList(
+                    new ServerAddress(properties.getClusterHost(), properties.getClusterPort()),
+                    new ServerAddress(properties.getSecondaryHost(), properties.getSecondaryPort())));
         });
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/RedisTemplateConfig.java

@@ -31,8 +31,8 @@ public class RedisTemplateConfig {
 
     @Bean("redisFactory")
     @Primary
-    public LettuceConnectionFactory factory(GenericObjectPoolConfig<LettucePoolingClientConfiguration> redisPool,
-                                            RedisStandaloneConfiguration redisConfig) {
+    public LettuceConnectionFactory factory(@Qualifier("redisPool") GenericObjectPoolConfig<LettucePoolingClientConfiguration> redisPool,
+                                            @Qualifier("redisConfig") RedisStandaloneConfiguration redisConfig) {
         LettuceClientConfiguration lettuceClientConfiguration =
                 LettucePoolingClientConfiguration.builder().poolConfig(redisPool).build();
         return new LettuceConnectionFactory(redisConfig, lettuceClientConfiguration);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/RecommendParam.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.recommend.server.model;
 
+import com.alibaba.fastjson.annotation.JSONField;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -41,7 +42,10 @@ public class RecommendParam {
     private int expansionFactor;
     private Boolean riskFilterFlag;
     private Map<Integer, List<String>> appRegionFiltered;
+
+    @JSONField(serialize = false)
     private List<Long> videosWithRisk;
+
     private String regionCode;
     private MachineInfo machineInfo;
     private String province;
@@ -52,3 +56,4 @@ public class RecommendParam {
     private Map<String, String> expIdMap;
 
 }
+

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -26,6 +26,7 @@ import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import com.tzld.piaoquan.recommend.server.util.ParserUtils;
+import com.tzld.piaoquan.recommend.server.util.TraceUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
@@ -83,6 +84,8 @@ public class RecommendService {
     private PreViewedService preViewedService;
     @Autowired
     private FlowPoolService flowPoolService;
+    @Autowired
+    private StatisticsLogService statisticsLogService;
 
     @PostConstruct
     public void init() {
@@ -115,7 +118,6 @@ public class RecommendService {
                 && redisTemplate.opsForSet().isMember("special:mid", request.getMid())) {
             return specialMidRecommend(request);
         }
-        // log.info("sprcial mid cost={}", stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
         stopwatch.reset().start();
 
         RecommendParam param = genRecommendParam(request, recommendType);
@@ -127,8 +129,8 @@ public class RecommendService {
                 stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
         stopwatch.reset().start();
         updateCache(request, param, videos);
-//        log.info("updateCache={}, updateCache cost={}", JSONUtils.toJson(videos), stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
 
+        logStatisticsInfo(request, param, videos);
         // 更新position
         List<VideoProto> vps = new ArrayList<>();
         for (int i = 0; i < videos.size(); i++) {
@@ -152,6 +154,57 @@ public class RecommendService {
 
     }
 
+    private void logStatisticsInfo(RecommendRequest request, RecommendParam param, List<Video> videos) {
+
+        if (CollectionUtils.isEmpty(videos)) {
+            return;
+        }
+
+        try {
+            List<Map<String, String>> data = videos.stream().map(v -> {
+                Map<String, String> map = new HashMap<>();
+
+                map.put("traceId", String.valueOf(TraceUtils.currentTraceId()));
+
+                // TODO user
+                map.put("sessionId", String.valueOf(request.getSessionId()));
+                map.put("subsessionid", String.valueOf(request.getSubSessionId()));
+                map.put("mid", param.getMid());
+
+                // ab
+                map.put("newexpgroup", String.valueOf(request.getNewExpGroup()));
+
+                // scene
+                map.put("apptype", String.valueOf(request.getAppType()));
+                map.put("pagesource", String.valueOf(request.getPageSource()));
+
+                // video
+                map.put("videoId", String.valueOf(v.getVideoId()));
+                map.put("relevantVideoId", String.valueOf(request.getVideoId()));
+                map.put("rovScore", String.valueOf(v.getRovScore()));
+                map.put("sortScore", String.valueOf(v.getSortScore()));
+                map.put("pushFrom", String.valueOf(v.getPushFrom()));
+
+                map.put("flowPool", String.valueOf(v.getFlowPool()));
+                map.put("level", String.valueOf(v.getLevel()));
+                map.put("flowPoolAbtestGroup", String.valueOf(v.getFlowPoolAbtestGroup()));
+                map.put("inFlowPool", String.valueOf(v.isInFlowPool()));
+
+                map.put("scoreRos", String.valueOf(v.getScoreRos()));
+                map.put("scoreStr", String.valueOf(v.getScoreStr()));
+                map.put("score", String.valueOf(v.getScore()));
+                map.put("scoresMap", JSONUtils.toJson(String.valueOf(v.getScoresMap())));
+
+                return map;
+
+            }).collect(Collectors.toList());
+
+            statisticsLogService.log(data);
+        } catch (Exception e) {
+            log.error("logStatisticsInfo error", e);
+        }
+    }
+
     private RecommendResponse specialMidRecommend(RecommendRequest request) {
         log.info("hit special mid recommend request={}", JSONUtils.toJson(request));
         if (request == null) {
@@ -168,7 +221,6 @@ public class RecommendService {
         recallParam.setProvince(request.getProvince());
 
         RecallResult recallResult = recallService.recall(recallParam);
-        log.info("recallParam {}, recallResult {}", JSONUtils.toJson(recallParam), JSONUtils.toJson(recallResult));
 
         RankParam rankParam = new RankParam();
         rankParam.setRecallResult(recallResult);
@@ -176,7 +228,6 @@ public class RecommendService {
         rankParam.setSpecialRecommend(true);
 
         RankResult rankResult = rankRouter.rank(rankParam);
-        log.info("rankParam, rankResult {}", JSONUtils.toJson(rankParam), JSONUtils.toJson(rankResult));
 
         if (rankResult == null || CollectionUtils.isEmpty(rankResult.getVideos())) {
             return RecommendResponse.newBuilder()
@@ -192,7 +243,6 @@ public class RecommendService {
 
         if (CollectionUtils.isNotEmpty(videos)) {
             Video lastVideo = videos.get(videos.size() - 1);
-            log.info("update recall last video  lastVideo={}", JSONUtils.toJson(lastVideo));
             redisTemplate.opsForValue().set(lastVideo.getLastVideoKey(), String.valueOf(lastVideo.getVideoId()),
                     1, TimeUnit.DAYS);
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/StatisticsLogService.java

@@ -0,0 +1,81 @@
+package com.tzld.piaoquan.recommend.server.service;
+
+import com.aliyun.openservices.aliyun.log.producer.LogProducer;
+import com.aliyun.openservices.aliyun.log.producer.Producer;
+import com.aliyun.openservices.aliyun.log.producer.ProducerConfig;
+import com.aliyun.openservices.aliyun.log.producer.ProjectConfig;
+import com.aliyun.openservices.log.common.LogItem;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Service
+@Slf4j
+public class StatisticsLogService {
+
+    @Value("${aliyun.log.project}")
+    private String project;
+    @Value("${aliyun.log.endpoint}")
+    private String endpoint;
+    @Value("${aliyun.log.accessKeyId}")
+    private String accessKeyId;
+    @Value("${aliyun.log.accessKeySecret}")
+    private String accessKeySecret;
+
+    private String logStore = "statistics-log";
+
+    private Producer producer;
+
+
+    @PostConstruct
+    public void init() {
+        ProducerConfig producerConfig = new ProducerConfig();
+        producer = new LogProducer(producerConfig);
+        producer.putProjectConfig(new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret));
+    }
+
+    public void log(Map<String, String> data) {
+        if (MapUtils.isEmpty(data)) {
+            return;
+        }
+        try {
+            LogItem logItem = new LogItem();
+            data.entrySet().stream().forEach(e -> {
+                logItem.PushBack(e.getKey(), e.getValue());
+            });
+            producer.send(project, logStore, logItem);
+        } catch (InterruptedException e) {
+            log.warn("The current thread has been interrupted during send logs.");
+        } catch (Exception e) {
+            log.error("Failed to send logs", e);
+        }
+    }
+
+    public void log(List<Map<String, String>> data) {
+        if (CollectionUtils.isEmpty(data)) {
+            return;
+        }
+        try {
+            List<LogItem> items = data.stream().map(d -> {
+                LogItem logItem = new LogItem();
+                d.entrySet().stream().forEach(e -> {
+                    logItem.PushBack(e.getKey(), e.getValue());
+                });
+                return logItem;
+            }).collect(Collectors.toList());
+            producer.send(project, logStore, items);
+        } catch (InterruptedException e) {
+            log.warn("The current thread has been interrupted during send logs.");
+        } catch (Exception e) {
+            log.error("Failed to send logs", e);
+        }
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/ViewedService.java

@@ -64,7 +64,7 @@ public class ViewedService {
             param.put("recommendStatus", recommendStatus);
             post.setEntity(new StringEntity(JSONUtils.toJson(param)));
 
-            log.info("request={}", JSONUtils.toJson(param));
+            // log.info("request={}", JSONUtils.toJson(param));
             chr = client.execute(post);
             if (chr == null
                     || chr.getStatusLine() == null
@@ -77,7 +77,7 @@ public class ViewedService {
                 return Collections.emptyList();
             }
             String content = EntityUtils.toString(entity);
-            log.info("response entity={}", JSONUtils.toJson(content));
+            // log.info("response entity={}", JSONUtils.toJson(content));
             Map<String, String> data = JSONUtils.fromJson(content,
                     new TypeToken<Map<String, String>>() {
                     },

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -49,25 +49,25 @@ public abstract class AbstractFilterService {
                 param.getAppType(), param.getRegionCode(), param.getAppRegionFiltered(), param.getVideosWithRisk(),
                 param.getVideoIds(), param.getForceTruncation());
         // log.info("filterByRiskVideos videoIds={}", JSONUtils.toJson(videoIds));
-        JSONObject obj = new JSONObject();
-        obj.put("zhangbotest", "");
-        obj.put("concurrent", param.concurrent);
-        obj.put("notUsePreView", param.notUsePreView);
-        obj.put("size1", videoIds.size());
+//        JSONObject obj = new JSONObject();
+//        obj.put("zhangbotest", "");
+//        obj.put("concurrent", param.concurrent);
+//        obj.put("notUsePreView", param.notUsePreView);
+//        obj.put("size1", videoIds.size());
         if (param.isNotUsePreView()) {
             ;
         } else {
             videoIds = filterByPreViewed(param.getAppType(), param.getMid(), videoIds);
         }
-        obj.put("size2", videoIds.size());
+//        obj.put("size2", videoIds.size());
         // log.info("filterByPreViewed videoIds={}", JSONUtils.toJson(videoIds));
         if (param.isConcurrent()) {
             videoIds = filterByViewedConcurrent(param.getAppType(), param.getMid(), param.getUid(), videoIds);
         } else {
             videoIds = filterByViewed(param.getAppType(), param.getMid(), param.getUid(), videoIds);
         }
-        obj.put("size3", videoIds.size());
-        log.info(obj.toString());
+//        obj.put("size3", videoIds.size());
+//        log.info(obj.toString());
         // log.info("filterByViewed videoIds={}", JSONUtils.toJson(videoIds));
         return videoIds;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/AllowListStrategy.java

@@ -1,5 +1,8 @@
 package com.tzld.piaoquan.recommend.server.service.filter.strategy;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 import com.google.common.collect.Lists;
 import com.google.common.hash.Hashing;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
@@ -12,7 +15,9 @@ import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.PostConstruct;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author dyp
@@ -24,15 +29,39 @@ public class AllowListStrategy implements FilterStrategy {
     @Qualifier("longVideoRedisTemplate")
     private RedisTemplate<String, String> redisTemplate;
 
-    // private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_PREFIX = "movie:videoid:allowlist:";
+    private LoadingCache<Integer, Set<String>> allowVideoCache = CacheBuilder.newBuilder()
+            .maximumSize(1000000) // TODO 评估容量
+            .refreshAfterWrite(600, TimeUnit.SECONDS)
+            .expireAfterWrite(600, TimeUnit.SECONDS)
+            .expireAfterAccess(600, TimeUnit.SECONDS)
+            .build(new CacheLoader<Integer, Set<String>>() {
+                @Override
+                public Set<String> load(Integer idx) {
+                    String key = VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX + idx;
+                    Set<String> result = redisTemplate.opsForSet().members(key);
+                    if (CollectionUtils.isEmpty(result)) {
+                        return Collections.emptySet();
+                    }
+                    return result;
+                }
+            });
+
     private static final String VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX = "movie:videoid:allowSet:";
-    // private static final String VIDEO_ALLOW_LIST_BITMAP_KEY = "movie.store.mp.allowlist.videoid.bitmap";
     private static final String RELIGION_VIDEO_ALLOW_LIST_BITMAP_KEY = "mp:religion:allowlist:videoid:bitmap";
-//    @Value("${movie.videoid.allowlist.compatible:1}")
-//    private Integer mvalCompatible;
+
+
+    @PostConstruct
+    public void init() {
+        // 预加载本地缓存
+        for (int i = 0; i < 100; i++) {
+            int finalI = i;
+            allowVideoCache.getUnchecked(finalI);
+            // ThreadPoolFactory.defaultPool().submit(() -> allowVideoCache.getUnchecked(finalI));
+        }
+        log.info("allowVideoCache size {} ", allowVideoCache.size());
+    }
 
     @Override
-    // TODO 未找到优化方法 暂时保留原代码
     public List<Long> filter(FilterParam param) {
         if (param == null
                 || CollectionUtils.isEmpty(param.getVideoIds())) {
@@ -74,22 +103,19 @@ public class AllowListStrategy implements FilterStrategy {
     }
 
     private boolean isMemberOfVideoAllowList(Long videoId) {
+        // TODO 优化
+        // 1. 每个idx 2000条数据 总共20w 约1.6M 可放入本地内存
+        // 2. 部分视频已下推荐，可清理
         if (Objects.isNull(videoId)) {
             return false;
         }
         try {
-            int newIdx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 100;
-            String newPrefix = VIDEO_ALLOW_LIST_BITMAP_KEY_SET_PREFIX + newIdx;
-            Boolean result = redisTemplate.opsForSet().isMember(newPrefix, String.valueOf(videoId));
-//            // 兼容旧 key apollo配置0 所以删除
-//            if (Objects.equals(1, mvalCompatible) && !result) {
-//                int idx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 10;
-//                result = redisTemplate.opsForValue().getBit(VIDEO_ALLOW_LIST_BITMAP_KEY_PREFIX + idx, videoId);
-//            }
-//            if (Objects.equals(1, mvalCompatible) && !result) {
-//                result = redisTemplate.opsForValue().getBit(VIDEO_ALLOW_LIST_BITMAP_KEY, videoId);
-//            }
-            return result;
+            int idx = Math.abs(Hashing.murmur3_32().hashLong(videoId).asInt()) % 100;
+            Set<String> allowList = allowVideoCache.get(idx);
+            if (CollectionUtils.isEmpty(allowList)) {
+                return false;
+            }
+            return allowList.contains(String.valueOf(videoId));
         } catch (Exception e) {
             log.error("isMemberOfVideoAllowList error {}", videoId, e);
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/ViewedStrategy.java

@@ -30,7 +30,7 @@ import java.util.stream.Collectors;
 @Slf4j
 public class ViewedStrategy implements FilterStrategy {
     @Autowired
-    @Qualifier("longVideoRedisTemplate")
+    @Qualifier("filterRedisTemplate")
     private RedisTemplate<String, String> redisTemplate;
 
     @Autowired
@@ -44,7 +44,7 @@ public class ViewedStrategy implements FilterStrategy {
         if (StringUtils.isBlank(user)) {
             return param.getVideoIds();
         }
-        String key = String.format(keyFormat, param.getMid());
+        String key = String.format(keyFormat, user);
         Set<String> viewedVideoIds = redisTemplate.opsForSet().members(key);
         if (CollectionUtils.isEmpty(viewedVideoIds)) {
             // 从mongo取曝光数据

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -31,6 +31,10 @@ public class RankRouter {
     @Autowired
     private RankStrategy4RegionMergeModelV6 rankStrategy4RegionMergeModelV6;
     @Autowired
+    private RankStrategy4RegionMergeModelV561 rankStrategy4RegionMergeModelV561;
+    @Autowired
+    private RankStrategy4RegionMergeModelV562 rankStrategy4RegionMergeModelV562;
+    @Autowired
     private FestivalStrategy4RankModel festivalStrategy4RankModel;
 
     @Autowired
@@ -38,7 +42,6 @@ public class RankRouter {
     @Autowired
     private RankStrategy4RegionMerge rankStrategy4RegionMerge;
 
-
     public RankResult rank(RankParam param) {
         log.info("RankParam {}", JSONUtils.toJson(param));
         String abCode = param.getAbCode();
@@ -51,8 +54,10 @@ public class RankRouter {
             case "60101":
                 return rankStrategy4RankModel.rank(param);
             case "60098":
-            case "60111":
-            case "60112":
+            case "60111": // 561
+                return rankStrategy4RegionMergeModelV561.rank(param);
+            case "60112": // 562
+                return rankStrategy4RegionMergeModelV562.rank(param);
             case "60113":
             case "60114":
             case "60115":

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -153,8 +153,6 @@ public class RankService {
                 || param.getAbCode().equals("60105")
                 || param.getAbCode().equals("60107")
                 || param.getAbCode().equals("60110")
-                || param.getAbCode().equals("60111")
-                || param.getAbCode().equals("60112")
                 || param.getAbCode().equals("60113")
                 || param.getAbCode().equals("60114")
                 || param.getAbCode().equals("60115")

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/RankExtractorItemFeatureV2.java

@@ -0,0 +1,338 @@
+package com.tzld.piaoquan.recommend.server.service.rank.extractor;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class RankExtractorItemFeatureV2 {
+    public static Map<String, Double> getItemRateFeature(Map<String, String> maps) {
+
+        double d;
+        Map<String, Double> result = new HashMap<>();
+        d = ExtractorUtils.division("i_1day_exp_cnt", "i_1day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_1day_ctr",d);
+        }
+        d = ExtractorUtils.division("i_1day_exp_cnt", "i_1day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_1day_str",d);
+        }
+        d = ExtractorUtils.division("i_1day_exp_cnt", "i_1day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_1day_rov",d);
+        }
+        d = ExtractorUtils.division("i_1day_share_cnt", "i_1day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_1day_ros",d);
+        }
+
+        d = ExtractorUtils.division("i_3day_exp_cnt", "i_3day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3day_ctr",d);
+        }
+        d = ExtractorUtils.division("i_3day_exp_cnt", "i_3day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3day_str",d);
+        }
+        d = ExtractorUtils.division("i_3day_exp_cnt", "i_3day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3day_rov",d);
+        }
+        d = ExtractorUtils.division("i_3day_share_cnt", "i_3day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3day_ros",d);
+        }
+
+        d = ExtractorUtils.division("i_7day_exp_cnt", "i_7day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_7day_ctr",d);
+        }
+        d = ExtractorUtils.division("i_7day_exp_cnt", "i_7day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_7day_str",d);
+        }
+        d = ExtractorUtils.division("i_7day_exp_cnt", "i_7day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_7day_rov",d);
+        }
+        d = ExtractorUtils.division("i_7day_share_cnt", "i_7day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_7day_ros",d);
+        }
+
+        d = ExtractorUtils.division("i_3month_exp_cnt", "i_3month_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3month_ctr",d);
+        }
+        d = ExtractorUtils.division("i_3month_exp_cnt", "i_3month_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3month_str",d);
+        }
+        d = ExtractorUtils.division("i_3month_exp_cnt", "i_3month_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3month_rov",d);
+        }
+        d = ExtractorUtils.division("i_3month_share_cnt", "i_3month_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("i_3month_ros",d);
+        }
+
+
+//        Map<String, String> result2 = new HashMap<>();
+//        for (Map.Entry<String, Double> entry : result.entrySet()){
+//            result2.put(entry.getKey(), String.valueOf(entry.getValue()));
+//        }
+
+        return result;
+    }
+
+    public static Map<String, String> getItemRealtimeTrend(Map<String, Map<String, Double>> maps, String date, String hour){
+        Map<String, Double> result1 = new HashMap<>();
+        Map<String, Double> result2 = new HashMap<>();
+        if (date.isEmpty() || hour.isEmpty()){
+            return rateFeatureChange(result1);
+        }
+        int N = 6;
+
+        List<String> hourStrs = ExtractorUtils.generateHourStrings(date + hour, N);
+
+        String key;
+
+        key = "share_uv_list_1day";
+        if (maps.containsKey(key)){
+            Map<String, Double> fList = maps.get(key);
+            List<Double> arrs = hourStrs.stream().map(r -> fList.getOrDefault(r, 0.0D)).collect(Collectors.toList());
+            Collections.reverse(arrs);
+            result1.put(key+"_"+N+"_avg", ExtractorUtils.calculateAverage(arrs));
+            result1.put(key+"_"+N+"_var", ExtractorUtils.calculateVariance(arrs));
+
+            List<Double> arrsDiff = ExtractorUtils.calculateDifferences(arrs);
+            result2.put(key+"_diff_"+N+"_avg", ExtractorUtils.calculateAverage(arrsDiff));
+            result2.put(key+"_diff_"+N+"_var", ExtractorUtils.calculateVariance(arrsDiff));
+        }
+
+        key = "return_uv_list_1day";
+        if (maps.containsKey(key)){
+            Map<String, Double> fList = maps.get(key);
+            List<Double> arrs = hourStrs.stream().map(r -> fList.getOrDefault(r, 0.0D)).collect(Collectors.toList());
+            Collections.reverse(arrs);
+            result1.put(key+"_"+N+"_avg", ExtractorUtils.calculateAverage(arrs));
+            result1.put(key+"_"+N+"_var", ExtractorUtils.calculateVariance(arrs));
+
+            List<Double> arrsDiff = ExtractorUtils.calculateDifferences(arrs);
+            result2.put(key+"_diff_"+N+"_avg", ExtractorUtils.calculateAverage(arrsDiff));
+            result2.put(key+"_diff_"+N+"_var", ExtractorUtils.calculateVariance(arrsDiff));
+        }
+
+        key = "share_uv_list_1h";
+        if (maps.containsKey(key)){
+            Map<String, Double> fList = maps.get(key);
+            List<Double> arrs = hourStrs.stream().map(r -> fList.getOrDefault(r, 0.0D)).collect(Collectors.toList());
+            Collections.reverse(arrs);
+            result1.put(key+"_"+N+"_avg", ExtractorUtils.calculateAverage(arrs));
+            result1.put(key+"_"+N+"_var", ExtractorUtils.calculateVariance(arrs));
+
+            List<Double> arrsDiff = ExtractorUtils.calculateDifferences(arrs);
+            result2.put(key+"_diff_"+N+"_avg", ExtractorUtils.calculateAverage(arrsDiff));
+            result2.put(key+"_diff_"+N+"_var", ExtractorUtils.calculateVariance(arrsDiff));
+        }
+
+        key = "return_uv_list_1h";
+        if (maps.containsKey(key)){
+            Map<String, Double> fList = maps.get(key);
+            List<Double> arrs = hourStrs.stream().map(r -> fList.getOrDefault(r, 0.0D)).collect(Collectors.toList());
+            Collections.reverse(arrs);
+            result1.put(key+"_"+N+"_avg", ExtractorUtils.calculateAverage(arrs));
+            result1.put(key+"_"+N+"_var", ExtractorUtils.calculateVariance(arrs));
+
+            List<Double> arrsDiff = ExtractorUtils.calculateDifferences(arrs);
+            result2.put(key+"_diff_"+N+"_avg", ExtractorUtils.calculateAverage(arrsDiff));
+            result2.put(key+"_diff_"+N+"_var", ExtractorUtils.calculateVariance(arrsDiff));
+        }
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, Double> entry : result1.entrySet()){
+            result.put(entry.getKey(), String.valueOf(entry.getValue()));
+        }
+        for (Map.Entry<String, Double> entry : result2.entrySet()){
+            result.put(entry.getKey(), String.valueOf(entry.getValue()));
+        }
+
+
+        return result;
+    }
+
+
+    public static Map<String, String> rateFeatureChange(Map<String, Double> maps){
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, Double> entry : maps.entrySet()){
+            int value = ExtractorUtils.ceilLogRate(entry.getValue());
+            result.put(entry.getKey(), String.valueOf(value));
+        }
+        return result;
+    }
+    public static Map<String, String> cntFeatureChange4Double(Map<String, Double> maps){
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, Double> entry : maps.entrySet()){
+            int value = ExtractorUtils.bucketCnt(entry.getValue());
+            result.put(entry.getKey(), String.valueOf(value));
+        }
+        return result;
+    }
+
+    public static Map<String, String> cntFeatureChange(Map<String, String> maps,
+                                                       Set<String> names){
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, String> entry : maps.entrySet()){
+            if (!names.contains(entry.getKey())){
+                continue;
+            }
+            int value = ExtractorUtils.bucketCnt(Double.valueOf(entry.getValue()));
+            result.put(entry.getKey(), String.valueOf(value));
+        }
+        return result;
+    }
+
+    public static Map<String, String> getItemRealtimeCnt(Map<String, Map<String, Double>> maps,
+                                                         Set<String> names,
+                                                         String date, String hour){
+        Map<String, String> result = new HashMap<>();
+        if (date.isEmpty() || hour.isEmpty()){
+            return result;
+        }
+        String dateHour = ExtractorUtils.subtractHours(date + hour, 0);
+        for (Map.Entry<String, Map<String, Double>> entry : maps.entrySet()){
+            if (!names.contains(entry.getKey())){
+                continue;
+            }
+            Double num = entry.getValue().getOrDefault(dateHour, 0.0);
+            if (!ExtractorUtils.isDoubleEqualToZero(num)){
+                result.put(entry.getKey(), String.valueOf(num));
+            }
+        }
+        return result;
+    }
+
+    public static Map<String, Double> getItemRealtimeRate(Map<String, Map<String, Double>> maps,
+                                                         String datehour){
+        Map<String, Double> result = new HashMap<>();
+        if (datehour.isEmpty()){
+            return result;
+        }
+        String dateHour = ExtractorUtils.subtractHours(datehour, 0);
+
+        double d, d1, d2;
+        String k1, k2;
+
+        k1 = "view_pv_list_1day";
+        k2 = "play_pv_list_1day";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1day_ctr_rt", d);
+            }
+        }
+
+        k1 = "view_pv_list_1day";
+        k2 = "share_pv_list_1day";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1day_str_rt", d);
+            }
+        }
+
+        k1 = "share_pv_list_1day";
+        k2 = "return_uv_list_1day";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1day_ros_rt", d);
+            }
+        }
+
+        k1 = "view_pv_list_1day";
+        k2 = "return_uv_list_1day";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1day_rov_rt", d);
+            }
+        }
+
+        //---
+        k1 = "view_pv_list_1h";
+        k2 = "play_pv_list_1h";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1h_ctr_rt", d);
+            }
+        }
+
+        k1 = "view_pv_list_1h";
+        k2 = "share_pv_list_1h";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1h_str_rt", d);
+            }
+        }
+
+        k1 = "share_pv_list_1day";
+        k2 = "return_uv_list_1h";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1h_ros_rt", d);
+            }
+        }
+
+        k1 = "view_pv_list_1h";
+        k2 = "return_uv_list_1h";
+        if (maps.containsKey(k1) && maps.containsKey(k2)){
+            d1 = maps.get(k1).getOrDefault(dateHour, 0.0);
+            d2 = maps.get(k2).getOrDefault(dateHour, 0.0);
+            d = ExtractorUtils.divisionDouble(d1, d2);
+            if (!ExtractorUtils.isDoubleEqualToZero(d)){
+                result.put("i_1h_rov_rt", d);
+            }
+        }
+//        Map<String, String> result2 = new HashMap<>();
+//        for (Map.Entry<String, Double> entry : result.entrySet()){
+//            result2.put(entry.getKey(), String.valueOf(entry.getValue()));
+//        }
+
+
+        return result;
+    }
+
+    public static void main(String[] args) {
+        String s1 = "share_uv_list_1day";
+        String s2 = "2024011300:2,2024011301:2,2024011304:2,2024011309:3,2024011311:3,2024011314:4,2024011315:4,2024011321:1,2024011323:1,2024011400:1,2024011401:1,2024011404:1,2024011406:1,2024011407:1,2024011408:1,2024011410:1,2024011423:1,2024011302:2,2024011305:2,2024011312:4,2024011313:4,2024011317:4,2024011318:4,2024011319:3,2024011320:1,2024011403:1,2024011409:1,2024011411:1,2024011419:1,2024011420:1,2024011422:1,2024011303:2,2024011306:2,2024011307:2,2024011308:2,2024011310:3,2024011316:4,2024011322:1,2024011402:1,2024011405:1,2024011421:1";
+        Map<String, Double> m1 = new HashMap<>();
+        Map<String, Map<String, Double>> maps = new HashMap<>();
+        for (String s : s2.split(",")){
+            String s3 = s.split(":")[0];
+            String s4 = s.split(":")[1];
+            m1.put(s3, Double.valueOf(s4));
+        }
+        maps.put(s1, m1);
+
+        String date = "20240114";
+        String hour = "20";
+        System.out.println(getItemRealtimeTrend(maps, date, hour));
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/RankExtractorUserFeatureV2.java

@@ -0,0 +1,120 @@
+package com.tzld.piaoquan.recommend.server.service.rank.extractor;
+
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class RankExtractorUserFeatureV2 {
+    public static Map<String, Double> getUserRateFeature(Map<String, String> maps) {
+
+        double d;
+        Map<String, Double> result = new HashMap<>();
+        d = ExtractorUtils.division("u_1day_exp_cnt", "u_1day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_1day_ctr",d);
+        }
+        d = ExtractorUtils.division("u_1day_exp_cnt", "u_1day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_1day_str",d);
+        }
+        d = ExtractorUtils.division("u_1day_exp_cnt", "u_1day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_1day_rov",d);
+        }
+        d = ExtractorUtils.division("u_1day_share_cnt", "u_1day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_1day_ros",d);
+        }
+
+        d = ExtractorUtils.division("u_3day_exp_cnt", "u_3day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3day_ctr",d);
+        }
+        d = ExtractorUtils.division("u_3day_exp_cnt", "u_3day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3day_str",d);
+        }
+        d = ExtractorUtils.division("u_3day_exp_cnt", "u_3day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3day_rov",d);
+        }
+        d = ExtractorUtils.division("u_3day_share_cnt", "u_3day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3day_ros",d);
+        }
+
+        d = ExtractorUtils.division("u_7day_exp_cnt", "u_7day_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_7day_ctr",d);
+        }
+        d = ExtractorUtils.division("u_7day_exp_cnt", "u_7day_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_7day_str",d);
+        }
+        d = ExtractorUtils.division("u_7day_exp_cnt", "u_7day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_7day_rov",d);
+        }
+        d = ExtractorUtils.division("u_7day_share_cnt", "u_7day_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_7day_ros",d);
+        }
+
+        d = ExtractorUtils.division("u_3month_exp_cnt", "u_3month_click_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3month_ctr",d);
+        }
+        d = ExtractorUtils.division("u_3month_exp_cnt", "u_3month_share_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3month_str",d);
+        }
+        d = ExtractorUtils.division("u_3month_exp_cnt", "u_3month_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3month_rov",d);
+        }
+        d = ExtractorUtils.division("u_3month_share_cnt", "u_3month_return_cnt", maps);
+        if (!ExtractorUtils.isDoubleEqualToZero(d)){
+            result.put("u_3month_ros",d);
+        }
+//        Map<String, String> result2 = new HashMap<>();
+//        for (Map.Entry<String, Double> entry : result.entrySet()){
+//            result2.put(entry.getKey(), String.valueOf(entry.getValue()));
+//        }
+
+        return result;
+    }
+
+
+    public static Map<String, String> rateFeatureChange(Map<String, Double> maps){
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, Double> entry : maps.entrySet()){
+            int value = ExtractorUtils.ceilLogRate(entry.getValue());
+            result.put(entry.getKey(), String.valueOf(value));
+        }
+        return result;
+    }
+
+    public static Map<String, String> cntFeatureChange(Map<String, String> maps, Set<String> names){
+        Map<String, String> result = new HashMap<>();
+        for (Map.Entry<String, String> entry : maps.entrySet()){
+            if (!names.contains(entry.getKey())){
+                continue;
+            }
+            int value = ExtractorUtils.bucketCnt(Double.valueOf(entry.getValue()));
+            result.put(entry.getKey(), String.valueOf(value));
+        }
+        return result;
+    }
+
+    public static Map<String, String> getOriginFeature(Map<String, String> maps, Set<String> names){
+        Map<String, String> result = new HashMap<>();
+        for (String name: names){
+            if (maps.containsKey(name)){
+                result.put(name, maps.get(name));
+            }
+        }
+        return result;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/OfflineVlogFeatureGroup.java

@@ -96,7 +96,23 @@ public enum OfflineVlogFeatureGroup {
     i_1h_ctr_rt,
     i_1h_str_rt,
     i_1h_ros_rt,
-    i_1h_rov_rt
+    i_1h_rov_rt,
+    u_7day_exp_cnt,
+    u_7day_click_cnt,
+    u_7day_share_cnt,
+    u_7day_return_cnt,
+    i_7day_exp_cnt,
+    i_7day_click_cnt,
+    i_7day_share_cnt,
+    i_7day_return_cnt,
+    u_7day_ctr,
+    u_7day_str,
+    u_7day_rov,
+    u_7day_ros,
+    i_7day_ctr,
+    i_7day_str,
+    i_7day_rov,
+    i_7day_ros
     ;
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV1.java

@@ -3,39 +3,98 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeatureV2;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeatureV2;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
-
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
 
 /**
  * @author zhangbo
- * @desc 地域召回融合
+ * @desc 地域召回融合 流量池汤姆森
  */
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV1 extends RankService {
-//    @ApolloJsonValue("${video.model.weightv3:}")
-//    private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${rank.score.merge.weightv1:}")
+    private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
+    @Override
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
+        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
+            return quickFlowPoolVideos;
+        } else {
+            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
+        }
+    }
+    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
+
+        //初始化 userid
+        UserFeature userFeature = new UserFeature();
+        userFeature.setMid(param.getMid());
+
+        // 初始化RankItem
+        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                .filter(d -> d.getPushFrom().equals(pushFrom))
+                .findFirst();
+        List<Video> videoList = data.get().getVideos();
+        if (videoList == null) {
+            return Collections.emptyList();
+        }
+        List<RankItem> rankItems = new ArrayList<>();
+        for (int i = 0; i < videoList.size(); i++) {
+            RankItem rankItem = new RankItem(videoList.get(i));
+            rankItems.add(rankItem);
+        }
+
+        // 初始化上下文参数
+        ScoreParam scoreParam = convert(param);
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
+                .scoring(scoreParam, userFeature, rankItems);
+
+        if (rovRecallScore == null) {
+            return Collections.emptyList();
+        }
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
         while(iterator.hasNext()){
@@ -49,58 +108,64 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
     }
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        List<Video> oldRovs = new ArrayList<>();
+        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        int sizeReturn = param.getSize();
+        removeDuplicate(oldRovs);
+        oldRovs = oldRovs.size() <= sizeReturn
+                ? oldRovs
+                : oldRovs.subList(0, sizeReturn);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, oldRovs);
+
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1_default.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3_default.PUSH_FORM);
         List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v1);
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
+        //-------------------节日扶持召回 融合+去重-------------------
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
+
+        rovRecallRank.addAll(oldRovs);
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 0.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
+
 
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
 
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
-//        List<String> videoIdKeys = rovRecallRank.stream()
-//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-//                .collect(Collectors.toList());
-//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-//                JSONUtils.toJson(videoScores));
-//        if (CollectionUtils.isNotEmpty(videoScores)
-//                && videoScores.size() == rovRecallRank.size()) {
-//            for (int i = 0; i < videoScores.size(); i++) {
-//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-//            }
-//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        }
 
         // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<RankItem> items = model(rovRecallRank, param);
         List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
         List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
         Calendar calendar = Calendar.getInstance();
@@ -114,7 +179,7 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         }
         // 2 统计分
         String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
         for (int i=0; i<24; ++i){
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
@@ -141,25 +206,103 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流的rov和ros
+            List<Double> share2allreturn = getRateData(returns, shares, 1.0, 10.0);
+            Double share2allreturnScore = calScoreWeight(share2allreturn);
+            List<Double> view2allreturn = getRateData(returns, views, 0.0, 0.0);
+            Double view2allreturnScore = calScoreWeight(view2allreturn);
+            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
+            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
+
+            // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
+        double a = mergeWeight.getOrDefault("a", 0.1);
+        double b = mergeWeight.getOrDefault("b", 0.0);
+        double bb = mergeWeight.getOrDefault("bb", 0.005);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double f = mergeWeight.getOrDefault("f", 0.1);
+        double g = mergeWeight.getOrDefault("g", 1.0);
+        double h = mergeWeight.getOrDefault("h", 20.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
         for (RankItem item : items){
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double strScore = item.getScoreStr();
+            double rosScoreModel = item.getScoreRos();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
+            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScore + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+
+            }
+            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
+            if (allreturnsScore > h){
+                score += (bb * rosScoreModel + f * share2allreturnScore + g * view2allreturnScore);
+            }
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
             video.setScoreStr(item.getScoreStr());
+            video.setScoreRos(item.getScoreRos());
             video.setScoresMap(item.getScoresMap());
             result.add(video);
         }
         Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
         return result;
     }
-
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
+    public double calTrendScore(List<Double> data){
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -172,13 +315,16 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
     public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
         List<Double> data = new LinkedList<>();
         for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
+            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)){
+                data.add(0.0);
+            }else{
+                data.add(
+                        (ups.get(i) + up) / (downs.get(i) + down)
+                );
+            }
         }
         return data;
     }
-
     public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
                                       List<String> datehours, String key){
         List<Double> views = new LinkedList<>();
@@ -190,9 +336,7 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         }
         return views;
     }
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                             List<String> rtFeaPart){
+    public List<RankItem> model(List<Video> videos, RankParam param){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -228,14 +372,13 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 JSONObject obj = new JSONObject();
                 obj.put("name", "user_key_in_model_is_null");
                 obj.put("class", this.CLASS_NAME);
-                log.info(obj.toString());
-//                return videos;
             }
         }
         final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
                 "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
                 "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
         ));
         Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
         while (iterator.hasNext()) {
@@ -244,27 +387,29 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 iterator.remove();
             }
         }
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+        Map<String, String> f1 = RankExtractorUserFeatureV2.getOriginFeature(userFeatureMap,
                 new HashSet<String>(Arrays.asList(
                         "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
                 ))
         );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+        Map<String, Double> f2__ = RankExtractorUserFeatureV2.getUserRateFeature(userFeatureMap);
+        Map<String, String> f2 = RankExtractorUserFeatureV2.rateFeatureChange(f2__);
+        Map<String, String> f3 = RankExtractorUserFeatureV2.cntFeatureChange(userFeatureMap,
                 new HashSet<String>(Arrays.asList(
                         "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                        "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
                 ))
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
                 "total_time", "play_count_total",
                 "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"
         ));
 
         List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
@@ -281,7 +426,8 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 }
                 try{
                     vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
                     Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
                     while (iteratorIn.hasNext()) {
                         Map.Entry<String, String> entry = iteratorIn.next();
@@ -289,12 +435,14 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                             iteratorIn.remove();
                         }
                     }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                    Map<String, Double> f4__ = RankExtractorItemFeatureV2.getItemRateFeature(vfMap);
+                    Map<String, String> f4 = RankExtractorItemFeatureV2.rateFeatureChange(f4__);
+                    Map<String, String> f5 = RankExtractorItemFeatureV2.cntFeatureChange(vfMap,
                             new HashSet<String>(Arrays.asList(
                                     "total_time", "play_count_total",
                                     "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                                    "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"))
                     );
                     f4.putAll(f5);
                     rankItems.get(i).setFeatureMap(f4);
@@ -327,7 +475,6 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         videoRtKeys1.addAll(videoRtKeys2);
         List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
 
-
         if (videoRtFeatures != null){
             int j = 0;
             for (RankItem item: rankItems){
@@ -356,7 +503,8 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 }catch (Exception e){
                     log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
                 item.getFeatureMap().putAll(f8);
             }
             for (RankItem item: rankItems){
@@ -387,26 +535,19 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 }catch (Exception e){
                     log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
                 item.getFeatureMap().putAll(f8);
             }
         }
 
-
-        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
-
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline("feeds_score_config_20240228.conf")
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
         JSONObject obj = new JSONObject();
         obj.put("name", "user_key_in_model_is_not_null");
         obj.put("class", this.CLASS_NAME);
-        log.info(obj.toString());
         return rovRecallScore;
     }
-
     private Map<String, String> getSceneFeature(RankParam param) {
         Map<String, String> sceneFeatureMap = new HashMap<>();
         String provinceCn = param.getProvince();
@@ -435,5 +576,92 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
 
         return sceneFeatureMap;
     }
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV2.java

@@ -2,40 +2,101 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 
 import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
-
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
 
 /**
  * @author zhangbo
- * @desc 地域召回融合
+ * @desc 地域召回融合 流量池汤姆森
  */
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV2 extends RankService {
-//    @ApolloJsonValue("${video.model.weightv3:}")
-//    private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${rank.score.merge.weightv2:}")
+    private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
+    @Override
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
+        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
+            return quickFlowPoolVideos;
+        } else {
+            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
+        }
+    }
+    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
+
+        //初始化 userid
+        UserFeature userFeature = new UserFeature();
+        userFeature.setMid(param.getMid());
+
+        // 初始化RankItem
+        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                .filter(d -> d.getPushFrom().equals(pushFrom))
+                .findFirst();
+        List<Video> videoList = data.get().getVideos();
+        if (videoList == null) {
+            return Collections.emptyList();
+        }
+        List<RankItem> rankItems = new ArrayList<>();
+        for (int i = 0; i < videoList.size(); i++) {
+            RankItem rankItem = new RankItem(videoList.get(i));
+            rankItems.add(rankItem);
+        }
+
+        // 初始化上下文参数
+        ScoreParam scoreParam = convert(param);
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
+                .scoring(scoreParam, userFeature, rankItems);
+
+        if (rovRecallScore == null) {
+            return Collections.emptyList();
+        }
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
         while(iterator.hasNext()){
@@ -49,54 +110,61 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
     }
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        List<Video> oldRovs = new ArrayList<>();
+        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        int sizeReturn = param.getSize();
+        removeDuplicate(oldRovs);
+        oldRovs = oldRovs.size() <= sizeReturn
+                ? oldRovs
+                : oldRovs.subList(0, sizeReturn);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, oldRovs);
+
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
         List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
         List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v1);
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
+        //-------------------节日扶持召回 融合+去重-------------------
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
+
+        rovRecallRank.addAll(oldRovs);
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 0.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
+
 
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
 
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
-//        List<String> videoIdKeys = rovRecallRank.stream()
-//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-//                .collect(Collectors.toList());
-//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-//                JSONUtils.toJson(videoScores));
-//        if (CollectionUtils.isNotEmpty(videoScores)
-//                && videoScores.size() == rovRecallRank.size()) {
-//            for (int i = 0; i < videoScores.size(); i++) {
-//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-//            }
-//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        }
 
         // 1 模型分
         List<String> rtFeaPart = new ArrayList<>();
@@ -114,7 +182,7 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         }
         // 2 统计分
         String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
         for (int i=0; i<24; ++i){
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
@@ -141,15 +209,65 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流的rov和ros
+            List<Double> share2allreturn = getRateData(returns, shares, 1.0, 10.0);
+            Double share2allreturnScore = calScoreWeight(share2allreturn);
+            List<Double> view2allreturn = getRateData(returns, views, 0.0, 0.0);
+            Double view2allreturnScore = calScoreWeight(view2allreturn);
+            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
+            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
+
+            // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
+        double a = mergeWeight.getOrDefault("a", 0.1);
+        double b = mergeWeight.getOrDefault("b", 0.0);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double f = mergeWeight.getOrDefault("f", 0.1);
+        double g = mergeWeight.getOrDefault("g", 1.0);
+        double h = mergeWeight.getOrDefault("h", 20.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
         for (RankItem item : items){
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0) *
-                    Math.log(1 + item.scoresMap.getOrDefault("allreturnsScore", 0.0));
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double strScore = item.getScoreStr();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
+            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScore + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+
+            }
+            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
+            if (allreturnsScore > h){
+                score += (f * share2allreturnScore + g * view2allreturnScore);
+            }
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -160,7 +278,31 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
         return result;
     }
-
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
+    public double calTrendScore(List<Double> data){
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -173,9 +315,13 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
     public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
         List<Double> data = new LinkedList<>();
         for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
+            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)){
+                data.add(0.0);
+            }else{
+                data.add(
+                        (ups.get(i) + up) / (downs.get(i) + down)
+                );
+            }
         }
         return data;
     }
@@ -190,9 +336,8 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         }
         return views;
     }
-
     public List<RankItem> model(List<Video> videos, RankParam param,
-                             List<String> rtFeaPart){
+                                List<String> rtFeaPart){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -228,7 +373,7 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
                 JSONObject obj = new JSONObject();
                 obj.put("name", "user_key_in_model_is_null");
                 obj.put("class", this.CLASS_NAME);
-                log.info(obj.toString());
+//                log.info(obj.toString());
 //                return videos;
             }
         }
@@ -258,7 +403,7 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+//        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -281,7 +426,8 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
                 }
                 try{
                     vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
                     Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
                     while (iteratorIn.hasNext()) {
                         Map.Entry<String, String> entry = iteratorIn.next();
@@ -393,20 +539,19 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         }
 
 
-        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+//        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
 
 
 
         List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+//        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
         JSONObject obj = new JSONObject();
         obj.put("name", "user_key_in_model_is_not_null");
         obj.put("class", this.CLASS_NAME);
-        log.info(obj.toString());
+//        log.info(obj.toString());
         return rovRecallScore;
     }
-
     private Map<String, String> getSceneFeature(RankParam param) {
         Map<String, String> sceneFeatureMap = new HashMap<>();
         String provinceCn = param.getProvince();
@@ -435,5 +580,92 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
 
         return sceneFeatureMap;
     }
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV3.java

@@ -46,7 +46,7 @@ import java.util.stream.Collectors;
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV3 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weight:}")
+    @ApolloJsonValue("${rank.score.merge.weightv3:}")
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
     private final Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
@@ -128,20 +128,13 @@ public class RankStrategy4RegionMergeModelV3 extends RankService {
         List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
         this.duplicate(setVideo, v7);
 
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(10, v7.size())));
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -333,7 +326,7 @@ public class RankStrategy4RegionMergeModelV3 extends RankService {
                 JSONObject obj = new JSONObject();
                 obj.put("name", "user_key_in_model_is_null");
                 obj.put("class", this.CLASS_NAME);
-                log.info(obj.toString());
+//                log.info(obj.toString());
 //                return videos;
             }
         }
@@ -363,7 +356,7 @@ public class RankStrategy4RegionMergeModelV3 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+//        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -499,17 +492,17 @@ public class RankStrategy4RegionMergeModelV3 extends RankService {
         }
 
 
-        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+//        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
 
 
 
         List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+//        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
         JSONObject obj = new JSONObject();
         obj.put("name", "user_key_in_model_is_not_null");
         obj.put("class", this.CLASS_NAME);
-        log.info(obj.toString());
+//        log.info(obj.toString());
         return rovRecallScore;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV4.java

@@ -4,18 +4,31 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
@@ -29,14 +42,61 @@ import java.util.stream.Collectors;
 
 /**
  * @author zhangbo
- * @desc 地域召回融合
+ * @desc 地域召回融合 流量池汤姆森
  */
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV4 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weight:}")
+    @ApolloJsonValue("${rank.score.merge.weightv4:}")
     private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
+    @Override
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
+        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
+            return quickFlowPoolVideos;
+        } else {
+            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
+        }
+    }
+    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
+
+        //初始化 userid
+        UserFeature userFeature = new UserFeature();
+        userFeature.setMid(param.getMid());
+
+        // 初始化RankItem
+        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                .filter(d -> d.getPushFrom().equals(pushFrom))
+                .findFirst();
+        List<Video> videoList = data.get().getVideos();
+        if (videoList == null) {
+            return Collections.emptyList();
+        }
+        List<RankItem> rankItems = new ArrayList<>();
+        for (int i = 0; i < videoList.size(); i++) {
+            RankItem rankItem = new RankItem(videoList.get(i));
+            rankItems.add(rankItem);
+        }
+
+        // 初始化上下文参数
+        ScoreParam scoreParam = convert(param);
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
+                .scoring(scoreParam, userFeature, rankItems);
+
+        if (rovRecallScore == null) {
+            return Collections.emptyList();
+        }
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
         while(iterator.hasNext()){
@@ -50,7 +110,13 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
     }
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
@@ -61,43 +127,27 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
-
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+        //-------------------节日扶持召回 融合+去重-------------------
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
-//        List<String> videoIdKeys = rovRecallRank.stream()
-//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-//                .collect(Collectors.toList());
-//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-//                JSONUtils.toJson(videoScores));
-//        if (CollectionUtils.isNotEmpty(videoScores)
-//                && videoScores.size() == rovRecallRank.size()) {
-//            for (int i = 0; i < videoScores.size(); i++) {
-//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-//            }
-//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        }
 
         // 1 模型分
         List<String> rtFeaPart = new ArrayList<>();
@@ -115,7 +165,7 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         }
         // 2 统计分
         String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
         for (int i=0; i<24; ++i){
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
@@ -129,7 +179,7 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
             List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
             List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
 
-            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            List<Double> share2return = getRateData(returns, shares, 1.0, 10.0);
             Double share2returnScore = calScoreWeight(share2return);
             List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
             Double view2returnScore = calScoreWeight(view2return);
@@ -142,24 +192,47 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
 
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
             // rov的趋势
             double trendScore = calTrendScore(view2return);
             item.scoresMap.put("trendScore", trendScore);
 
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
-        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
+        double a = mergeWeight.getOrDefault("a", 1.0);
+        double b = mergeWeight.getOrDefault("b", 1.0);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
         for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
-                    + alpha * trendScore
-                    ;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double strScore = item.getScoreStr();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScore + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -170,22 +243,31 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
         return result;
     }
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
     public double calTrendScore(List<Double> data){
-        int cnt = 0;
         double sum = 0.0;
         int size = data.size();
         for (int i=0; i<size-4; ++i){
             sum += data.get(i) - data.get(i+4);
-            cnt ++;
         }
         if (sum * 10 > 0.6){
             sum = 0.6;
         }else{
             sum = sum * 10;
         }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
         return sum;
     }
-
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -215,9 +297,8 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         }
         return views;
     }
-
     public List<RankItem> model(List<Video> videos, RankParam param,
-                             List<String> rtFeaPart){
+                                List<String> rtFeaPart){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -253,7 +334,7 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
                 JSONObject obj = new JSONObject();
                 obj.put("name", "user_key_in_model_is_null");
                 obj.put("class", this.CLASS_NAME);
-                log.info(obj.toString());
+//                log.info(obj.toString());
 //                return videos;
             }
         }
@@ -283,7 +364,7 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+//        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -306,7 +387,8 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
                 }
                 try{
                     vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
                     Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
                     while (iteratorIn.hasNext()) {
                         Map.Entry<String, String> entry = iteratorIn.next();
@@ -418,20 +500,19 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         }
 
 
-        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+//        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
 
 
 
         List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+//        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
         JSONObject obj = new JSONObject();
         obj.put("name", "user_key_in_model_is_not_null");
         obj.put("class", this.CLASS_NAME);
-        log.info(obj.toString());
+//        log.info(obj.toString());
         return rovRecallScore;
     }
-
     private Map<String, String> getSceneFeature(RankParam param) {
         Map<String, String> sceneFeatureMap = new HashMap<>();
         String provinceCn = param.getProvince();
@@ -460,5 +541,92 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
 
         return sceneFeatureMap;
     }
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV5.java

@@ -4,18 +4,32 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
@@ -29,14 +43,61 @@ import java.util.stream.Collectors;
 
 /**
  * @author zhangbo
- * @desc 地域召回融合
+ * @desc 地域召回融合 流量池汤姆森
  */
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV5 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weight:}")
+    @ApolloJsonValue("${rank.score.merge.weightv5:}")
     private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
+    @Override
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
+        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
+            return quickFlowPoolVideos;
+        } else {
+            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
+        }
+    }
+    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
+
+        //初始化 userid
+        UserFeature userFeature = new UserFeature();
+        userFeature.setMid(param.getMid());
+
+        // 初始化RankItem
+        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                .filter(d -> d.getPushFrom().equals(pushFrom))
+                .findFirst();
+        List<Video> videoList = data.get().getVideos();
+        if (videoList == null) {
+            return Collections.emptyList();
+        }
+        List<RankItem> rankItems = new ArrayList<>();
+        for (int i = 0; i < videoList.size(); i++) {
+            RankItem rankItem = new RankItem(videoList.get(i));
+            rankItems.add(rankItem);
+        }
+
+        // 初始化上下文参数
+        ScoreParam scoreParam = convert(param);
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
+                .scoring(scoreParam, userFeature, rankItems);
+
+        if (rovRecallScore == null) {
+            return Collections.emptyList();
+        }
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
         while(iterator.hasNext()){
@@ -50,7 +111,13 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
     }
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
@@ -61,129 +128,140 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
-
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+        //-------------------节日扶持召回 融合+去重-------------------
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
-//        List<String> videoIdKeys = rovRecallRank.stream()
-//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-//                .collect(Collectors.toList());
-//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-//                JSONUtils.toJson(videoScores));
-//        if (CollectionUtils.isNotEmpty(videoScores)
-//                && videoScores.size() == rovRecallRank.size()) {
-//            for (int i = 0; i < videoScores.size(); i++) {
-//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-//            }
-//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        }
 
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+        List<String> videoIdKeys = rovRecallRank.stream()
+                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+                .collect(Collectors.toList());
+        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+        if (CollectionUtils.isNotEmpty(videoScores)
+                && videoScores.size() == rovRecallRank.size()) {
+            for (int i = 0; i < videoScores.size(); i++) {
+                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
             }
+            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
         }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
-        for (int i=0; i<24; ++i){
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items){
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeight(share2return);
-            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeight(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeight(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeight(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            Double allreturnsScore = calScoreWeight(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
-        double beta = this.mergeWeight.getOrDefault("beta", 1.0);
-        for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 0.0 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
-                    + alpha * trendScore
-                    + beta * newVideoScore
-                    ;
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
+        return rovRecallRank;
+
+//        // 1 模型分
+//        List<String> rtFeaPart = new ArrayList<>();
+//        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+//        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+//        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+//        Calendar calendar = Calendar.getInstance();
+//        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+//        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+//        String rtFeaPart1h = date + hour;
+//        if (rtFeaPartKeyResult != null){
+//            if (rtFeaPartKeyResult.get(1) != null){
+//                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+//            }
+//        }
+//        // 2 统计分
+//        String cur = rtFeaPart1h;
+//        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
+//        for (int i=0; i<24; ++i){
+//            datehours.add(cur);
+//            cur = ExtractorUtils.subtractHours(cur, 1);
+//        }
+//        for (RankItem item : items){
+//            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+//            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+//            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+//            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+//            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+//            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+//            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+//
+//            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+//            Double share2returnScore = calScoreWeight(share2return);
+//            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+//            Double view2returnScore = calScoreWeight(view2return);
+//            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+//            Double view2playScore = calScoreWeight(view2play);
+//            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+//            Double play2shareScore = calScoreWeight(play2share);
+//            item.scoresMap.put("share2returnScore", share2returnScore);
+//            item.scoresMap.put("view2returnScore", view2returnScore);
+//            item.scoresMap.put("view2playScore", view2playScore);
+//            item.scoresMap.put("play2shareScore", play2shareScore);
+//
+//            // 全部回流
+//            Double allreturnsScore = calScoreWeight(allreturns);
+//            item.scoresMap.put("allreturnsScore", allreturnsScore);
+//
+//            // 平台回流
+//            Double preturnsScore = calScoreWeight(returns);
+//            item.scoresMap.put("preturnsScore", preturnsScore);
+//
+//            // rov的趋势
+//            double trendScore = calTrendScore(view2return);
+//            item.scoresMap.put("trendScore", trendScore);
+//
+//            // 新视频提取
+//            double newVideoScore = calNewVideoScore(itemBasicMap);
+//            item.scoresMap.put("newVideoScore", newVideoScore);
+//        }
+//        // 3 融合公式
+//        List<Video> result = new ArrayList<>();
+//        double a = mergeWeight.getOrDefault("a", 1.0);
+//        double b = mergeWeight.getOrDefault("b", 1.0);
+//        double c = mergeWeight.getOrDefault("c", 0.0002);
+//        double d = mergeWeight.getOrDefault("d", 1.0);
+//        double e = mergeWeight.getOrDefault("e", 1.0);
+//        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
+//        for (RankItem item : items){
+//            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
+//                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+//            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+//                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+//            double strScore = item.getScoreStr();
+//            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+//            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+//            double score = 0.0;
+//            if (ifAdd < 0.5){
+//                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+//                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+//            }else {
+//                score = a * strScore + b * rosScore + c * preturnsScore +
+//                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+//            }
+//            Video video = item.getVideo();
+//            video.setScore(score);
+//            video.setSortScore(score);
+//            video.setScoreStr(item.getScoreStr());
+//            video.setScoresMap(item.getScoresMap());
+//            result.add(video);
+//        }
+//        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        return result;
     }
     public double calNewVideoScore(Map<String, String> itemBasicMap){
         double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 8){
+        if (existenceDays > 5){
             return 0.0;
         }
-        double score = 1.0 / (existenceDays + 5.0);
+        double score = 1.0 / (existenceDays + 10.0);
         return score;
     }
     public double calTrendScore(List<Double> data){
@@ -203,7 +281,6 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         }
         return sum;
     }
-
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -233,9 +310,8 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         }
         return views;
     }
-
     public List<RankItem> model(List<Video> videos, RankParam param,
-                             List<String> rtFeaPart){
+                                List<String> rtFeaPart){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -271,7 +347,7 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
                 JSONObject obj = new JSONObject();
                 obj.put("name", "user_key_in_model_is_null");
                 obj.put("class", this.CLASS_NAME);
-                log.info(obj.toString());
+//                log.info(obj.toString());
 //                return videos;
             }
         }
@@ -301,7 +377,7 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+//        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -437,20 +513,19 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         }
 
 
-        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+//        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
 
 
 
         List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+//        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
         JSONObject obj = new JSONObject();
         obj.put("name", "user_key_in_model_is_not_null");
         obj.put("class", this.CLASS_NAME);
-        log.info(obj.toString());
+//        log.info(obj.toString());
         return rovRecallScore;
     }
-
     private Map<String, String> getSceneFeature(RankParam param) {
         Map<String, String> sceneFeatureMap = new HashMap<>();
         String provinceCn = param.getProvince();
@@ -479,5 +554,92 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
 
         return sceneFeatureMap;
     }
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV561.java

@@ -0,0 +1,562 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeatureV2;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeatureV2;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV561 extends RankService {
+    @ApolloJsonValue("${rank.score.merge.weightv561:}")
+    private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        List<Video> rovRecallRank = new ArrayList<>();
+        rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        int sizeReturn = param.getSize();
+        removeDuplicate(rovRecallRank);
+        rovRecallRank = rovRecallRank.size() <= sizeReturn
+                ? rovRecallRank
+                : rovRecallRank.subList(0, sizeReturn);
+        rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
+        removeDuplicate(rovRecallRank);
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        // 1 模型分
+        List<RankItem> items = model(rovRecallRank, param);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            // 全部回流
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        double a = mergeWeight.getOrDefault("a", 1.0);
+        double b = mergeWeight.getOrDefault("b", 1.0);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
+        for (RankItem item : items){
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double strScore = item.getScoreStr();
+            double rosScoreModel = item.getScoreRos();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScoreModel, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScoreModel + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoreRos(item.getScoreRos());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
+    public double calTrendScore(List<Double> data){
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+    public List<RankItem> model(List<Video> videos, RankParam param){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeatureV2.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, Double> f2__ = RankExtractorUserFeatureV2.getUserRateFeature(userFeatureMap);
+        Map<String, String> f2 = RankExtractorUserFeatureV2.rateFeatureChange(f2__);
+        Map<String, String> f3 = RankExtractorUserFeatureV2.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                        "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, Double> f4__ = RankExtractorItemFeatureV2.getItemRateFeature(vfMap);
+                    Map<String, String> f4 = RankExtractorItemFeatureV2.rateFeatureChange(f4__);
+                    Map<String, String> f5 = RankExtractorItemFeatureV2.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                                    "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline("feeds_score_config_20240228.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        return rovRecallScore;
+    }
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV562.java

@@ -0,0 +1,586 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeatureV2;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeatureV2;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV562 extends RankService {
+    @ApolloJsonValue("${rank.score.merge.weightv562:}")
+    private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        //-------------------地域相关召回 融合+去重-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------相关性召回 融合+去重-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+        //-------------------节日扶持召回 融合+去重-------------------
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        // 1 模型分
+        List<RankItem> items = model(rovRecallRank, param);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            // 全部回流
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        double a = mergeWeight.getOrDefault("a", 1.0);
+        double b = mergeWeight.getOrDefault("b", 1.0);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
+        for (RankItem item : items){
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double strScore = item.getScoreStr();
+            double rosScoreModel = item.getScoreRos();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScoreModel, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScoreModel + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoreRos(item.getScoreRos());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
+    public double calTrendScore(List<Double> data){
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+    public List<RankItem> model(List<Video> videos, RankParam param){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeatureV2.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, Double> f2__ = RankExtractorUserFeatureV2.getUserRateFeature(userFeatureMap);
+        Map<String, String> f2 = RankExtractorUserFeatureV2.rateFeatureChange(f2__);
+        Map<String, String> f3 = RankExtractorUserFeatureV2.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
+                        "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, Double> f4__ = RankExtractorItemFeatureV2.getItemRateFeature(vfMap);
+                    Map<String, String> f4 = RankExtractorItemFeatureV2.rateFeatureChange(f4__);
+                    Map<String, String> f5 = RankExtractorItemFeatureV2.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
+                                    "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline("feeds_score_config_20240228.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        return rovRecallScore;
+    }
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV6.java

@@ -41,7 +41,7 @@ import java.util.stream.Collectors;
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV6 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weight:}")
+    @ApolloJsonValue("${rank.score.merge.weightv6:}")
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
     private final Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
@@ -78,20 +78,13 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
         this.duplicate(setVideo, v7);
 
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(10, v7.size())));
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -110,7 +103,6 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
 //            }
 //            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
 //        }
-
         // 1 模型分
         List<String> rtFeaPart = new ArrayList<>();
         List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
@@ -313,7 +305,7 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+//        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -449,13 +441,13 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         }
 
 
-        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+//        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
 
 
 
         List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+//        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
         JSONObject obj = new JSONObject();
         obj.put("name", "user_key_in_model_is_not_null");
         obj.put("class", this.CLASS_NAME);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -95,13 +95,26 @@ public class RecallService implements ApplicationContextAware {
             return strategies;
         } else {
             switch (abCode) {
-                case "60120": // 576
                 case "60121": // 536
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1_default.class.getSimpleName()));
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3_default.class.getSimpleName()));
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
+                    strategies.addAll(getRegionRecallStrategy(param));
+                    break;
                 case "60122": // 537
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
+                    strategies.addAll(getRegionRecallStrategy(param));
+                    break;
+                case "60120": // 576
                 case "60123": // 541
                 case "60124": // 546
                 case "60125": // 547
                 case "60126": // 548
+                case "60112": // 562
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
@@ -110,7 +123,7 @@ public class RecallService implements ApplicationContextAware {
                 default:
                     strategies.addAll(getRegionRecallStrategy(param));
             }
-            //2:通过“流量池标记”控制“流量池召回子策略”
+            //2:通过“流量池标记”控制“流量池召回子策略” 其中有9组会走EXPERIMENTAL_FLOW_SET_LEVEL 有1组会走EXPERIMENTAL_FLOW_SET_LEVEL_SCORE
             if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL)) {
                 strategies.add(strategyMap.get(QuickFlowPoolWithLevelRecallStrategy.class.getSimpleName()));
                 strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategy.class.getSimpleName()));
@@ -130,8 +143,8 @@ public class RecallService implements ApplicationContextAware {
             ;
         } else {
             switch (abCode) {
-                case "60111":
-                case "60112":
+
+
                 case "60113":
                 case "60114":
                 case "60115":
@@ -144,6 +157,7 @@ public class RecallService implements ApplicationContextAware {
                     strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
                     strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                     break;
+                case "60111": // 561
                 case "60107": // 556
                 case "60106":
                 case "60068":
@@ -156,15 +170,16 @@ public class RecallService implements ApplicationContextAware {
                 case "60103": // 增加地域1小时扩量，通过配置实现
                 case "60105": // 通过更改param中的配置实现使用不同数据源 data66 rule68 + 有排序模块
                 case "60120": // 576
+                    strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
+                    strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                    break;
                 case "60121": // 536
                 case "60122": // 537
                 case "60124": // 546
                 case "60125": // 547
-                    strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
-                    strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
-                    break;
                 case "60123": // 541
                 case "60126": // 548
+                case "60112": // 562
                     strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
                     strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                     strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV1.java

@@ -26,10 +26,12 @@ import org.springframework.stereotype.Component;
 @Slf4j
 @Component
 public class RegionRealtimeRecallStrategyV1 implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
     @Autowired
     private RegionFilterService filterService;
     @Override
     public List<Video> recall(RecallParam param) {
+        long t0 = System.currentTimeMillis();
         // 1 获取省份key 放入参数map中
         String provinceCn = param.getProvince();
         if (provinceCn == null){
@@ -50,47 +52,48 @@ public class RegionRealtimeRecallStrategyV1 implements RecallStrategy {
         for (Pair<Long, Double> v: result){
             videoMap.put(v.getLeft(), v.getRight());
         }
-
-
-        long t1 = new Long(System.currentTimeMillis());
-//        int chunkSize = 25;
-//        List<List<Long>> groupedKeys = groupKeys(videoMap, chunkSize);
-//        List<Long> videoids = new ArrayList<>();
-//        for (List<Long> tmp : groupedKeys){
-//            FilterParam filterParam = FilterParamFactory.create(param, tmp);
-//            filterParam.setForceTruncation(10000);
-//            filterParam.setConcurrent(true);
-//            filterParam.setUsePreView(false);
-//            FilterResult filterResult = filterService.filter(filterParam);
-//            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())){
-//                videoids.addAll(filterResult.getVideoIds());
-//            }
-//        }
+        long t1 = System.currentTimeMillis();
+        // 3 召回内部过滤
         FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
         filterParam.setForceTruncation(10000);
         filterParam.setConcurrent(true);
         filterParam.setNotUsePreView(false);
         FilterResult filterResult = filterService.filter(filterParam);
-        long t2 = new Long(System.currentTimeMillis());
+        long t2 = System.currentTimeMillis();
         JSONObject obj = new JSONObject();
-        obj.put("name", "RegionRealtimeRecallStrategyV1");
+        obj.put("name", this.CLASS_NAME);
         obj.put("filter_time", t2-t1);
+        obj.put("recall_time", t1-t0);
         obj.put("provinceCn", provinceCn);
         obj.put("sizeOld", videoMap.size());
+        obj.put("mid", param.getMid());
+        obj.put("regionCode", param.getRegionCode());
+        obj.put("video_input", result.stream()
+                .map(pair -> String.valueOf(pair.getLeft()))
+                .collect(Collectors.joining(",")));
         List<Video> videosResult = new ArrayList<>();
+        List<Long> videosResultId = new ArrayList<>();
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
-            obj.put("sizeNew", filterResult.getVideoIds().size());
-            filterResult.getVideoIds().stream().forEach(vid -> {
+            filterResult.getVideoIds().forEach(vid -> {
                 Video video = new Video();
                 video.setVideoId(vid);
                 video.setAbCode(param.getAbCode());
                 video.setRovScore(videoMap.get(vid));
                 video.setPushFrom(pushFrom());
                 videosResult.add(video);
+                videosResultId.add(vid);
             });
         }
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
+//        obj.put("video_output", videosResult.stream()
+//                .map(v -> String.valueOf(v.getVideoId()))
+//                .collect(Collectors.joining(",")));
+        obj.put("video_filter", result.stream()
+                .filter(v -> !videosResultId.contains(v.getLeft()))
+                .map(v -> String.valueOf(v.getLeft()))
+                .collect(Collectors.joining(",")));
+        obj.put("sizeNew", videosResult.size());
         log.info(obj.toString());
-        Collections.sort(videosResult, Comparator.comparingDouble(o -> -o.getRovScore()));
         return videosResult;
     }
     public static final String PUSH_FORM = "recall_strategy_region_1h";

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV1_default.java

@@ -0,0 +1,131 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ */
+@Slf4j
+@Component
+public class RegionRealtimeRecallStrategyV1_default implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private RegionFilterService filterService;
+    @Override
+    public List<Video> recall(RecallParam param) {
+        long t0 = System.currentTimeMillis();
+        // 1 获取省份key 放入参数map中
+        String provinceCn = param.getProvince();
+        if (provinceCn == null){
+            // provinceCn = "中国";
+            return new ArrayList<>();
+        }else{
+            provinceCn = provinceCn.replaceAll("省$", "");
+        }
+        final Set<String> provinceCns = new HashSet<>(Arrays.asList(
+                "北京", "天津", "河北", "山西", "内蒙古", "辽宁", "吉林",
+                "黑龙江", "上海", "江苏", "浙江", "安徽", "福建", "江西",
+                "山东", "河南", "湖北", "湖南", "广东", "广西", "海南",
+                "重庆", "四川", "贵州", "云南", "西藏", "陕西", "甘肃",
+                "青海", "宁夏", "新疆", "台湾", "香港", "澳门", "中国"
+        ));
+        if (!provinceCns.contains(provinceCn)){
+            return new ArrayList<>();
+        }
+        Map<String, String> param4Model = new HashMap<>(1);
+        param4Model.put("region_province", provinceCn);
+        // 2 通过model拿到召回list
+        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_region_v1.conf");
+        List<List<Pair<Long, Double>>> results = pipeline.recall(param4Model);
+        List<Pair<Long, Double>> result = results.get(0);
+        for (int i=1; i<results.size(); ++i){
+            result.addAll(results.get(i));
+        }
+        Map<Long, Double> videoMap = new LinkedHashMap<>();
+        for (Pair<Long, Double> v: result){
+            videoMap.put(v.getLeft(), v.getRight());
+        }
+        long t1 = System.currentTimeMillis();
+        // 3 召回内部过滤
+        FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        long t2 = System.currentTimeMillis();
+        JSONObject obj = new JSONObject();
+        obj.put("name", this.CLASS_NAME);
+        obj.put("filter_time", t2-t1);
+        obj.put("recall_time", t1-t0);
+        obj.put("provinceCn", provinceCn);
+        obj.put("sizeOld", videoMap.size());
+        obj.put("mid", param.getMid());
+        obj.put("regionCode", param.getRegionCode());
+        obj.put("video_input", result.stream()
+                .map(pair -> String.valueOf(pair.getLeft()))
+                .collect(Collectors.joining(",")));
+        List<Video> videosResult = new ArrayList<>();
+        List<Long> videosResultId = new ArrayList<>();
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            filterResult.getVideoIds().forEach(vid -> {
+                Video video = new Video();
+                video.setVideoId(vid);
+                video.setAbCode(param.getAbCode());
+                video.setRovScore(videoMap.get(vid));
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+                videosResultId.add(vid);
+            });
+        }
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
+//        obj.put("video_output", videosResult.stream()
+//                .map(v -> String.valueOf(v.getVideoId()))
+//                .collect(Collectors.joining(",")));
+        obj.put("video_filter", result.stream()
+                .filter(v -> !videosResultId.contains(v.getLeft()))
+                .map(v -> String.valueOf(v.getLeft()))
+                .collect(Collectors.joining(",")));
+        obj.put("sizeNew", videosResult.size());
+        log.info(obj.toString());
+        return videosResult;
+    }
+    public static final String PUSH_FORM = "recall_strategy_region_1h";
+    @Override
+    public String pushFrom(){
+        return PUSH_FORM;
+    }
+
+
+    public static List<List<Long>> groupKeys(Map<Long, Double> videoMap, int groupSize) {
+        List<List<Long>> result = new ArrayList<>();
+        List<Long> keys = new ArrayList<>(videoMap.keySet());
+
+        int size = keys.size();
+        for (int i = 0; i < size; i += groupSize) {
+            int endIndex = Math.min(i + groupSize, size);
+            List<Long> group = keys.subList(i, endIndex);
+            result.add(group);
+        }
+
+        return result;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV3.java

@@ -49,22 +49,14 @@ public class RegionRealtimeRecallStrategyV3 implements RecallStrategy {
         for (Pair<Long, Double> v: result){
             videoMap.put(v.getLeft(), v.getRight());
         }
-        long t1 = new Long(System.currentTimeMillis());
         FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
         filterParam.setForceTruncation(10000);
         filterParam.setConcurrent(true);
         filterParam.setNotUsePreView(false);
         FilterResult filterResult = filterService.filter(filterParam);
-        long t2 = new Long(System.currentTimeMillis());
-        JSONObject obj = new JSONObject();
-        obj.put("name", "RegionRealtimeRecallStrategyV3");
-        obj.put("provinceCn", provinceCn);
-        obj.put("sizeOld", videoMap.size());
-        obj.put("filter_time", t2-t1);
         List<Video> videosResult = new ArrayList<>();
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
-            obj.put("sizeNew", filterResult.getVideoIds().size());
-            filterResult.getVideoIds().stream().forEach(vid -> {
+            filterResult.getVideoIds().forEach(vid -> {
                 Video video = new Video();
                 video.setVideoId(vid);
                 video.setAbCode(param.getAbCode());
@@ -73,8 +65,7 @@ public class RegionRealtimeRecallStrategyV3 implements RecallStrategy {
                 videosResult.add(video);
             });
         }
-        log.info(obj.toString());
-        Collections.sort(videosResult, Comparator.comparingDouble(o -> -o.getRovScore()));
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
         return videosResult;
     }
     public static final String PUSH_FORM = "recall_strategy_region_24h";

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV3_default.java

@@ -0,0 +1,87 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+/**
+ * @author zhangbo
+ */
+@Slf4j
+@Component
+public class RegionRealtimeRecallStrategyV3_default implements RecallStrategy {
+    @Autowired
+    protected RegionFilterService filterService;
+    @Override
+    public List<Video> recall(RecallParam param) {
+        // 1 获取省份key 放入参数map中
+        String provinceCn = param.getProvince();
+        if (provinceCn == null){
+            provinceCn = "中国";
+            return new ArrayList<>();
+        }else{
+            provinceCn = provinceCn.replaceAll("省$", "");
+        }
+        final Set<String> provinceCns = new HashSet<>(Arrays.asList(
+                "北京", "天津", "河北", "山西", "内蒙古", "辽宁", "吉林",
+                "黑龙江", "上海", "江苏", "浙江", "安徽", "福建", "江西",
+                "山东", "河南", "湖北", "湖南", "广东", "广西", "海南",
+                "重庆", "四川", "贵州", "云南", "西藏", "陕西", "甘肃",
+                "青海", "宁夏", "新疆", "台湾", "香港", "澳门", "中国"
+        ));
+        if (!provinceCns.contains(provinceCn)){
+            return new ArrayList<>();
+        }
+        Map<String, String> param4Model = new HashMap<>(1);
+        param4Model.put("region_province", provinceCn);
+        // 2 通过model拿到召回list
+        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_region_v1.conf");
+        List<List<Pair<Long, Double>>> results = pipeline.recall(param4Model);
+        List<Pair<Long, Double>> result = results.get(0);
+        for (int i=1; i<results.size(); ++i){
+            result.addAll(results.get(i));
+        }
+        Map<Long, Double> videoMap = new LinkedHashMap<>();
+        for (Pair<Long, Double> v: result){
+            videoMap.put(v.getLeft(), v.getRight());
+        }
+        FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        List<Video> videosResult = new ArrayList<>();
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            filterResult.getVideoIds().forEach(vid -> {
+                Video video = new Video();
+                video.setVideoId(vid);
+                video.setAbCode(param.getAbCode());
+                video.setRovScore(videoMap.get(vid));
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+            });
+        }
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
+        return videosResult;
+    }
+    public static final String PUSH_FORM = "recall_strategy_region_24h";
+    @Override
+    public String pushFrom(){
+        return PUSH_FORM;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -33,6 +33,7 @@ public final class ScorerUtils {
         ScorerUtils.init(BASE_CONF);
         ScorerUtils.init(FLOWPOOL_CONF);
         ScorerUtils.init(VIDEO_SCORE_CONF_FOR_AD);
+        ScorerUtils.init("feeds_score_config_20240228.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v1.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v2.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v3.conf");

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/RegionRecallScorerV1.java

@@ -30,7 +30,7 @@ public class RegionRecallScorerV1 extends AbstractScorer4Recall {
             lists = model.kv.getOrDefault("中国", new ArrayList<>());
         }
 
-        return lists.subList(0, Math.min(100, lists.size()));
+        return lists.subList(0, Math.min(400, lists.size()));
     }
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/RegionRecallScorerV2.java

@@ -24,7 +24,7 @@ public class RegionRecallScorerV2 extends AbstractScorer4Recall {
         // todo zhangbo 这里要写实现功能
         Model4RecallList model = (Model4RecallList) this.getModel();
         List<Pair<Long, Double>> lists = model.recallList;
-        return lists.subList(0, Math.min(100, lists.size()));
+        return lists.subList(0, Math.min(160, lists.size()));
     }
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/RegionRecallScorerV3.java

@@ -30,7 +30,7 @@ public class RegionRecallScorerV3 extends AbstractScorer4Recall {
             lists = model.kv.getOrDefault("中国", new ArrayList<>());
         }
 
-        return lists.subList(0, Math.min(100, lists.size()));
+        return lists.subList(0, Math.min(160, lists.size()));
     }
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/JSONUtils.java

@@ -10,6 +10,17 @@ import org.apache.commons.lang3.StringUtils;
 public class JSONUtils {
 
 
+    public static String toJsonNew(Object obj) {
+        if (obj == null) {
+            return "";
+        }
+        try {
+            return JSONObject.toJSONString(obj);
+        } catch (Exception e) {
+            log.error("toJson exception", e);
+            return "";
+        }
+    }
     public static String toJson(Object obj) {
         if (obj == null) {
             return "";

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/TraceUtils.java

@@ -17,6 +17,10 @@ public class TraceUtils {
         MDC.put(TraceUtils.TRACE_ID_KEY, genTraceId());
     }
 
+    public static String currentTraceId() {
+        return MDC.get(TraceUtils.TRACE_ID_KEY);
+    }
+
     public static void removeMDC() {
         MDC.remove(TraceUtils.TRACE_ID_KEY);
     }

recommend-server-service/src/main/resources/application-dev.yml

@@ -46,10 +46,23 @@ spring:
         max-wait: -1
         max-idle: 8
         min-idle: 0
+  filter-redis:
+    hostName: r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com
+    port: 6379
+    password: Wqsd@2019
+    timeout: 1000
+    lettuce:
+      pool:
+        max-active: 8
+        max-wait: -1
+        max-idle: 8
+        min-idle: 0
   data:
     mongodb:
-      host: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
-      port: 3717
+      clusterHost: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
+      clusterPort: 3717
+      secondaryHost: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
+      secondaryPort: 3717
       database: longvideo
       username: lv
       password: lv@2018

recommend-server-service/src/main/resources/application-pre.yml

@@ -46,13 +46,26 @@ spring:
         max-wait: -1
         max-idle: 8
         min-idle: 0
+  filter-redis:
+    hostName: r-bp109v1gjl63qi00cb.redis.rds.aliyuncs.com
+    port: 6379
+    password: Wqsd@2019
+    timeout: 1000
+    lettuce:
+      pool:
+        max-active: 8
+        max-wait: -1
+        max-idle: 8
+        min-idle: 0
   data:
     mongodb:
-      host: dds-bp1982ea69508a541455.mongodb.rds.aliyuncs.com
-      port: 3717
+      clusterHost: s-bp14ce206f81b754.mongodb.rds.aliyuncs.com
+      clusterPort: 3717
+      secondaryHost: s-bp137073555e7bc4.mongodb.rds.aliyuncs.com
+      secondaryPort: 3717
       database: longvideo
       username: lv
-      password: lv@2018
+      password: lv2018
       # 连接池
       maxConnectionIdleTime: 180000
       # maxConnectionLifeTime: 30000

recommend-server-service/src/main/resources/application-prod.yml

@@ -46,13 +46,26 @@ spring:
         max-wait: -1
         max-idle: 8
         min-idle: 0
+  filter-redis:
+    hostName: r-bp109v1gjl63qi00cb.redis.rds.aliyuncs.com
+    port: 6379
+    password: Wqsd@2019
+    timeout: 1000
+    lettuce:
+      pool:
+        max-active: 8
+        max-wait: -1
+        max-idle: 8
+        min-idle: 0
   data:
     mongodb:
-      host: dds-bp1982ea69508a541455.mongodb.rds.aliyuncs.com
-      port: 3717
+      clusterHost: s-bp14ce206f81b754.mongodb.rds.aliyuncs.com
+      clusterPort: 3717
+      secondaryHost: s-bp137073555e7bc4.mongodb.rds.aliyuncs.com
+      secondaryPort: 3717
       database: longvideo
       username: lv
-      password: lv@2018
+      password: lv2018
       # 连接池
       maxConnectionIdleTime: 180000
       # maxConnectionLifeTime: 30000

recommend-server-service/src/main/resources/application-test.yml

@@ -14,8 +14,10 @@ eureka:
 
 spring:
   redis:
-    hostName: r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com
-    port: 6379
+    clusterHost: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
+    clusterPort: 3717
+    secondaryHost: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com
+    secondaryPort: 3717
     password: Wqsd@2019
     timeout: 1000
     lettuce:
@@ -46,6 +48,17 @@ spring:
         max-wait: -1
         max-idle: 8
         min-idle: 0
+  filter-redis:
+    hostName: r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com
+    port: 6379
+    password: Wqsd@2019
+    timeout: 1000
+    lettuce:
+      pool:
+        max-active: 8
+        max-wait: -1
+        max-idle: 8
+        min-idle: 0
   data:
     mongodb:
       host: dds-bp1de4fc73029b241978.mongodb.rds.aliyuncs.com

recommend-server-service/src/main/resources/feeds_score_config_20240228.conf

@@ -0,0 +1,12 @@
+scorer-config = {
+  str-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogShareLRScorer"
+    scorer-priority = 99
+    model-path = "zhangbo/model_tom.txt"
+  }
+  ros-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogShareLRScorer4Ros"
+    scorer-priority = 99
+    model-path = "zhangbo/model_jerry.txt"
+  }
+}