add deepfm

config.py

@@ -162,7 +162,7 @@ class BaseConfig(object):
             'abtest_368': 60063,
             'abtest_373': 60064,
             'abtest_393': 60065,
-            #'abtest_394': 60066,
+            'abtest_394': 60066,
         },  # 地域分组小时级规则实验
 
         'rank_by_24h': {
@@ -410,6 +410,10 @@ class BaseConfig(object):
         '393': {
             'data_key': 'data10', 'rule_key': 'rule7',
             'ab_code': AB_CODE['region_rank_by_h'].get('abtest_393')
+        },
+        '394': {
+            'data_key': 'data10', 'rule_key': 'rule7',
+            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_394')
         }
     }
 
@@ -1295,6 +1299,9 @@ class DevelopmentConfig(BaseConfig):
     # 获取用户近30天是否有回流接口地址
     GET_USER_30DayReturnCnt_URL = 'http://videotest-internal.yishihui.com/longvideoapi/openapi/user/has30DayReturnCnt?mid='
 
+    #tf serving url
+    TF_SERVING_URL = 'http://localhost:8501/v1/models/deepfm:predict'
+
     # 日志服务配置
     ALIYUN_LOG = {
         'ENDPOINT': 'cn-hangzhou-intranet.log.aliyuncs.com',
@@ -1350,6 +1357,9 @@ class TestConfig(BaseConfig):
     # 获取用户近30天是否有回流接口地址
     GET_USER_30DayReturnCnt_URL = 'http://videotest-internal.yishihui.com/longvideoapi/openapi/user/has30DayReturnCnt?mid='
 
+    # tf serving url
+    TF_SERVING_URL = 'http://localhost:8501/v1/models/deepfm:predict'
+
     # 日志服务配置
     ALIYUN_LOG = {
         'ENDPOINT': 'cn-hangzhou-intranet.log.aliyuncs.com',
@@ -1405,6 +1415,9 @@ class PreProductionConfig(BaseConfig):
     # 获取用户近30天是否有回流接口地址
     GET_USER_30DayReturnCnt_URL = 'http://videopre-internal.piaoquantv.com/longvideoapi/openapi/user/has30DayReturnCnt?mid='
 
+    # tf serving url
+    TF_SERVING_URL = 'http://localhost:8501/v1/models/deepfm:predict'
+
     # 日志服务配置
     ALIYUN_LOG = {
         'ENDPOINT': 'cn-hangzhou-intranet.log.aliyuncs.com',
@@ -1460,6 +1473,9 @@ class ProductionConfig(BaseConfig):
     # 获取用户近30天是否有回流接口地址
     GET_USER_30DayReturnCnt_URL = 'http://longvideoapi-internal.piaoquantv.com/longvideoapi/openapi/user/has30DayReturnCnt?mid='
 
+    # tf serving url
+    TF_SERVING_URL = 'http://localhost:8501/v1/models/deepfm:predict'
+
     # 日志服务配置
     ALIYUN_LOG = {
         'ENDPOINT': 'cn-hangzhou-intranet.log.aliyuncs.com',

rank_service.py

@@ -0,0 +1,171 @@
+#coding utf-8
+from db_helper import RedisHelper
+from config import set_config
+from  utils import request_post
+import json
+
+config_ = set_config()
+def get_featurs(data, size, top_K, flow_pool_P, env_dict):
+    recall_list = []
+    vidKeys = []
+    feature_dict = {}
+    # defult value
+    apptype = 0
+    pagesource = ""
+    versioncode = ""
+    machineinfo_brand=""
+    recommendsource = ""
+    machineinfo_platform = ''
+    sencetype = ''
+    mid = ''
+    if env_dict:
+        apptype = env_dict.get('app_type','')
+        pagesource = env_dict.get('versioncode', '')
+        versioncode = env_dict.get('versioncode', '')
+        machineinfo_brand = env_dict.get('machineinfo_brand', '')
+        machineinfo_model = env_dict.get('machineinfo_model', '')
+        recommendsource = env_dict.get('recommendsource', '')
+        machineinfo_platform = env_dict.get('machineinfo_platform', '')
+        sencetype = env_dict.get('sencetype', '')
+    redisObj = RedisHelper()
+    pre_str = "v_ctr:"
+    mid_list = []
+    videoid_list = []
+    apptype_list = []
+    pagesource_list = []
+    versioncode_list = []
+    machineinfo_brand_list = []
+    machineinfo_model_list = []
+    recommendsource_list = []
+    machineinfo_platform_list = []
+    sencetype_list = []
+    day_rov_list = []
+    day_share_return_score_list = []
+    day_share_score_list = []
+    day_return_rate_list = []
+    day_ctr_score_list = []
+    day_play_pv_list = []
+    day_play_users_list = []
+    day_return_users_list = []
+    day_share_pv_list = []
+    day_view_pv_list = []
+    day_view_users_list = []
+    day_share_users = []
+    for recall_item in data['rov_pool_recall']:
+        if len(recall_item)<=0:
+            continue
+        vid = recall_item.get("videoId",0)
+        mid_list.append(mid)
+        videoid_list.append(vid)
+        apptype_list.append(apptype)
+        pagesource_list.append(pagesource)
+        versioncode_list.append(versioncode)
+        machineinfo_brand_list.append(machineinfo_brand)
+        machineinfo_model_list.append(machineinfo_model)
+        recommendsource_list.append(recommendsource)
+        machineinfo_platform_list.append(machineinfo_platform)
+        sencetype_list.append(sencetype)
+        vidKeys.append(pre_str + str(vid))
+        recall_list.append(recall_item)
+    video_static_info = redisObj.get_batch_key(vidKeys)
+    if video_static_info:
+        for i in range(len(video_static_info)):
+            try:
+                # print(video_scores[i])
+                vid = vidKeys[i].replace(pre_str,"")
+                if video_static_info[i] :
+                    per_video_staic = json.loads(video_static_info[i])
+                    if vid in feature_dict:
+                        vid_fea_info = feature_dict[vid]
+                        day_rov_list.append(per_video_staic[0])
+                        day_share_return_score_list.append(per_video_staic[1])
+                        day_share_score_list.append(per_video_staic[2])
+                        day_return_rate_list.append(per_video_staic[3])
+                        day_ctr_score_list.append(per_video_staic[4])
+                        day_view_users_list.append(per_video_staic[5])
+                        day_view_pv_list.append(per_video_staic[6])
+                        day_play_users_list.append(per_video_staic[7])
+                        day_play_pv_list.append(per_video_staic[8])
+                        day_share_users.append(per_video_staic[9])
+                        day_share_pv_list.append(per_video_staic[10])
+                        day_return_users_list.append(per_video_staic[11])
+            except Exception:
+                day_rov_list.append(0.0)
+                day_share_return_score_list.append(0.0)
+                day_share_score_list.append(0.0)
+                day_return_rate_list.append(0.0)
+                day_ctr_score_list.append(0.0)
+                day_view_users_list.append(0)
+                day_view_pv_list.append(0)
+                day_play_users_list.append(0)
+                day_play_pv_list.append(0)
+                day_share_users.append(0)
+                day_share_pv_list.append(0)
+                day_return_users_list.append(0)
+    feature_dict["mid_list"] = mid_list
+    feature_dict["videoid_list"] = videoid_list
+    feature_dict["apptype_list"]  = apptype_list
+    feature_dict["pagesource_list"] = pagesource_list
+    feature_dict["versioncode_list"] = versioncode_list
+    feature_dict["machineinfo_brand_list"] = machineinfo_brand_list
+    feature_dict["machineinfo_model_list"] = machineinfo_model_list
+    feature_dict["recommendsource_list"] = recommendsource_list
+    feature_dict["machineinfo_model_list"] = machineinfo_model_list
+    feature_dict["machineinfo_platform_list"] = machineinfo_platform_list
+    feature_dict["sencetype_lis"] = sencetype_list
+    feature_dict["day_rov_list"] = day_rov_list
+    feature_dict["day_share_return_score_list"] = day_share_return_score_list
+    feature_dict["day_share_score_list"] = day_share_score_list
+    feature_dict["day_return_rate_list"] = day_return_rate_list
+    feature_dict["day_ctr_score_list"] = day_ctr_score_list
+    feature_dict["day_ctr_score_list"] = day_ctr_score_list
+    feature_dict["day_play_pv_list"] = day_play_pv_list
+    feature_dict["day_play_users_list"] = day_play_users_list
+    feature_dict["day_return_users_list"] = day_return_users_list
+    feature_dict["day_share_pv_list"] = day_share_pv_list
+    feature_dict["day_view_pv_list"] = day_view_pv_list
+    feature_dict["day_view_users_list"] = day_view_users_list
+    feature_dict["day_share_users_list"] = day_share_users
+    return feature_dict
+
+def get_tf_serving_sores(feature_dict):
+    #for k, v in feature_dict:
+    inputs_data = {
+                    "mid": feature_dict["mid_list"],
+                    "videoid": feature_dict["videoid_list"],
+                    "apptype": feature_dict["apptype_list"],
+                    "pagesource": feature_dict["pagesource_list"],
+                    "versioncode": feature_dict["versioncode_list"],
+                    "machineinfo_brand": feature_dict["machineinfo_brand_list"],
+                    "machineinfo_model": feature_dict["machineinfo_model_list"],
+                    "recommendsource": feature_dict["recommendsource_list"],
+                    "machineinfo_platform": feature_dict["machineinfo_platform_list"],
+                    "sencetype": feature_dict["sencetype_list"],
+                    "day_rov": feature_dict["day_rov_list"],
+                    "day_share_return_score": feature_dict["day_share_return_score_list"],
+                    "day_share_score": feature_dict["day_share_score_list"],
+                    "day_return_rate": feature_dict["day_return_rate_list"],
+                    "day_ctr_score": feature_dict["day_ctr_score_list"],
+                    "day_play_pv": feature_dict["day_play_pv_list"],
+                    "day_play_users": feature_dict["day_play_users_list"],
+                    "day_share_pv": feature_dict["day_share_pv_list"],
+                    "day_return_users": feature_dict["day_return_users_list"],
+                    "day_share_users": feature_dict["day_share_users_list"],
+                    "day_view_pv": feature_dict["day_view_pv_list"],
+                    "day_view_users":feature_dict["day_view_users_list"]
+    }
+    request_data= {}
+    request_data["inputs"] = inputs_data
+    print(request_data)
+    # 调用http接口
+    result = request_post(request_url=config_.TF_SERVING_URL, request_data=request_data, timeout=(0.1, 1))
+
+    # print("result:", result)
+    if result is None:
+        # print("result is None")
+        # log_.info('过滤失败，types: {}'.format(types))
+        return []
+
+    print(result)
+
+    filtered_videos = result['data']

recommend.py

@@ -1628,7 +1628,8 @@ def video_homepage_recommend(request_id, mid, uid, size, app_type, algo_type,
     # return video, return video2
     # old video: 60056, test2
     elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 \
-            or ab_code == 60055 or ab_code == 60056 or ab_code==60057 or ab_code==60065:
+            or ab_code == 60055 or ab_code == 60056 or ab_code==60057  \
+            or ab_code==60065 or ab_code == 60066:
         result = video_old_recommend(request_id=request_id,
                                      mid=mid, uid=uid, app_type=app_type,
                                      size=size, top_K=top_K, flow_pool_P=flow_pool_P,
@@ -1746,7 +1747,7 @@ def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_
                                  old_video_index=old_video_index, video_id=video_id,
                                  params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
     elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or \
-            ab_code == 60056 or ab_code==60057 or ab_code ==60065:
+            ab_code == 60056 or ab_code==60057 or ab_code ==60065 or ab_code ==60066:
         result = video_old_recommend(request_id=request_id,
                                  mid=mid, uid=uid, app_type=app_type,
                                  size=size, top_K=top_K, flow_pool_P=flow_pool_P,

video_rank.py

@@ -7,6 +7,7 @@ from config import set_config
 from video_recall import PoolRecall
 from db_helper import RedisHelper
 from utils import FilterVideos, send_msg_to_feishu
+from  rank_service import get_featurs, get_tf_serving_sores
 
 log_ = Log()
 config_ = set_config()
@@ -604,9 +605,7 @@ def video_new_rank2(data, size, top_K, flow_pool_P, ab_code, exp_config=None):
     if not data['rov_pool_recall'] and not data['flow_pool_recall']:
         return [], 0
 
-    redisObj = RedisHelper()
-    vidKeys = []
-    recall_list = []
+
     #全量的是vlog,票圈精选， 334，60057,
     # 60054: simrecall,
     # 60052: 票圈精选，融合排序，60053:空置
@@ -626,26 +625,34 @@ def video_new_rank2(data, size, top_K, flow_pool_P, ab_code, exp_config=None):
     elif ab_code == 60056:
         pre_str = "k_p7:"
     #print("pre_str:", pre_str)
-    for recall_item in data['rov_pool_recall']:
-        if len(recall_item)<=0:
-            continue
-        vid = recall_item.get("videoId",0)
-        vidKeys.append(pre_str+ str(vid))
-        recall_list.append(recall_item)
-    #print("vidKeys:", vidKeys)
-    video_scores = redisObj.get_batch_key(vidKeys)
-    #print("video_score:",video_scores)
-    for i in range(len(video_scores)):
-         try:
-            # print(video_scores[i])
-            if video_scores[i] is None:
-                recall_list[i]['sort_score']= 0.0
-            else:
-                video_score_str = json.loads(video_scores[i])
-                #print("video_score_str:", video_score_str)
-                recall_list[i]['sort_score'] = video_score_str[0]
-         except Exception :
-                recall_list[i]['sort_score'] = 0.0
+    recall_list = []
+    #call rank service
+    if ab_code == 60066:
+        get_featurs()
+
+    else:
+        redisObj = RedisHelper()
+        vidKeys = []
+        for recall_item in data['rov_pool_recall']:
+            if len(recall_item)<=0:
+                continue
+            vid = recall_item.get("videoId",0)
+            vidKeys.append(pre_str+ str(vid))
+            recall_list.append(recall_item)
+        #print("vidKeys:", vidKeys)
+        video_scores = redisObj.get_batch_key(vidKeys)
+        #print("video_score:",video_scores)
+        for i in range(len(video_scores)):
+             try:
+                # print(video_scores[i])
+                if video_scores[i] is None:
+                    recall_list[i]['sort_score']= 0.0
+                else:
+                    video_score_str = json.loads(video_scores[i])
+                    #print("video_score_str:", video_score_str)
+                    recall_list[i]['sort_score'] = video_score_str[0]
+             except Exception :
+                    recall_list[i]['sort_score'] = 0.0
     #sort_items = sorted(video_items, key=lambda k: k[1], reverse=True)
     rov_recall_rank =sorted(recall_list, key=lambda k: k.get('sort_score', 0), reverse=True)
     #print(rov_recall_rank)